Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.1 KB
Raw Blame
import { strToU8, zipSync } from 'fflate'
import { describe, expect, it, vi } from 'vitest'

import { OmnibusSkillsCache } from '@/hono/cache/OmnibusSkillsCache'
import type { RedisLike } from '@/hono/cache/RedisCache'
import { hashBytes } from '@/hono/cache/SharedBlobCache'
import { SkillArchiveCache, type SkillArchiveFetchResult } from '@/hono/cache/SkillArchiveCache'
import { skillArchiveLastValidatedTimestampSeconds } from '@/hono/metrics'
import { SkillCatalogService } from '@/hono/skill-catalog-service'

import { makeSharedBlobRedisStubs } from '../hono/helpers/shared-blob-redis-stubs'

interface MockRedis extends RedisLike {
    store: Map<string, string>
}

function makeRedis(): MockRedis {
    const store = new Map<string, string>()
    return {
        store,
        get: vi.fn(async (key: string) => store.get(key) ?? null),
        set: vi.fn(async (key: string, value: string, ...args: (string | number)[]) => {
            if (args.includes('NX') && store.has(key)) {
                return null
            }
            store.set(key, value)
            return 'OK'
        }),
        del: vi.fn(async (...keys: string[]) => keys.filter((key) => store.delete(key)).length),
        scan: vi.fn(async () => ['0', []] as [string, string[]]),
        incr: vi.fn(async () => 1),
        ...makeSharedBlobRedisStubs(store),
    }
}

function makeArchive(seed = 'sample'): Uint8Array {
    return zipSync(
        {
            [`${seed}/SKILL.md`]: strToU8(
                `---\nname: ${seed}\ndescription: Sample skill for tests.\n---\n\n# ${seed}\n\nUse the ${seed}.`
            ),
        },
        { mtime: new Date('2026-01-01T00:00:00Z') }
    )
}

function downloaded(bytes: Uint8Array = makeArchive(), etag?: string): SkillArchiveFetchResult {
    return { status: 'downloaded', bytes, etag }
}

const CURRENT_KEY = 'mcp:shared-blob:{product-skills}:v3:current'
const LOCK_KEY = 'mcp:shared-blob:{product-skills}:v3:lock'

function current(redis: MockRedis): { sha: string; etag?: string; validatedAt: number } {
    return JSON.parse(redis.store.get(CURRENT_KEY)!)
}

function bytesKey(redis: MockRedis): string {
    return `mcp:shared-blob:{product-skills}:v3:blob:${current(redis).sha}`
}

function expireSharedCopy(redis: MockRedis): void {
    redis.store.set(CURRENT_KEY, JSON.stringify({ ...current(redis), validatedAt: Date.now() - 600_001 }))
}

describe('SkillArchiveCache', () => {
    it('downloads once for a fleet and serves the shared copy to every later pod', async () => {
        const redis = makeRedis()
        const fetchArchive = vi.fn(async () => downloaded())
        const first = new SkillArchiveCache(redis, { fetchArchive })
        const second = new SkillArchiveCache(redis, { fetchArchive })

        const cold = await first.loadOrRefresh()
        const warm = await second.loadOrRefresh()

        expect(cold.result).toBe('cold_refresh')
        expect(warm).toMatchObject({ result: 'fresh_hit', sha: cold.sha })
        expect(fetchArchive).toHaveBeenCalledTimes(1)
        expect(current(redis).sha).toBe(cold.sha)
    })

    it('serves a stale shared copy at startup without touching the source', async () => {
        const redis = makeRedis()
        const fetchArchive = vi.fn(async () => downloaded())
        const cache = new SkillArchiveCache(redis, { fetchArchive })
        await cache.loadOrRefresh()
        expireSharedCopy(redis)

        await expect(cache.loadOrRefresh()).resolves.toMatchObject({ result: 'stale_hit' })
        expect(fetchArchive).toHaveBeenCalledTimes(1)
    })

    it('starts a separate cache without trusting legacy split-key metadata', async () => {
        const redis = makeRedis()
        redis.store.set('mcp:shared-blob:product-skills:bytes', Buffer.from('old bytes').toString('base64'))
        redis.store.set('mcp:shared-blob:product-skills:sha', 'mismatched-sha')
        const cache = new SkillArchiveCache(redis, { fetchArchive: vi.fn(async () => downloaded()) })
        const loaded = await cache.loadOrRefresh()

        expect(loaded.result).toBe('cold_refresh')
        expect(current(redis).sha).toBe(loaded.sha)
        expect(redis.store.get('mcp:shared-blob:product-skills:sha')).toBe('mismatched-sha')
    })

    it('isolates custom archive URLs from the published bundle cache', async () => {
        const redis = makeRedis()
        const fetchArchive = vi.fn(async () => downloaded())
        const first = new SkillArchiveCache(redis, { archiveUrl: 'http://localhost/skills.zip?v=one', fetchArchive })
        const second = new SkillArchiveCache(redis, { archiveUrl: 'http://localhost/skills.zip?v=two', fetchArchive })

        await expect(first.loadOrRefresh()).resolves.toMatchObject({ result: 'cold_refresh' })
        await expect(second.loadOrRefresh()).resolves.toMatchObject({ result: 'cold_refresh' })
        expect(first.currentKey).not.toBe(second.currentKey)
        expect(fetchArchive).toHaveBeenCalledTimes(2)
    })

    // The July incident: every handshake pulled the whole archive out of Redis. A pod
    // that already holds the current sha must read only the small keys.
    it('reads no archive bytes when the shared sha matches what the pod holds', async () => {
        const redis = makeRedis()
        const cache = new SkillArchiveCache(redis, { fetchArchive: vi.fn(async () => downloaded()) })
        const { sha } = await cache.loadOrRefresh()
        vi.mocked(redis.get).mockClear()

        await expect(cache.readIfChanged(sha)).resolves.toBeNull()
        expect(redis.get).not.toHaveBeenCalledWith(bytesKey(redis))
    })

    it('returns the new bytes when another pod published a different sha', async () => {
        const redis = makeRedis()
        const cache = new SkillArchiveCache(redis, {
            fetchArchive: vi.fn(async () => downloaded(makeArchive('first'))),
        })
        const { sha } = await cache.loadOrRefresh()
        const publisher = new SkillArchiveCache(redis, {
            fetchArchive: vi.fn(async () => downloaded(makeArchive('second'), 'etag-v2')),
        })
        expireSharedCopy(redis)
        await expect(publisher.refreshIfStale()).resolves.toBe('downloaded')

        const changed = await cache.readIfChanged(sha)

        expect(changed?.sha).not.toBe(sha)
        expect(changed?.bytes).toEqual(makeArchive('second'))
    })

    it('keeps a read consistent when a writer publishes between the pointer and blob reads', async () => {
        const redis = makeRedis()
        const firstBytes = makeArchive('first')
        const secondBytes = makeArchive('second')
        const reader = new SkillArchiveCache(redis, { fetchArchive: vi.fn(async () => downloaded(firstBytes)) })
        const first = await reader.loadOrRefresh()
        const publisher = new SkillArchiveCache(redis, { fetchArchive: vi.fn(async () => downloaded(secondBytes)) })
        expireSharedCopy(redis)
        let publishDuringRead = true
        vi.mocked(redis.get).mockImplementation(async (key) => {
            const value = redis.store.get(key) ?? null
            if (key === CURRENT_KEY && publishDuringRead) {
                publishDuringRead = false
                await publisher.refreshIfStale()
            }
            return value
        })

        expect(await reader.readIfChanged(undefined)).toEqual({ bytes: firstBytes, sha: first.sha })
        expect(await reader.readIfChanged(first.sha)).toEqual({ bytes: secondBytes, sha: current(redis).sha })
    })

    describe('refreshIfStale', () => {
        it('does nothing while the shared copy is fresh', async () => {
            const redis = makeRedis()
            const fetchArchive = vi.fn(async () => downloaded())
            const cache = new SkillArchiveCache(redis, { fetchArchive })
            await cache.loadOrRefresh()

            await expect(cache.refreshIfStale()).resolves.toBe('fresh')
            expect(fetchArchive).toHaveBeenCalledTimes(1)
        })

        it('revalidates a stale copy with If-None-Match and only bumps freshness on 304', async () => {
            const redis = makeRedis()
            const archive = makeArchive()
            const fetchArchive = vi.fn(
                async (_url: string, etag?: string): Promise<SkillArchiveFetchResult> =>
                    etag ? { status: 'not_modified' } : downloaded(archive, 'etag-v1')
            )
            const cache = new SkillArchiveCache(redis, { fetchArchive })
            const { sha } = await cache.loadOrRefresh()
            const originalBytes = redis.store.get(bytesKey(redis))
            expireSharedCopy(redis)

            await expect(cache.refreshIfStale()).resolves.toBe('not_modified')

            expect(fetchArchive).toHaveBeenLastCalledWith(expect.any(String), 'etag-v1')
            expect(current(redis).validatedAt).toBeGreaterThan(Date.now() - 1000)
            expect(redis.store.get(bytesKey(redis))).toBe(originalBytes)
            expect(current(redis).sha).toBe(sha)
        })

        it('replaces bytes, etag and sha when the source has a new archive', async () => {
            const redis = makeRedis()
            const second = makeArchive('second')
            const fetchArchive = vi.fn(
                async (_url: string, etag?: string): Promise<SkillArchiveFetchResult> =>
                    etag ? downloaded(second, 'etag-v2') : downloaded(makeArchive('first'), 'etag-v1')
            )
            const cache = new SkillArchiveCache(redis, { fetchArchive })
            const { sha } = await cache.loadOrRefresh()
            expireSharedCopy(redis)

            await expect(cache.refreshIfStale()).resolves.toBe('downloaded')

            expect(current(redis).etag).toBe('etag-v2')
            expect(redis.store.get(bytesKey(redis))).toBe(Buffer.from(second).toString('base64'))
            expect(current(redis).sha).not.toBe(sha)
        })

        it('never sends If-None-Match when the server stored no validator', async () => {
            const redis = makeRedis()
            const fetchArchive = vi.fn(async (_url: string, _etag?: string) => downloaded())
            const cache = new SkillArchiveCache(redis, { fetchArchive })
            await cache.loadOrRefresh()
            expect(current(redis).etag).toBeUndefined()
            expireSharedCopy(redis)

            await cache.refreshIfStale()

            expect(fetchArchive).toHaveBeenCalledTimes(2)
            for (const [, sentEtag] of fetchArchive.mock.calls) {
                expect(sentEtag).toBeUndefined()
            }
        })

        it('leaves the network to the pod holding the writer lock', async () => {
            const redis = makeRedis()
            const fetchArchive = vi.fn(async () => downloaded())
            const cache = new SkillArchiveCache(redis, { fetchArchive })
            await cache.loadOrRefresh()
            expireSharedCopy(redis)
            redis.store.set(LOCK_KEY, 'another-pod')

            await expect(cache.refreshIfStale()).resolves.toBe('lock_busy')
            expect(fetchArchive).toHaveBeenCalledTimes(1)
        })

        it('reports a missing shared copy instead of downloading outside the startup path', async () => {
            const cache = new SkillArchiveCache(makeRedis(), { fetchArchive: vi.fn(async () => downloaded()) })

            await expect(cache.refreshIfStale()).resolves.toBe('missing')
        })

        it.each(['before revalidation', 'during revalidation'])('repairs bytes evicted %s', async (when) => {
            const redis = makeRedis()
            const archive = makeArchive()
            const fetchArchive = vi.fn(async (_url: string, etag?: string): Promise<SkillArchiveFetchResult> => {
                if (etag) {
                    redis.store.delete(bytesKey(redis))
                    return { status: 'not_modified' }
                }
                return downloaded(archive, 'etag-v1')
            })
            const cache = new SkillArchiveCache(redis, { fetchArchive })
            await cache.loadOrRefresh()
            expireSharedCopy(redis)
            const pointer = redis.store.get(CURRENT_KEY)
            if (when === 'before revalidation') {
                redis.store.delete(bytesKey(redis))
            }

            await expect(cache.refreshIfStale()).resolves.toBe('missing')
            expect(redis.store.get(CURRENT_KEY)).toBe(pointer)
            await expect(cache.loadOrRefresh()).resolves.toMatchObject({ bytes: archive, result: 'cold_refresh' })
            expect(fetchArchive).toHaveBeenLastCalledWith(expect.any(String), undefined)
        })

        it('does not let a delayed 304 replace a newer pointer after the writer lock expires', async () => {
            const redis = makeRedis()
            let signalStarted!: () => void
            const started = new Promise<void>((resolve) => {
                signalStarted = resolve
            })
            let signalFinished!: () => void
            const finish = new Promise<void>((resolve) => {
                signalFinished = resolve
            })
            const first = new SkillArchiveCache(redis, {
                fetchArchive: async (_url, etag) => {
                    if (etag) {
                        signalStarted()
                        await finish
                        return { status: 'not_modified' }
                    }
                    return downloaded(makeArchive('first'), 'etag-v1')
                },
            })
            await first.loadOrRefresh()
            expireSharedCopy(redis)
            const pending = first.refreshIfStale()
            await started
            redis.store.delete(LOCK_KEY)
            const second = new SkillArchiveCache(redis, {
                fetchArchive: async () => downloaded(makeArchive('second'), 'etag-v2'),
            })
            await second.refreshIfStale()
            const pointer = redis.store.get(CURRENT_KEY)
            signalFinished()

            await expect(pending).resolves.toBe('missing')
            expect(redis.store.get(CURRENT_KEY)).toBe(pointer)
            expect((await first.loadOrRefresh()).bytes).toEqual(makeArchive('second'))
        })
    })

    it('keeps the previous shared copy when a downloaded archive fails validation', async () => {
        const redis = makeRedis()
        let release = 'good'
        const fetchArchive = vi.fn(async (): Promise<SkillArchiveFetchResult> => {
            return release === 'good' ? downloaded(makeArchive('good'), 'etag-good') : downloaded(strToU8('not a zip'))
        })
        const cache = new SkillArchiveCache(redis, {
            fetchArchive,
            validateArchive: (bytes) => {
                if (bytes.length < 100) {
                    throw new Error('corrupt archive')
                }
            },
        })
        const { sha } = await cache.loadOrRefresh()
        release = 'bad'
        expireSharedCopy(redis)

        await expect(cache.refreshIfStale()).rejects.toThrow('corrupt archive')

        expect(current(redis).sha).toBe(sha)
        expect(current(redis).etag).toBe('etag-good')
        // The lock is released, so the next stale poll can try the source again.
        expect(redis.store.has(LOCK_KEY)).toBe(false)
    })
})

describe('SkillCatalogService', () => {
    function quietConsole(): ReturnType<typeof vi.spyOn> {
        return vi.spyOn(console, 'error').mockImplementation(() => undefined)
    }

    it.each([true, false])('checks the pinned archive before startup (matching=%s)', async (matching) => {
        const bytes = makeArchive()
        const service = new SkillCatalogService(makeRedis(), {
            fetchArchive: async () => downloaded(bytes),
            expectedArchiveSha: matching ? hashBytes(bytes) : '0'.repeat(64),
        })
        if (matching) {
            await service.warmup()
            expect(service.getCatalog()?.size).toBe(1)
        } else {
            await expect(service.warmup()).rejects.toThrow('does not match the expected SHA-256')
            expect(service.getCatalog()).toBeUndefined()
        }
    })

    it('overlays the shared omnibus skills, follows their updates, and refuses a copy that does not parse', async () => {
        const redis = makeRedis()
        const omnibus = new OmnibusSkillsCache(redis)
        await omnibus.publish(makeArchive('instrument-logs'))
        const service = new SkillCatalogService(redis, {
            fetchArchive: async () => downloaded(makeArchive('product')),
            omnibusSkills: omnibus,
        })

        await service.warmup()
        expect(service.getCatalog()?.listNames()).toEqual(['instrument-logs', 'product'])

        await omnibus.publish(makeArchive('instrument-tracing'))
        await service.poll()
        expect(service.getCatalog()?.listNames()).toEqual(['instrument-tracing', 'product'])

        await expect(omnibus.publish(zipSync({ 'broken/notes.md': strToU8('no entry point') }))).rejects.toThrow(
            'has no SKILL.md'
        )
        await service.poll()
        expect(service.getCatalog()?.listNames()).toEqual(['instrument-tracing', 'product'])

        const pinned = new SkillCatalogService(redis, {
            fetchArchive: async () => downloaded(makeArchive('product')),
            expectedArchiveSha: hashBytes(makeArchive('product')),
            omnibusSkills: omnibus,
        })
        await pinned.warmup()
        expect(pinned.getCatalog()?.listNames()).toEqual(['product'])
    })

    it('loads a new pin at the same URL without reusing an older cached archive', async () => {
        const redis = makeRedis()
        const firstBytes = makeArchive('first')
        const secondBytes = makeArchive('second')
        const fetchArchive = vi.fn(async () => downloaded(firstBytes))
        const first = new SkillCatalogService(redis, { fetchArchive, expectedArchiveSha: hashBytes(firstBytes) })
        await first.warmup()
        fetchArchive.mockResolvedValue(downloaded(secondBytes))
        const second = new SkillCatalogService(redis, { fetchArchive, expectedArchiveSha: hashBytes(secondBytes) })

        await second.warmup()

        expect(second.getCatalog()?.listNames()).toEqual(['second'])
        expect(fetchArchive).toHaveBeenCalledTimes(2)
    })

    it('reports upstream validation time across restarts, unchanged releases and refresh failures', async () => {
        vi.useFakeTimers()
        const timestamp = vi.spyOn(skillArchiveLastValidatedTimestampSeconds, 'set')
        const consoleError = quietConsole()
        try {
            const redis = makeRedis()
            const fetchArchive = vi.fn(
                async (_url: string, etag?: string): Promise<SkillArchiveFetchResult> =>
                    etag ? { status: 'not_modified' } : downloaded(makeArchive(), 'etag-v1')
            )
            const first = new SkillCatalogService(redis, { fetchArchive })
            await first.warmup()
            const validatedAt = Date.now()
            vi.setSystemTime(validatedAt + 20 * 60_000)
            const second = new SkillCatalogService(redis, { fetchArchive })
            await second.warmup()
            expect(timestamp).toHaveBeenLastCalledWith(validatedAt / 1000)
            const catalog = second.getCatalog()

            await second.poll()
            expect(timestamp).toHaveBeenLastCalledWith(Date.now() / 1000)
            expect(second.getCatalog()).toBe(catalog)
            const revalidatedAt = Date.now()
            vi.setSystemTime(revalidatedAt + 31 * 60_000)
            fetchArchive.mockRejectedValue(new Error('offline'))

            await second.poll()
            expect(timestamp).toHaveBeenLastCalledWith(revalidatedAt / 1000)
            expect(Date.now() / 1000 - Number(timestamp.mock.lastCall![0])).toBeGreaterThan(1800)
        } finally {
            timestamp.mockRestore()
            consoleError.mockRestore()
            vi.useRealTimers()
        }
    })

    it('retries warmup inside its budget and starts without skills when the source stays down', async () => {
        const fetchArchive = vi.fn(async () => {
            throw new Error('offline')
        })
        const service = new SkillCatalogService(makeRedis(), { fetchArchive, warmupTimeoutMs: 50, warmupRetryMs: 10 })
        const consoleError = quietConsole()

        await expect(service.warmup()).resolves.toBeUndefined()

        expect(service.getCatalog()).toBeUndefined()
        expect(fetchArchive.mock.calls.length).toBeGreaterThan(1)
        expect(consoleError).toHaveBeenCalledWith(
            expect.stringContaining('starting without product skills'),
            expect.any(Error)
        )
        consoleError.mockRestore()
    })

    it.each([true, false])(
        'retries pinned archive fetch failures within the startup budget (recovers=%s)',
        async (recovers) => {
            vi.useFakeTimers()
            const consoleError = quietConsole()
            try {
                const bytes = makeArchive()
                let attempts = 0
                const fetchArchive = vi.fn(async () => {
                    attempts += 1
                    if (attempts === 1 || !recovers) {
                        throw new Error('offline')
                    }
                    return downloaded(bytes)
                })
                const service = new SkillCatalogService(makeRedis(), {
                    fetchArchive,
                    expectedArchiveSha: hashBytes(bytes),
                    warmupTimeoutMs: 50,
                    warmupRetryMs: 10,
                })
                const outcome = service.warmup().catch((error: unknown) => error)
                await vi.runAllTimersAsync()
                expect(await outcome).toEqual(recovers ? undefined : new Error('offline'))
                expect(fetchArchive.mock.calls.length).toBeGreaterThan(1)
                expect(service.getCatalog()?.size).toBe(recovers ? 1 : undefined)
            } finally {
                consoleError.mockRestore()
                vi.useRealTimers()
            }
        }
    )

    it('loads the catalog at warmup and adopts a newer archive another pod published', async () => {
        const redis = makeRedis()
        const first = new SkillCatalogService(redis, {
            fetchArchive: vi.fn(async () => downloaded(makeArchive('first'), 'etag-v1')),
        })
        await first.warmup()
        expect(first.getCatalog()?.listNames()).toEqual(['first'])

        const second = new SkillCatalogService(redis, {
            fetchArchive: vi.fn(async () => downloaded(makeArchive('second'), 'etag-v2')),
        })
        await second.warmup()
        expect(second.getCatalog()?.listNames()).toEqual(['first'])
        expireSharedCopy(redis)

        // The second pod wins the refresh and adopts its own download; the first pod
        // sees the new sha on its next poll and reparses without touching the source.
        await second.poll()
        await first.poll()

        expect(second.getCatalog()?.listNames()).toEqual(['second'])
        expect(first.getCatalog()?.listNames()).toEqual(['second'])
    })

    it('keeps the last catalog when the shared copy turns corrupt', async () => {
        const redis = makeRedis()
        const service = new SkillCatalogService(redis, { fetchArchive: vi.fn(async () => downloaded()) })
        await service.warmup()
        const original = service.getCatalog()
        const sha = 'a'.repeat(64)
        redis.store.set(CURRENT_KEY, JSON.stringify({ sha, validatedAt: Date.now() }))
        redis.store.set(bytesKey(redis), Buffer.from('not a zip').toString('base64'))
        const consoleError = quietConsole()

        await service.poll()

        expect(service.getCatalog()).toBe(original)
        expect(consoleError).toHaveBeenCalledWith(
            expect.stringContaining('keeping the current catalog'),
            expect.any(Error)
        )
        await service.poll()
        expect(service.getCatalog()?.listNames()).toEqual(['sample'])
        expect(current(redis).sha).not.toBe(sha)
        consoleError.mockRestore()
    })

    it('repairs a shared copy that Redis lost', async () => {
        const redis = makeRedis()
        const fetchArchive = vi.fn(async () => downloaded())
        const service = new SkillCatalogService(redis, { fetchArchive })
        await service.warmup()
        redis.store.clear()

        await service.poll()

        expect(fetchArchive).toHaveBeenCalledTimes(2)
        expect(redis.store.has(bytesKey(redis))).toBe(true)
        expect(service.getCatalog()?.listNames()).toEqual(['sample'])
    })
})