Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

18.8 KB
Raw Blame
import { SignJWT } from 'jose'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'
import type { AddressInfo } from 'node:net'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'

import { AUDIENCE } from '@/auth/types'
import { KubernetesSecretSource } from '@/kubernetes'
import type { Config } from '@/lib/config'
import { mountErrorsTotal, register, secretWatchRestartsTotal, servingStaleSeconds } from '@/metrics'
import { CredentialsServer } from '@/server'
import type { SecretSnapshot } from '@/source'

const NAMESPACE = 'posthog'
const SECRET_NAME = 'credentials-service-secrets'
const KEY = 'HUBSPOT_APP_CLIENT_SECRET'
const SIGNING_KEY = 'k8s-test-signing-key'
const NEW_SIGNING_KEY = 'k8s-test-signing-key-new'

const WAIT = { timeout: 2000, interval: 5 }

interface Recorded {
    query: URLSearchParams
    auth: string | undefined
}

interface Watch extends Recorded {
    res: ServerResponse
    closed: boolean
}

/** Answers the list and watch calls the source makes, the way the API server does. */
class FakeApiServer {
    secret: { resourceVersion: string; data: Record<string, string> } | null = null
    listResourceVersion = '100'
    listStatus = 200
    /** Statuses to answer the next watch connections with, instead of opening a stream. */
    watchStatuses: number[] = []
    /** While set, list responses wait for it, so a test can deliver a watch event first. */
    holdLists: Promise<void> | null = null
    /** Like `holdLists`, but only for the next list, which is answered with the secret as it is now. */
    holdNextList: Promise<void> | null = null
    readonly lists: Recorded[] = []
    readonly watches: Watch[] = []

    private readonly server: Server = createServer((req, res) => void this.handle(req, res))

    async start(): Promise<string> {
        await new Promise<void>((resolve) => this.server.listen(0, '127.0.0.1', resolve))
        return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`
    }

    async close(): Promise<void> {
        this.server.closeAllConnections()
        await new Promise<void>((resolve) => this.server.close(() => resolve()))
    }

    get openWatch(): Watch | undefined {
        return this.watches.filter((watch) => !watch.closed).at(-1)
    }

    /** Write one event to the open watch. */
    send(type: string, object: object): void {
        const watch = this.openWatch
        if (!watch) {
            throw new Error('no open watch')
        }
        watch.res.write(`${JSON.stringify({ type, object })}\n`)
    }

    private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
        const url = new URL(req.url ?? '/', 'http://fake')
        expect(url.pathname).toBe(`/api/v1/namespaces/${NAMESPACE}/secrets`)
        const recorded = { query: url.searchParams, auth: req.headers.authorization }

        if (url.searchParams.get('watch') === '1') {
            const status = this.watchStatuses.shift()
            const watch: Watch = { ...recorded, res, closed: status !== undefined }
            this.watches.push(watch)
            if (status !== undefined) {
                res.statusCode = status
                res.end()
                return
            }
            res.on('close', () => {
                watch.closed = true
            })
            res.writeHead(200, { 'Content-Type': 'application/json' })
            res.flushHeaders()
            return
        }

        this.lists.push(recorded)
        const held = this.holdNextList
        this.holdNextList = null
        const body = JSON.stringify({
            kind: 'SecretList',
            metadata: { resourceVersion: this.listResourceVersion },
            items: this.secret ? [secretObject(this.secret.resourceVersion, this.secret.data)] : [],
        })
        await held
        await this.holdLists
        if (this.listStatus !== 200) {
            res.statusCode = this.listStatus
            res.end()
            return
        }
        res.end(body)
    }
}

function secretObject(resourceVersion: string, values: Record<string, string>): object {
    return {
        metadata: { name: SECRET_NAME, namespace: NAMESPACE, resourceVersion },
        data: Object.fromEntries(Object.entries(values).map(([k, v]) => [k, Buffer.from(v).toString('base64')])),
    }
}

const cleanups: (() => Promise<void> | void)[] = []

afterEach(async () => {
    for (const cleanup of cleanups.splice(0).reverse()) {
        await cleanup()
    }
    vi.useRealTimers()
})

async function fakeApi(values: Record<string, string> | null): Promise<{ api: FakeApiServer; url: string }> {
    const api = new FakeApiServer()
    if (values) {
        api.secret = { resourceVersion: '90', data: values }
    }
    const url = await api.start()
    cleanups.push(() => api.close())
    return { api, url }
}

async function tokenFile(token: string): Promise<string> {
    const dir = await mkdtemp(join(tmpdir(), 'credentials-k8s-'))
    cleanups.push(() => rm(dir, { recursive: true, force: true }))
    const file = join(dir, 'token')
    await writeFile(file, token)
    return file
}

async function kubernetesSource(apiUrl: string, tokenPath?: string): Promise<KubernetesSecretSource> {
    const source = new KubernetesSecretSource({
        apiUrl,
        namespace: NAMESPACE,
        secretName: SECRET_NAME,
        tokenFile: tokenPath ?? (await tokenFile('token-1')),
        caFile: '/nonexistent/ca.crt',
        backoff: { initialMs: 1, maxMs: 5 },
    })
    cleanups.push(() => source.stop())
    return source
}

async function restarts(reason: string): Promise<number> {
    const values = (await secretWatchRestartsTotal.get()).values
    return values.find((v) => v.labels.reason === reason)?.value ?? 0
}

describe('kubernetes secret source', () => {
    beforeEach(() => {
        register.resetMetrics()
    })

    it('lists the secret by field selector and decodes it into one snapshot', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'sec\n', __CALLER_KEY_POSTHOG_DJANGO: SIGNING_KEY })
        const source = await kubernetesSource(url)

        expect(await source.read()).toEqual({
            values: { [KEY]: 'sec', __CALLER_KEY_POSTHOG_DJANGO: SIGNING_KEY },
            revision: '90',
        })
        // RBAC restricted by resourceNames denies a list without this selector.
        expect(api.lists[0]?.query.get('fieldSelector')).toBe(`metadata.name=${SECRET_NAME}`)
    })

    it.each([
        ['the API server fails', 500],
        ['RBAC denies the read', 403],
    ])('reads nothing when %s', async (_label, status) => {
        const { api, url } = await fakeApi({ [KEY]: 'sec' })
        api.listStatus = status
        const source = await kubernetesSource(url)

        expect(await source.read()).toBeNull()
    })

    it('watches from the listed resourceVersion and pushes each change', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const source = await kubernetesSource(url)
        await source.read()

        const pushed: (SecretSnapshot | null)[] = []
        source.watch((snapshot) => pushed.push(snapshot))
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)

        const query = api.openWatch?.query
        expect(query?.get('resourceVersion')).toBe('100')
        expect(query?.get('fieldSelector')).toBe(`metadata.name=${SECRET_NAME}`)
        expect(query?.get('allowWatchBookmarks')).toBe('true')

        api.send('MODIFIED', secretObject('101', { [KEY]: 'two' }))
        await vi.waitFor(() => expect(pushed).toEqual([{ values: { [KEY]: 'two' }, revision: '101' }]), WAIT)
    })

    it('reconnects from the last resourceVersion when the watch closes, without listing again', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const source = await kubernetesSource(url)
        await source.read()
        source.watch(() => {})
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)

        // A bookmark moves the resourceVersion forward without a change to the secret.
        api.send('BOOKMARK', { metadata: { resourceVersion: '150' } })
        await new Promise((resolve) => setImmediate(resolve))
        api.openWatch?.res.end()

        await vi.waitFor(() => expect(api.watches).toHaveLength(2), WAIT)
        expect(api.watches[1]?.query.get('resourceVersion')).toBe('150')
        expect(api.lists).toHaveLength(1)
        expect(await restarts('closed')).toBe(1)
    })

    it.each([
        ['an ERROR event with code 410', 'event'],
        ['an HTTP 410 on connect', 'status'],
    ])('lists again and watches from the new resourceVersion after %s', async (_label, how) => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const source = await kubernetesSource(url)
        await source.read()
        if (how === 'status') {
            api.watchStatuses.push(410)
        }
        const pushed: (SecretSnapshot | null)[] = []
        source.watch((snapshot) => pushed.push(snapshot))

        if (how === 'event') {
            await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)
            api.send('ERROR', { kind: 'Status', code: 410, reason: 'Expired' })
        }
        api.secret = { resourceVersion: '205', data: { [KEY]: 'two' } }
        api.listResourceVersion = '210'

        await vi.waitFor(() => expect(api.openWatch?.query.get('resourceVersion')).toBe('210'), WAIT)
        expect(api.lists).toHaveLength(2)
        expect(pushed).toEqual([{ values: { [KEY]: 'two' }, revision: '205' }])
        expect(await restarts('gone')).toBe(1)
    })

    it('backs off and retries when the watch cannot connect', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        api.watchStatuses.push(500, 500)
        const source = await kubernetesSource(url)
        await source.read()
        source.watch(() => {})

        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)
        expect(api.watches).toHaveLength(3)
        expect(api.openWatch?.query.get('resourceVersion')).toBe('100')
        expect(await restarts('error')).toBe(2)
    })

    it('reads the token file again for every request and watch connection', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const file = await tokenFile('token-1')
        const source = await kubernetesSource(url, file)

        await source.read()
        source.watch(() => {})
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)

        // Kubelet rotates the projected token in place.
        await writeFile(file, 'token-2\n')
        await source.read()
        api.openWatch?.res.end()
        await vi.waitFor(() => expect(api.watches).toHaveLength(2), WAIT)

        expect(api.lists.map((r) => r.auth)).toEqual(['Bearer token-1', 'Bearer token-2'])
        expect(api.watches.map((r) => r.auth)).toEqual(['Bearer token-1', 'Bearer token-2'])
    })

    // A watch can hang without an error. The safety read is what notices, and it must
    // replace the watch instead of leaving it stalled until the next change.
    it('replaces a watch that missed a change the safety read found', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const source = await kubernetesSource(url)
        await source.read()
        source.watch(() => {})
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)

        api.secret = { resourceVersion: '300', data: { [KEY]: 'two' } }
        api.listResourceVersion = '301'
        expect(await source.read()).toEqual({ values: { [KEY]: 'two' }, revision: '300' })

        await vi.waitFor(() => expect(api.openWatch?.query.get('resourceVersion')).toBe('301'), WAIT)
        expect(api.watches[0]?.closed).toBe(true)
        expect(await restarts('stalled')).toBe(1)
    })

    // The list started before the watch delivered a change, so its body is older. Returning
    // it would briefly undo the change, e.g. bring back a signing key a revocation removed.
    it('does not let a slower safety read undo a change the watch delivered', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'old' })
        const source = await kubernetesSource(url)
        await source.read()
        source.watch(() => {})
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)

        let release!: () => void
        api.holdLists = new Promise((resolve) => {
            release = resolve
        })
        const reading = source.read()
        await vi.waitFor(() => expect(api.lists).toHaveLength(2), WAIT)
        api.send('MODIFIED', secretObject('400', { [KEY]: 'new' }))
        await new Promise((resolve) => setTimeout(resolve, 10))
        release()

        expect(await reading).toEqual({ values: { [KEY]: 'new' }, revision: '400' })
    })

    it('discards a re-list after a 410 that a safety read overtook', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const source = await kubernetesSource(url)
        await source.read()
        api.watchStatuses.push(410)
        let release!: () => void
        api.holdNextList = new Promise((resolve) => {
            release = resolve
        })
        const pushed: (SecretSnapshot | null)[] = []
        source.watch((snapshot) => pushed.push(snapshot))
        await vi.waitFor(() => expect(api.lists).toHaveLength(2), WAIT)

        api.secret = { resourceVersion: '500', data: { [KEY]: 'two' } }
        api.listResourceVersion = '501'
        expect(await source.read()).toEqual({ values: { [KEY]: 'two' }, revision: '500' })
        release()

        await vi.waitFor(() => expect(api.openWatch?.query.get('resourceVersion')).toBe('501'), WAIT)
        expect(pushed).toEqual([])
    })

    it('closes the watch and stops reconnecting once stopped', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'one' })
        const source = await kubernetesSource(url)
        await source.read()
        source.watch(() => {})
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)

        source.stop()

        await vi.waitFor(() => expect(api.watches[0]?.closed).toBe(true), WAIT)
        await new Promise((resolve) => setTimeout(resolve, 20))
        expect(api.watches).toHaveLength(1)
        expect(api.lists).toHaveLength(1)
    })
})

describe('credentials server on the kubernetes source', () => {
    beforeEach(() => {
        register.resetMetrics()
    })

    function config(): Config {
        return {
            port: 0,
            host: '127.0.0.1',
            shutdownPrestopDelayMs: 0,
            env: 'test',
            mountDir: '/nonexistent/mount',
            reloadSeconds: 3600,
            metricsPort: 0,
        }
    }

    async function mint(signingKey: string): Promise<string> {
        return new SignJWT({ caller: 'warehouse-sources', keys: [KEY] })
            .setProtectedHeader({ alg: 'HS256' })
            .setAudience(AUDIENCE)
            .setIssuedAt()
            .setExpirationTime('5m')
            .sign(new TextEncoder().encode(signingKey))
    }

    async function boot(apiUrl: string): Promise<{
        server: CredentialsServer
        resolve: (signingKey: string) => Promise<Response>
    }> {
        let fetch!: (req: Request) => Response | Promise<Response>
        const server = new CredentialsServer(config(), {
            source: await kubernetesSource(apiUrl),
            serve: (options) => {
                fetch = options.fetch as typeof fetch
                return { close: (cb) => cb() }
            },
            exit: () => {},
        })
        await server.start()
        cleanups.push(() => server.stop('SIGTERM'))
        const resolve = async (signingKey: string): Promise<Response> =>
            await fetch(
                new Request('http://svc/v1/secrets/resolve', {
                    method: 'POST',
                    headers: { Authorization: `Bearer ${await mint(signingKey)}` },
                })
            )
        return { server, resolve }
    }

    // One edit of the secret can revoke a caller and rotate a credential. Both must come
    // from the same snapshot, so the revoked caller never reads the new value.
    it('applies a signing-key revocation and a credential change from one watch event', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'value-1', __CALLER_KEY_TEST_DEPLOYMENT: SIGNING_KEY })
        const { server, resolve } = await boot(url)

        expect(server.lifecycleState().ready).toBe(true)
        expect(await (await resolve(SIGNING_KEY)).json()).toMatchObject({ secrets: { [KEY]: { value: 'value-1' } } })

        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)
        api.send('MODIFIED', secretObject('101', { [KEY]: 'value-2', __CALLER_KEY_TEST_DEPLOYMENT: NEW_SIGNING_KEY }))

        await vi.waitFor(async () => expect((await resolve(SIGNING_KEY)).status).toBe(401), WAIT)
        expect(await (await resolve(NEW_SIGNING_KEY)).json()).toMatchObject({
            secrets: { [KEY]: { value: 'value-2' } },
        })
    })

    it('stays not-ready while the API server is down at boot, then becomes ready', async () => {
        const { api, url } = await fakeApi({ [KEY]: 'value-1', __CALLER_KEY_TEST_DEPLOYMENT: SIGNING_KEY })
        api.listStatus = 503
        const { server, resolve } = await boot(url)

        expect(server.lifecycleState().ready).toBe(false)
        expect((await resolve(SIGNING_KEY)).status).toBe(401)

        api.listStatus = 200
        await vi.waitFor(() => expect(server.lifecycleState().ready).toBe(true), WAIT)
        expect((await resolve(SIGNING_KEY)).status).toBe(200)
    })

    it.each([
        ['the secret is deleted', 'deleted'],
        ['the API server is down at the safety read', 'down'],
    ])('keeps serving what it holds when %s, and says so on the metrics', async (_label, how) => {
        const { api, url } = await fakeApi({ [KEY]: 'value-1', __CALLER_KEY_TEST_DEPLOYMENT: SIGNING_KEY })
        const { server, resolve } = await boot(url)
        await vi.waitFor(() => expect(api.openWatch).toBeDefined(), WAIT)
        // Only Date is faked, so the sockets and the watch keep real timers.
        vi.useFakeTimers({ toFake: ['Date'] })
        vi.setSystemTime(Date.now() + 60 * 60 * 1000)

        if (how === 'deleted') {
            api.send('DELETED', secretObject('500', { [KEY]: 'value-1' }))
            await vi.waitFor(async () => expect((await mountErrorsTotal.get()).values[0]?.value).toBe(1), WAIT)
        } else {
            api.listStatus = 500
            await server.reload()
            expect((await mountErrorsTotal.get()).values[0]?.value).toBe(1)
        }

        expect(server.lifecycleState().ready).toBe(true)
        expect(await (await resolve(SIGNING_KEY)).json()).toMatchObject({ secrets: { [KEY]: { value: 'value-1' } } })
        expect(Math.round((await servingStaleSeconds.get()).values[0]?.value ?? 0)).toBe(3600)
    })
})