Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
13.7 KB
Raw Blame
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import bcrypt from 'bcryptjs'
import { randomBytes } from 'node:crypto'
import { Pool } from 'pg'
import { initDb } from '../src/db.js'
import { acquirePasswordWork } from '../src/services/auth/passwordWorkOwnership.js'
import {
  type PasswordMutationLease,
  runPasswordMutation,
  withPasswordWorkLease,
} from '../src/services/auth/passwordWorkOwnership.js'
import {
  acceptInvitationForEmail,
  createSilentInvitationForTeams,
  setInvitationPasswordForEmail,
  setInvitationPasswordForUser,
  withMemberPasswordWorkLease,
} from '../src/services/directory/membership.js'
import { createSilentPasswordResetInvitation } from '../src/services/directory/passwordResetInvitations.js'
import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

const holder = vi.hoisted(() => ({ pool: null as unknown as Pool }))
vi.mock('../src/db.js', async importOriginal => {
  const real = await importOriginal<typeof import('../src/db.js')>()
  return {
    ...real,
    pool: { query: (...args: unknown[]) => (holder.pool.query as Function)(...args) },
    withTransaction: (
      work: Parameters<typeof real.withTransaction>[0],
      _pool?: Parameters<typeof real.withTransaction>[1],
      options?: Parameters<typeof real.withTransaction>[2]
    ) => real.withTransaction(work, holder.pool, options),
  }
})

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const realPg = adminUrl ? describe : describe.skip
realPg('password-work operation preservation on PostgreSQL 16', () => {
  const database = `password_operation_${randomBytes(6).toString('hex')}`
  let admin: Pool
  let local: Pool
  let connectionString: string
  beforeAll(async () => {
    admin = new Pool({ connectionString: adminUrl })
    await admin.query(`CREATE DATABASE "${database}"`)
    const url = new URL(adminUrl!)
    url.pathname = `/${database}`
    connectionString = url.toString()
    local = new Pool({ connectionString: url.toString(), max: 1 })
    holder.pool = local
    await initDb(local)
  }, 60_000)
  beforeEach(async () => {
    vi.restoreAllMocks()
    await local.query('DELETE FROM password_verification_work')
  })
  afterAll(async () => {
    try {
      vi.restoreAllMocks()
      await endPoolAndWaitForClients(local)
      if (admin) await admin.query(`DROP DATABASE IF EXISTS "${database}"`)
    } finally {
      await admin?.end()
    }
  })

  async function ownerCount() {
    return (await local.query('SELECT COUNT(*)::int AS count FROM password_verification_work'))
      .rows[0].count
  }
  async function resetFixture() {
    const email = `${randomBytes(8).toString('hex')}@example.invalid`
    const user = (await local.query('INSERT INTO users(email) VALUES ($1) RETURNING id', [email]))
      .rows[0]
    const invitation = await createSilentPasswordResetInvitation({
      inviteeName: 'Synthetic preservation member',
      email,
    })
    return { email, userId: user.id as string, invitation }
  }

  it('releases an unused reservation and retains an original outer work failure', async () => {
    const failure = new Error('preservation work failure')
    await expect(
      withMemberPasswordWorkLease(async () => {
        throw failure
      })
    ).rejects.toBe(failure)
    expect(await ownerCount()).toBe(0)
  })

  it('keeps storage release failure above the outer work failure', async () => {
    try {
      await expect(
        withMemberPasswordWorkLease(async () => {
          await local.query('ALTER TABLE password_verification_work RENAME TO preservation_work')
          throw new Error('private work failure')
        })
      ).rejects.toMatchObject({ status: 503, message: 'authority_unavailable' })
    } finally {
      await local.query('ALTER TABLE preservation_work RENAME TO password_verification_work')
    }
    // Unconfirmed release remains occupied; only this disposable fixture is reset by beforeEach.
    expect(await ownerCount()).toBe(1)
  })

  it('composes an accepted admin/member operation with its existing reservation and one pool', async () => {
    const email = `${randomBytes(8).toString('hex')}@example.invalid`
    const invitation = await createSilentInvitationForTeams({
      email,
      inviteeName: 'Synthetic admin member',
      purpose: 'admin_desktop_access',
    })
    const result = await withMemberPasswordWorkLease(async lease => {
      const accepted = await acceptInvitationForEmail(email, invitation.token, invitation.id)
      if (!('data' in accepted)) throw new Error('fixture acceptance failed')
      expect(await ownerCount()).toBe(1)
      return setInvitationPasswordForUser(
        accepted.data.userId!,
        email,
        invitation.id,
        'Synthetic-preservation-password',
        lease
      )
    })
    expect(result).toMatchObject({
      data: { passwordUpdated: true, purpose: 'admin_desktop_access' },
    })
    expect(await ownerCount()).toBe(0)
    const row = (await local.query('SELECT password_hash FROM users WHERE email = $1', [email]))
      .rows[0]
    expect(await bcrypt.compare('Synthetic-preservation-password', row.password_hash)).toBe(true)
  })

  it('retains a committed mutation lease and does not release a successor owner', async () => {
    const fixture = await resetFixture()
    let lease: PasswordMutationLease | undefined
    const result = await withPasswordWorkLease(mutationLease => {
      lease = mutationLease
      return setInvitationPasswordForEmail(
        fixture.email,
        fixture.invitation.id,
        'Synthetic-preservation-password',
        fixture.userId,
        mutationLease
      )
    })
    expect(result).toMatchObject({ data: { passwordUpdated: true, status: 'accepted' } })
    const successor = await acquirePasswordWork()
    expect(successor).not.toBeNull()
    try {
      await expect(
        runPasswordMutation(async () => 'must not run', {
          mode: 'standard',
          lease: lease!,
        })
      ).rejects.toMatchObject({ status: 503, message: 'authority_unavailable' })
      expect(await ownerCount()).toBe(1)
    } finally {
      await successor?.release()
    }
  })

  it('rolls back a credential mutation when its transactional owner DELETE fails', async () => {
    const fixture = await resetFixture()
    await local.query(`CREATE FUNCTION preservation_delete_failure() RETURNS trigger LANGUAGE plpgsql
      AS $$ BEGIN RAISE EXCEPTION 'private fixture release detail'; END $$`)
    await local.query(`CREATE TRIGGER preservation_delete_failure BEFORE DELETE
      ON password_verification_work FOR EACH ROW EXECUTE FUNCTION preservation_delete_failure()`)
    try {
      await expect(
        setInvitationPasswordForEmail(
          fixture.email,
          fixture.invitation.id,
          'Synthetic-preservation-password',
          fixture.userId
        )
      ).rejects.toMatchObject({ status: 503, message: 'authority_unavailable' })
      const row = (
        await local.query(
          `SELECT i.status, u.password_hash FROM invitations i
        JOIN users u ON u.id = $2 WHERE i.id = $1`,
          [fixture.invitation.id, fixture.userId]
        )
      ).rows[0]
      expect(row).toMatchObject({ status: 'pending', password_hash: null })
      expect(await ownerCount()).toBe(1)
    } finally {
      await local.query('DROP TRIGGER preservation_delete_failure ON password_verification_work')
      await local.query('DROP FUNCTION preservation_delete_failure()')
    }
  })

  it('rolls back a prior database write when the mutation callback throws a TypeError', async () => {
    const fixture = await resetFixture()
    const callbackFailure = new TypeError('private callback diagnostic')
    const originalName = (
      await local.query('SELECT name FROM users WHERE id = $1', [fixture.userId])
    ).rows[0].name

    const failure = await runPasswordMutation(
      async transaction => {
        await transaction.query('UPDATE users SET name = $2 WHERE id = $1', [
          fixture.userId,
          'Must be rolled back',
        ])
        throw callbackFailure
      },
      { mode: 'standard' }
    ).catch(error => error)

    expect(failure).toMatchObject({
      name: 'PasswordMutationInternalError',
      code: 'internal_error',
      message: 'internal_error',
    })
    const causeDescriptor = Object.getOwnPropertyDescriptor(failure, 'cause')
    expect(causeDescriptor?.value).toBe(callbackFailure)
    expect(causeDescriptor?.enumerable).toBe(false)
    expect(JSON.stringify(failure)).not.toContain(callbackFailure.message)

    const persistedName = (
      await local.query('SELECT name FROM users WHERE id = $1', [fixture.userId])
    ).rows[0].name
    expect(persistedName).toBe(originalName)
    expect(await ownerCount()).toBe(0)
  })

  it('keeps a real lost-COMMIT outcome above failed owner cleanup in an outer reservation', async () => {
    const fixture = await resetFixture()
    const sibling = await createSilentPasswordResetInvitation({
      email: fixture.email,
      inviteeName: 'Synthetic sibling',
    })
    const blackhole = await createPostgresCommitReplyBlackhole(connectionString)
    const proxyPool = new Pool({ connectionString: blackhole.connectionString, max: 1 })
    const outcome = withMemberPasswordWorkLease(async lease => {
      // Acquisition is acknowledged on the ordinary pool; only the credential COMMIT is lost.
      holder.pool = proxyPool
      return setInvitationPasswordForEmail(
        fixture.email,
        fixture.invitation.id,
        'Synthetic-preservation-password',
        fixture.userId,
        lease
      )
    }).then(
      value => ({ value }),
      error => ({ error })
    )
    try {
      await blackhole.commitForwarded
      const deadline = Date.now() + 2_000
      while (blackhole.discardedReplyBytes === 0 && Date.now() < deadline) {
        await new Promise(resolve => setTimeout(resolve, 10))
      }
      expect(blackhole.discardedReplyBytes).toBeGreaterThan(0)
      const row = (
        await local.query(
          `SELECT u.password_hash, i.status, s.status AS sibling_status
        FROM users u JOIN invitations i ON i.id = $2 JOIN invitations s ON s.id = $3
        WHERE u.id = $1`,
          [fixture.userId, fixture.invitation.id, sibling.id]
        )
      ).rows[0]
      expect(row).toMatchObject({ status: 'accepted', sibling_status: 'revoked' })
      expect(await bcrypt.compare('Synthetic-preservation-password', row.password_hash)).toBe(true)
      expect(await ownerCount()).toBe(0)
      // A successor exists before fallback cleanup. The old owner must neither delete it
      // nor replace the chosen unknown outcome with its failed owned DELETE.
      holder.pool = local
      const successor = await acquirePasswordWork()
      holder.pool = proxyPool
      blackhole.failClientAfterCommitForwarded()
      const result = await outcome
      expect(result).toMatchObject({
        error: { status: 503, publicError: 'recovery_outcome_unknown' },
      })
      expect(blackhole.commands.filter(command => command === 'COMMIT')).toHaveLength(1)
      expect(await ownerCount()).toBe(1)
      holder.pool = local
      await successor!.release()
      expect(
        await setInvitationPasswordForEmail(
          fixture.email,
          fixture.invitation.id,
          'Synthetic-replay-password',
          fixture.userId
        )
      ).toEqual({ error: 'not_pending' })
    } finally {
      holder.pool = local
      blackhole.allowReplies()
      await blackhole.close()
      await endPoolAndWaitForClients(proxyPool)
      await outcome
    }
  })

  it('allows only one concurrent callback and rejects terminal reuse without touching a successor', async () => {
    await withPasswordWorkLease(async lease => {
      let resume!: () => void
      const gate = new Promise<void>(resolve => {
        resume = resolve
      })
      const work = vi.fn(async () => {
        await gate
        return 'committed'
      })
      const first = lease.commitWith(work, { mode: 'standard' })
      try {
        await expect(lease.commitWith(work, { mode: 'standard' })).rejects.toMatchObject({
          status: 503,
        })
      } finally {
        resume()
      }
      expect(await first).toBe('committed')
      expect(work).toHaveBeenCalledTimes(1)
      const successor = await acquirePasswordWork()
      try {
        await expect(runPasswordMutation(work, { mode: 'standard', lease })).rejects.toMatchObject({
          status: 503,
        })
        expect(work).toHaveBeenCalledTimes(1)
        expect(await ownerCount()).toBe(1)
      } finally {
        await successor!.release()
      }
    })
    expect(await ownerCount()).toBe(0)
  })

  it('keeps concurrent supplied domain operations at one committed reset', async () => {
    const fixture = await resetFixture()
    const sibling = await createSilentPasswordResetInvitation({
      email: fixture.email,
      inviteeName: 'Synthetic sibling',
    })
    const hash = vi.spyOn(bcrypt, 'hash')
    const outcomes = await withMemberPasswordWorkLease(lease =>
      Promise.allSettled(
        [fixture.invitation.id, sibling.id].map(id =>
          setInvitationPasswordForEmail(
            fixture.email,
            id,
            'Synthetic-preservation-password',
            fixture.userId,
            lease
          )
        )
      )
    )
    expect(outcomes.filter(outcome => outcome.status === 'fulfilled')).toHaveLength(1)
    expect(outcomes.filter(outcome => outcome.status === 'rejected')).toHaveLength(1)
    expect(hash).toHaveBeenCalledTimes(1)
    const rows = (
      await local.query('SELECT status FROM invitations WHERE id = ANY($1::uuid[])', [
        [fixture.invitation.id, sibling.id],
      ])
    ).rows
    expect(rows.map(row => row.status).sort()).toEqual(['accepted', 'revoked'])
    expect(await ownerCount()).toBe(0)
  })
})