Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
43.7 KB
Raw Blame
import { afterEach, describe, expect, it } from 'vitest'
import { createHash } from 'node:crypto'
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Readable } from 'node:stream'
import type { GfsUploadConfig } from '../config'
import {
  CommitOutcomeUnknownError,
  type TransactionOptions,
  type Transactor,
  type TxClient,
} from '../db/writeStore'
import {
  GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES,
  GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES,
  partGeometry,
} from './protocol'
import {
  GfsUploadSessionService,
  type UploadSessionServiceDeps,
  uploadCapabilities,
} from './uploadSession'

type Row = Record<string, unknown>

class MemoryDb {
  sessions: Row[] = []
  parts: Row[] = []
  idempotencyLookupCount = 0
  failReconciliationReads = false
  failSessionReconciliationReads = false

  async query(text: string, values: unknown[] = []): Promise<{ rows: Row[] }> {
    const sql = text.replace(/\s+/g, ' ').trim()
    if (sql === 'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ') return { rows: [] }
    if (sql.startsWith('SELECT pg_advisory_xact_lock')) return { rows: [] }
    if (sql.startsWith('SELECT * FROM gfs_upload_sessions WHERE owner_subject')) {
      this.idempotencyLookupCount += 1
      const [owner, drive, key] = values.map(String)
      return {
        rows: this.sessions.filter(
          row => row.owner_subject === owner && row.drive === drive && row.idempotency_key === key
        ),
      }
    }
    if (sql.startsWith('SELECT * FROM gfs_upload_sessions WHERE upload_id')) {
      if (this.failSessionReconciliationReads) {
        throw new CommitOutcomeUnknownError(new Error('synthetic session reconciliation read failure'))
      }
      const [id, drive, owner] = values.map(String)
      return {
        rows: this.sessions.filter(
          row => row.upload_id === id && row.drive === drive && row.owner_subject === owner
        ),
      }
    }
    if (sql.startsWith('SELECT session_epoch, state FROM gfs_upload_sessions WHERE upload_id')) {
      const [id, drive, owner] = values.map(String)
      return {
        rows: this.sessions
          .filter(row => row.upload_id === id && row.drive === drive && row.owner_subject === owner)
          .map(row => ({ session_epoch: row.session_epoch, state: row.state })),
      }
    }
    if (
      sql.startsWith('SELECT p.upload_id, p.part_number, p.staging_path FROM gfs_upload_parts p')
    ) {
      return { rows: [] }
    }
    if (
      sql.startsWith(
        "SELECT upload_id FROM gfs_upload_sessions WHERE expires_at <= now() AND ( state IN ('initiated','uploading','paused') OR (state = 'finalizing' AND finalizing_started_at IS NULL) )"
      )
    ) {
      return {
        rows: this.sessions
          .filter(row => {
            const expired = Date.parse(String(row.expires_at)) <= Date.now()
            const state = String(row.state)
            return (
              expired &&
              (['initiated', 'uploading', 'paused'].includes(state) ||
                (state === 'finalizing' && row.finalizing_started_at == null))
            )
          })
          .map(row => ({ upload_id: row.upload_id })),
      }
    }
    if (sql.startsWith('SELECT p.upload_id, p.part_number, p.offset_bytes, p.length_bytes')) {
      return { rows: [] }
    }
    if (
      sql.startsWith(
        "SELECT upload_id, state FROM gfs_upload_sessions WHERE state IN ('aborted', 'expired', 'failed', 'completed') AND cleanup_at IS NULL"
      )
    ) {
      return {
        rows: this.sessions
          .filter(
            row =>
              ['aborted', 'expired', 'failed', 'completed'].includes(String(row.state)) &&
              row.cleanup_at == null
          )
          .map(row => ({ upload_id: row.upload_id, state: row.state })),
      }
    }
    if (
      sql.startsWith(
        "SELECT upload_id FROM gfs_upload_sessions WHERE state = 'completed' AND cleanup_at <= now()"
      )
    ) {
      return { rows: [] }
    }
    if (
      sql.startsWith('SELECT COALESCE(SUM((expected_bytes - committed_bytes) + expected_bytes)')
    ) {
      const drive = String(values[0])
      const reserved = this.sessions
        .filter(
          row =>
            row.drive === drive &&
            ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state))
        )
        .reduce(
          (sum, row) =>
            sum +
            (Number(row.expected_bytes) - Number(row.committed_bytes ?? 0)) +
            Number(row.expected_bytes),
          0
        )
      return { rows: [{ reserved: String(reserved) }] }
    }
    if (
      sql === "SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'"
    ) {
      return {
        rows: [{ count: String(this.sessions.filter(row => row.state === 'finalizing').length) }],
      }
    }
    if (sql.startsWith('SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions')) {
      const [drive, owner] = values.map(value => (value === undefined ? undefined : String(value)))
      const active = this.sessions.filter(
        row =>
          ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state)) &&
          (drive === undefined || row.drive === drive) &&
          (owner === undefined || row.owner_subject === owner)
      ).length
      return { rows: [{ count: String(active) }] }
    }
    if (sql.startsWith('SELECT COUNT(*)::bigint')) {
      if (sql.includes('FROM gfs_upload_parts WHERE upload_id = $1 AND state = \'reserved\'')) {
        const uploadId = String(values[0])
        return {
          rows: [{
            count: String(
              this.parts.filter(part => part.upload_id === uploadId && part.state === 'reserved').length
            ),
          }],
        }
      }
      const drive = String(values[0])
      return {
        rows: [
          {
            count: String(
              this.parts.filter(
                part =>
                  part.state === 'reserved' &&
                  this.sessions.some(
                    session => session.upload_id === part.upload_id && session.drive === drive
                  )
              ).length
            ),
          },
        ],
      }
    }
    if (
      sql.startsWith('SELECT upload_id FROM gfs_upload_sessions') &&
      sql.includes("state = 'finalizing'")
    ) {
      return { rows: [] }
    }
    if (sql.startsWith('INSERT INTO gfs_upload_sessions')) {
      const [
        uploadId,
        key,
        drive,
        owner,
        primary,
        operation,
        fingerprint,
        parent,
        resource,
        name,
        ifMatch,
        expected,
        partBytes,
        partCount,
        sha,
        expiresAt,
      ] = values
      const row: Row = {
        upload_id: String(uploadId),
        idempotency_key: String(key),
        drive: String(drive),
        owner_subject: String(owner),
        primary_subject: String(primary),
        operation: String(operation),
        request_fingerprint: String(fingerprint),
        parent_rid: parent,
        resource_rid: resource,
        resource_name: name,
        if_match: ifMatch,
        expected_bytes: expected,
        part_bytes: partBytes,
        part_count: partCount,
        whole_sha256: sha,
        committed_bytes: 0,
        contiguous_bytes: 0,
        committed_part_count: 0,
        active_part_count: 0,
        session_epoch: 0,
        state: 'initiated',
        result_resource_id: null,
        result_version: null,
        result_sha256: null,
        failure_code: null,
        expires_at: expiresAt,
        completed_at: null,
        finalizing_started_at: null,
        cleanup_at: null,
      }
      this.sessions.push(row)
      return { rows: [row] }
    }
    if (sql.startsWith('SELECT upload_id, part_number')) {
      if (
        this.failReconciliationReads &&
        sql.includes('offset_bytes, length_bytes, sha256, state')
      ) {
        throw new Error('synthetic reconciliation read failure')
      }
      const [uploadId, partNumber] = values
      const rows = this.parts.filter(
        part =>
          part.upload_id === String(uploadId) &&
          (partNumber === undefined || Number(part.part_number) === Number(partNumber)) &&
          (!sql.includes("state = 'committed'") || part.state === 'committed')
      )
      return { rows }
    }
    if (sql.startsWith('SELECT state, lease_epoch FROM gfs_upload_parts')) {
      const [uploadId, partNumber] = values
      return {
        rows: this.parts.filter(
          part =>
            part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)
        ),
      }
    }
    if (sql.startsWith('INSERT INTO gfs_upload_parts')) {
      const [uploadId, partNumber, offset, length, sha, staging, epoch] = values
      const row: Row = {
        upload_id: String(uploadId),
        part_number: partNumber,
        offset_bytes: offset,
        length_bytes: length,
        sha256: sha,
        state: 'reserved',
        staging_path: staging,
        lease_epoch: epoch,
        committed_at: null,
      }
      this.parts.push(row)
      return { rows: [] }
    }
    if (sql.startsWith('UPDATE gfs_upload_parts SET offset_bytes')) {
      const [uploadId, partNumber, offset, length, sha, staging, epoch] = values
      const row = this.parts.find(
        part =>
          part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)
      )!
      Object.assign(row, {
        offset_bytes: offset,
        length_bytes: length,
        sha256: sha,
        state: 'reserved',
        staging_path: staging,
        lease_epoch: epoch,
        committed_at: null,
      })
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'uploading'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.state = 'uploading'
      if (sql.includes('session_epoch = session_epoch + 1')) {
        row.session_epoch = Number(row.session_epoch) + 1
        row.finalizing_started_at = null
        return { rows: [] }
      }
      row.active_part_count = sql.includes('active_part_count = (')
        ? this.parts.filter(part => part.upload_id === row.upload_id && part.state === 'reserved').length
        : Number(row.active_part_count) + 1
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'committed'")) {
      const [uploadId, partNumber, path] = values
      const row = this.parts.find(
        part =>
          part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)
      )!
      Object.assign(row, { state: 'committed', staging_path: path })
      return { rows: [] }
    }
    if (sql.startsWith('UPDATE gfs_upload_sessions SET committed_bytes')) {
      const [uploadId, length] = values
      const row = this.sessions.find(session => session.upload_id === String(uploadId))!
      row.committed_bytes = Number(row.committed_bytes) + Number(length)
      row.committed_part_count = Number(row.committed_part_count) + 1
      row.active_part_count = this.parts.filter(
        part => part.upload_id === row.upload_id && part.state === 'reserved'
      ).length
      let contiguous = 0
      for (let number = 0; number < Number(row.part_count); number += 1) {
        const part = this.parts.find(
          candidate =>
            candidate.upload_id === row.upload_id &&
            Number(candidate.part_number) === number &&
            candidate.state === 'committed'
        )
        if (!part) break
        contiguous += Number(part.length_bytes)
      }
      row.contiguous_bytes = contiguous
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'failed'")) {
      const uploadId = String(values[0])
      const row = this.parts.find(
        part =>
          part.upload_id === uploadId &&
          part.state === 'reserved' &&
          Number(part.part_number) === Number(values[1])
      )
      if (!row) return { rows: [] }
      row.state = 'failed'
      return { rows: [{ updated: 1 }] }
    }
    if (sql.startsWith('UPDATE gfs_upload_sessions SET active_part_count')) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.active_part_count = this.parts.filter(
        part => part.upload_id === row.upload_id && part.state === 'reserved'
      ).length
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'paused'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.state = 'paused'
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'uploading'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.state = 'uploading'
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'aborted'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.state = 'aborted'
      row.session_epoch = Number(row.session_epoch) + 1
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'expired'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.state = 'expired'
      row.session_epoch = Number(row.session_epoch) + 1
      row.active_part_count = 0
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'finalizing'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      row.state = 'finalizing'
      row.finalizing_started_at = new Date().toISOString()
      const extensionMs = Number(values[2] ?? 0)
      const currentExpiry = Date.parse(String(row.expires_at))
      const finalizerExpiry = Date.now() + extensionMs
      row.expires_at = new Date(Math.max(currentExpiry, finalizerExpiry)).toISOString()
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'completed'")) {
      const [uploadId, resourceId, version, sha256] = values
      const row = this.sessions.find(session => session.upload_id === String(uploadId))!
      row.state = 'completed'
      row.result_resource_id = resourceId
      row.result_version = version
      row.result_sha256 = sha256
      row.completed_at = new Date().toISOString()
      row.finalizing_started_at = null
      return { rows: [] }
    }
    if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'failed'")) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))!
      if (sql.includes("state = 'finalizing' AND session_epoch = $3")) {
        if (row.state !== 'finalizing' || Number(row.session_epoch) !== Number(values[2])) {
          return { rows: [] }
        }
        row.state = 'failed'
        row.failure_code = values[1]
        row.active_part_count = 0
        row.finalizing_started_at = null
        return { rows: [{ upload_id: row.upload_id }] }
      }
      row.state = 'failed'
      row.failure_code = values[1]
      row.active_part_count = 0
      row.finalizing_started_at = null
      return { rows: [] }
    }
    if (sql.startsWith('UPDATE gfs_upload_sessions SET cleanup_at')) {
      const row = this.sessions.find(session => session.upload_id === String(values[0]))
      if (row) row.cleanup_at = new Date().toISOString()
      return { rows: [] }
    }
    if (sql.startsWith('DELETE FROM gfs_upload_parts')) {
      const uploadId = String(values[0])
      this.parts = this.parts.filter(part => part.upload_id !== uploadId)
      return { rows: [] }
    }
    throw new Error(`unhandled fake SQL: ${sql}`)
  }
}

const CONFIG: GfsUploadConfig = {
  productMaxFileBytes: 209715200,
  protocolMaxFileBytes: 1073741824,
  preferredPartBytes: 8388608,
  maxPartBytes: 16777216,
  maxPartCount: 1024,
  maxConcurrentPartsPerSession: 4,
  maxConcurrentPartStreamsGlobal: 16,
  maxActivePerSubject: 2,
  maxActiveGlobal: 8,
  maxConcurrentFinalizations: 1,
  minFreeBytes: 10 * 1024 * 1024 * 1024,
  instabilityFailureThreshold: 3,
  fallbackConcurrency: 2,
  sessionTtlSeconds: 86400,
  partTimeoutMs: 300000,
  finalizeTimeoutMs: 600000,
  stalePartLeaseMs: 600000,
  receiptRetentionSeconds: 86400,
  enabled: true,
}

const TEST_CONFIG: GfsUploadConfig = {
  ...CONFIG,
  preferredPartBytes: 1048576,
  maxPartBytes: 1048576,
}

const principal = { drive: 'main', ownerSubject: 'user-1', primarySubject: 'user-1' }
const key = '11111111-1111-4111-8111-111111111111'

let tempRoot: string
afterEach(async () => {
  if (tempRoot) await rm(tempRoot, { recursive: true, force: true })
})

function service(
  db: MemoryDb,
  config: GfsUploadConfig = TEST_CONFIG,
  overrides: Partial<Pick<UploadSessionServiceDeps, 'now' | 'finalize' | 'tx' | 'authorize'>> = {}
): GfsUploadSessionService {
  return new GfsUploadSessionService({
    db,
    tx: overrides.tx ?? { transaction: async fn => fn(db) },
    blobs: { availableBytes: async () => 10n ** 12n } as never,
    storageMountPath: tempRoot,
    config,
    ...overrides,
  })
}

class ThrowAfterCommitTransactor implements Transactor {
  private transactionCount = 0

  constructor(
    private readonly delegate: Transactor,
    private readonly afterCommit: () => void
  ) {}

  async transaction<T>(
    fn: (client: TxClient) => Promise<T>,
    options?: TransactionOptions
  ): Promise<T> {
    this.transactionCount += 1
    const result = await this.delegate.transaction(fn, options)
    // create/reservation is the first transaction; the second is the part
    // commit. The callback makes the subsequent reconciliation read fail.
    if (this.transactionCount === 2) {
      this.afterCommit()
      throw new CommitOutcomeUnknownError(new Error('synthetic post-commit response loss'))
    }
    return result
  }
}

class ThrowBeforePartCommitTransactor implements Transactor {
  private transactionCount = 0

  constructor(private readonly delegate: Transactor) {}

  async transaction<T>(
    fn: (client: TxClient) => Promise<T>,
    options?: TransactionOptions
  ): Promise<T> {
    this.transactionCount += 1
    if (this.transactionCount === 2)
      throw new CommitOutcomeUnknownError(new Error('synthetic pre-commit failure'))
    return this.delegate.transaction(fn, options)
  }
}

class ThrowOnCommitTransactor implements Transactor {
  private transactionCount = 0

  constructor(
    private readonly delegate: Transactor,
    private readonly failOnTransaction: number,
    private readonly afterCommit: () => void
  ) {}

  async transaction<T>(
    fn: (client: TxClient) => Promise<T>,
    options?: TransactionOptions
  ): Promise<T> {
    this.transactionCount += 1
    const result = await this.delegate.transaction(fn, options)
    if (this.transactionCount === this.failOnTransaction) {
      this.afterCommit()
      throw new CommitOutcomeUnknownError(new Error('synthetic completion response loss'))
    }
    return result
  }
}

async function settleWithin<T>(
  promise: Promise<T>,
  timeoutMs = 500
): Promise<
  { kind: 'completed'; value: T } | { kind: 'rejected'; error: unknown } | { kind: 'timeout' }
> {
  let timer: ReturnType<typeof setTimeout> | undefined
  try {
    return await Promise.race([
      promise.then(
        value => ({ kind: 'completed' as const, value }),
        error => ({ kind: 'rejected' as const, error })
      ),
      new Promise<{ kind: 'timeout' }>(resolve => {
        timer = setTimeout(() => resolve({ kind: 'timeout' }), timeoutMs)
      }),
    ])
  } finally {
    if (timer) clearTimeout(timer)
  }
}

describe('GfsUploadSessionService', () => {
  it('accepts exactly 200 MiB, rejects one byte over before allocation, and enforces part-count geometry', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-boundary-'))
    const input = {
      ...principal,
      operation: 'create' as const,
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'boundary.bin',
    }

    const exactDb = new MemoryDb()
    const exact = await service(exactDb, CONFIG).create({
      ...input,
      sizeBytes: GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES,
      idempotencyKey: '11111111-1111-4111-8111-111111111112',
    })
    expect(exact.created).toBe(true)
    expect(exact.session.expectedBytes).toBe(GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES)
    expect(exact.session.partBytes).toBe(8 * 1024 * 1024)
    expect(exact.session.partCount).toBe(25)
    expect(exactDb.sessions).toHaveLength(1)

    const overDb = new MemoryDb()
    await expect(
      service(overDb, CONFIG).create({
        ...input,
        sizeBytes: GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES + 1,
        idempotencyKey: '11111111-1111-4111-8111-111111111113',
      })
    ).rejects.toMatchObject({ code: 'payload_too_large' })
    expect(overDb.sessions).toHaveLength(0)

    const partCountDb = new MemoryDb()
    await expect(
      service(partCountDb, { ...CONFIG, maxPartCount: 24 }).create({
        ...input,
        sizeBytes: GFS_UPLOAD_V2_DEFAULT_PRODUCT_MAX_BYTES,
        idempotencyKey: '11111111-1111-4111-8111-111111111114',
      })
    ).rejects.toMatchObject({ code: 'payload_too_large' })
    expect(partCountDb.sessions).toHaveLength(0)
  })

  it.each([
    ['100 MiB', 100 * 1024 * 1024],
    ['250 MiB', 250 * 1024 * 1024],
    ['300 MiB', 300 * 1024 * 1024],
  ] as const)(
    'advertises and enforces the configured %s product policy before allocation',
    async (_label, productMaxFileBytes) => {
      tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-runtime-boundary-'))
      const config = { ...CONFIG, productMaxFileBytes }
      const capability = uploadCapabilities(true, config).resumableV2
      expect(capability).toMatchObject({ enabled: true, maxFileBytes: productMaxFileBytes })

      const input = {
        ...principal,
        operation: 'create' as const,
        parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
        name: 'runtime-boundary.bin',
      }
      const exactDb = new MemoryDb()
      const exact = await service(exactDb, config).create({
        ...input,
        sizeBytes: productMaxFileBytes,
        idempotencyKey: '21111111-1111-4111-8111-111111111112',
      })
      expect(exact.session.expectedBytes).toBe(productMaxFileBytes)
      expect(exactDb.sessions).toHaveLength(1)

      const overDb = new MemoryDb()
      await expect(
        service(overDb, config).create({
          ...input,
          sizeBytes: productMaxFileBytes + 1,
          idempotencyKey: '21111111-1111-4111-8111-111111111113',
        })
      ).rejects.toMatchObject({
        code: 'payload_too_large',
        message: `sizeBytes must be between 0 and ${productMaxFileBytes}`,
      })
      expect(overDb.sessions).toHaveLength(0)
    }
  )

  it('admits exactly 1 GiB when product policy permits the protocol boundary', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-protocol-boundary-'))
    const config = { ...CONFIG, productMaxFileBytes: GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES }
    const input = {
      ...principal,
      operation: 'create' as const,
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'protocol-boundary.bin',
    }

    const exactDb = new MemoryDb()
    const exact = await service(exactDb, config).create({
      ...input,
      sizeBytes: GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES,
      idempotencyKey: '21111111-1111-4111-8111-111111111115',
    })
    expect(exact.session.expectedBytes).toBe(GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES)
    expect(exactDb.sessions).toHaveLength(1)
  })

  it('keeps zero-byte uploads valid with a positive configured product limit', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-empty-file-'))
    const db = new MemoryDb()
    const created = await service(db, { ...CONFIG, productMaxFileBytes: 1 }).create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'empty.bin',
      sizeBytes: 0,
      idempotencyKey: '21111111-1111-4111-8111-111111111114',
    })
    expect(created.session.expectedBytes).toBe(0)
    expect(created.session.partCount).toBe(0)
  })

  it(
    'creates idempotent sessions and keeps indexed commits separate from contiguous progress',
    { timeout: 15_000 },
    async () => {
      tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
      const db = new MemoryDb()
      const uploads = service(db)
      const created = await uploads.create({
        ...principal,
        operation: 'create',
        parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
        name: 'data.bin',
        sizeBytes: 2097152,
        idempotencyKey: key,
      })
      expect(created.created).toBe(true)
      const replay = await uploads.create({
        ...principal,
        operation: 'create',
        parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
        name: 'data.bin',
        sizeBytes: 2097152,
        idempotencyKey: key,
      })
      expect(replay.created).toBe(false)
      expect(replay.session.uploadId).toBe(created.session.uploadId)

      const first = Buffer.alloc(1048576, 0x11)
      const second = Buffer.alloc(1048576, 0x22)
      const firstGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 0)
      const secondGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 1)
      const checksum = (value: Buffer) => createSha(value)
      const outOfOrder = await uploads.putPart(
        created.session.uploadId,
        secondGeometry,
        checksum(second),
        Readable.from([second]) as never,
        principal
      )
      expect(outOfOrder.session.contiguousBytes).toBe(0)
      const ordered = await uploads.putPart(
        created.session.uploadId,
        firstGeometry,
        checksum(first),
        Readable.from([first]) as never,
        principal
      )
      expect(ordered.session.contiguousBytes).toBe(2097152)
      expect(ordered.session.committedPartCount).toBe(2)
      expect(await readFile(ordered.part.stagingPath)).toEqual(first)
      const duplicate = await uploads.putPart(
        created.session.uploadId,
        firstGeometry,
        checksum(first),
        Readable.from([first]) as never,
        principal
      )
      expect(duplicate.part.sha256).toBe(checksum(first))
    }
  )

  it('replays an admitted create after the runtime product policy is lowered', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-policy-replay-'))
    const db = new MemoryDb()
    const request = {
      ...principal,
      operation: 'create' as const,
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'admitted-before-policy-lowering.bin',
      sizeBytes: 250 * 1024 * 1024,
      idempotencyKey: '31111111-1111-4111-8111-111111111111',
    }
    const admitted = await service(db, {
      ...CONFIG,
      productMaxFileBytes: 300 * 1024 * 1024,
    }).create(request)
    expect(admitted.created).toBe(true)
    expect(db.sessions).toHaveLength(1)

    const lowered = service(db, { ...CONFIG, productMaxFileBytes: 100 * 1024 * 1024 })
    const replay = await lowered.create(request)
    expect(replay).toMatchObject({
      created: false,
      session: { uploadId: admitted.session.uploadId, expectedBytes: request.sizeBytes },
    })
    expect(db.sessions).toHaveLength(1)

    await expect(
      lowered.create({
        ...request,
        idempotencyKey: '31111111-1111-4111-8111-111111111112',
      })
    ).rejects.toMatchObject({ code: 'payload_too_large' })
    expect(db.sessions).toHaveLength(1)

    await expect(
      lowered.create({ ...request, name: 'conflicting-name.bin' })
    ).rejects.toMatchObject({
      code: 'idempotency_conflict',
    })
    expect(db.sessions).toHaveLength(1)

    const lookupsBeforeUnauthorizedAttempt = db.idempotencyLookupCount
    const guarded = service(
      db,
      { ...CONFIG, productMaxFileBytes: 100 * 1024 * 1024 },
      {
        authorize: async input => {
          if (input.ownerSubject !== principal.ownerSubject) throw new Error('not authorized')
        },
      }
    )
    await expect(
      guarded.create({
        ...request,
        ownerSubject: 'user-2',
        primarySubject: 'user-2',
        sizeBytes: 1,
      })
    ).rejects.toThrow('not authorized')
    expect(db.idempotencyLookupCount).toBe(lookupsBeforeUnauthorizedAttempt)
    expect(db.sessions).toHaveLength(1)
  })

  it('does not double-decrement active parts after an unknown commit and failed reconciliation', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
    const db = new MemoryDb()
    const created = await service(db).create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'counter-safe.bin',
      sizeBytes: 2 * TEST_CONFIG.preferredPartBytes,
      idempotencyKey: '66666666-6666-4666-8666-666666666666',
    })
    const session = db.sessions[0]!
    session.active_part_count = 1
    db.parts.push({
      upload_id: created.session.uploadId,
      part_number: 1,
      offset_bytes: TEST_CONFIG.preferredPartBytes,
      length_bytes: TEST_CONFIG.preferredPartBytes,
      sha256: 'f'.repeat(64),
      state: 'reserved',
      staging_path: join(tempRoot, 'other.part'),
      lease_epoch: 0,
      committed_at: null,
    })

    const tx = new ThrowAfterCommitTransactor({ transaction: async fn => fn(db) }, () => {
      db.failReconciliationReads = true
    })
    const uploads = service(db, TEST_CONFIG, { tx })
    const first = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x31)
    await expect(
      uploads.putPart(
        created.session.uploadId,
        partGeometry(
          {
            expectedBytes: 2 * TEST_CONFIG.preferredPartBytes,
            partBytes: TEST_CONFIG.preferredPartBytes,
          },
          0
        ),
        createSha(first),
        Readable.from([first]) as never,
        principal
      )
    ).rejects.toBeInstanceOf(CommitOutcomeUnknownError)

    expect(session.active_part_count).toBe(1)
    expect(db.parts.find(part => Number(part.part_number) === 0)?.state).toBe('committed')
    expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')
  })

  it('admits a new part from durable reserved rows when the session counter is stale', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-counter-drift-'))
    const db = new MemoryDb()
    const config = { ...TEST_CONFIG, maxConcurrentPartsPerSession: 1 }
    const uploads = service(db, config)
    const created = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'counter-drift.bin',
      sizeBytes: TEST_CONFIG.preferredPartBytes,
      idempotencyKey: '68686868-6868-4686-8686-686868686868',
    })

    // A stale scalar must not deny admission when no durable reserved row
    // exists. This is the inverse of the double-decrement regression: rows,
    // not a receipt field, own the concurrency contract.
    db.sessions[0]!.active_part_count = 99
    const payload = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x52)
    const committed = await uploads.putPart(
      created.session.uploadId,
      partGeometry(
        { expectedBytes: TEST_CONFIG.preferredPartBytes, partBytes: TEST_CONFIG.preferredPartBytes },
        0
      ),
      createSha(payload),
      Readable.from([payload]) as never,
      principal
    )

    expect(committed.part.state).toBe('committed')
    expect(committed.session.activePartCount).toBe(0)
    expect(db.sessions[0]).toMatchObject({ active_part_count: 0 })
  })

  it('extends the durable expiry window before entering finalization', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-finalizing-ttl-'))
    const db = new MemoryDb()
    const config = { ...TEST_CONFIG, sessionTtlSeconds: 1, finalizeTimeoutMs: 600_000 }
    let finalizerExpiry: number | undefined
    const uploads = service(db, config, {
      finalize: async session => {
        finalizerExpiry = Date.parse(session.expiresAt)
        return { resourceId: 'resource-finalizing-ttl', version: 1, sha256: 'a'.repeat(64) }
      },
    })
    const created = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'finalizing-ttl.bin',
      sizeBytes: 0,
      idempotencyKey: '69696969-6969-4696-8696-696969696969',
    })
    const originalExpiry = Date.parse(created.session.expiresAt)

    await expect(uploads.complete(created.session.uploadId, principal)).resolves.toMatchObject({
      state: 'completed',
    })
    expect(finalizerExpiry).toBeGreaterThan(originalExpiry)
    expect(Date.parse(db.sessions[0]!.expires_at as string)).toBeGreaterThan(originalExpiry)
  })

  it('expires a finalizing session whose durable start timestamp is missing', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-finalizing-null-start-'))
    const db = new MemoryDb()
    const uploads = service(db)
    const created = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'finalizing-null-start.bin',
      sizeBytes: 0,
      idempotencyKey: '6a6a6a6a-6a6a-4a6a-8a6a-6a6a6a6a6a6a',
    })
    const row = db.sessions.find(session => session.upload_id === created.session.uploadId)!
    row.state = 'finalizing'
    row.finalizing_started_at = null
    row.expires_at = new Date(Date.now() - 1_000).toISOString()

    await expect(uploads.reconcile()).resolves.toMatchObject({ expiredSessions: 1 })
    expect(row).toMatchObject({ state: 'expired', active_part_count: 0 })
  })

  it('preserves a published final path until reconciliation resolves an ambiguous completion commit', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-final-path-reconcile-'))
    const db = new MemoryDb()
    const created = await service(db).create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'ambiguous-completion.bin',
      sizeBytes: 0,
      idempotencyKey: '70707070-7070-4070-8070-707070707070',
    })
    let publishedPath = ''
    const tx = new ThrowOnCommitTransactor(
      { transaction: async fn => fn(db) },
      2,
      () => {
        db.failSessionReconciliationReads = true
      }
    )
    const uploads = service(db, TEST_CONFIG, {
      tx,
      finalize: async session => {
        publishedPath = join(
          tempRoot,
          '.uploads',
          session.uploadId,
          'parts',
          'published.part'
        )
        await mkdir(join(tempRoot, '.uploads', session.uploadId, 'parts'), { recursive: true })
        await writeFile(publishedPath, 'published before completion response was lost')
        return { resourceId: 'resource-ambiguous', version: 1, sha256: 'b'.repeat(64) }
      },
    })

    await expect(uploads.complete(created.session.uploadId, principal)).rejects.toBeInstanceOf(
      CommitOutcomeUnknownError
    )
    expect(db.sessions[0]).toMatchObject({ state: 'completed', cleanup_at: null })
    await expect(stat(publishedPath)).resolves.toBeDefined()

    db.failSessionReconciliationReads = false
    await expect(uploads.reconcile()).resolves.toEqual({ staleParts: 0, expiredSessions: 0 })
    await expect(stat(publishedPath)).rejects.toMatchObject({ code: 'ENOENT' })
    expect(db.sessions[0]?.cleanup_at).toEqual(expect.any(String))
  })

  it('removes a rename-before-commit orphan while the reserved lease is still locked', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-orphan-'))
    const db = new MemoryDb()
    const tx = new ThrowBeforePartCommitTransactor({ transaction: async fn => fn(db) })
    const created = await service(db).create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'orphan.bin',
      sizeBytes: TEST_CONFIG.preferredPartBytes,
      idempotencyKey: '67676767-6767-4676-8676-676767676767',
    })
    const uploads = service(db, TEST_CONFIG, { tx })
    const payload = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x41)
    const finalPath = join(tempRoot, '.uploads', created.session.uploadId, 'parts', '0.part')

    await expect(
      uploads.putPart(
        created.session.uploadId,
        partGeometry(
          {
            expectedBytes: TEST_CONFIG.preferredPartBytes,
            partBytes: TEST_CONFIG.preferredPartBytes,
          },
          0
        ),
        createSha(payload),
        Readable.from([payload]) as never,
        principal
      )
    ).rejects.toBeInstanceOf(CommitOutcomeUnknownError)

    await expect(stat(finalPath)).rejects.toMatchObject({ code: 'ENOENT' })
    expect(db.parts[0]).toMatchObject({ state: 'failed' })
    expect(db.sessions[0]).toMatchObject({ active_part_count: 0 })
  })

  it('cancels terminally and rejects a conflicting idempotency fingerprint', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
    const db = new MemoryDb()
    const uploads = service(db)
    await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'data.bin',
      sizeBytes: 0,
      idempotencyKey: key,
    })
    await expect(
      uploads.create({
        ...principal,
        operation: 'create',
        parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
        name: 'other.bin',
        sizeBytes: 0,
        idempotencyKey: key,
      })
    ).rejects.toMatchObject({ code: 'idempotency_conflict' })
    const session = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'cancel.bin',
      sizeBytes: 0,
      idempotencyKey: '22222222-2222-4222-8222-222222222222',
    })
    await uploads.cancel(session.session.uploadId, principal)
    await expect(uploads.get(session.session.uploadId, principal)).rejects.toMatchObject({
      code: 'upload_aborted',
    })
  })

  it('releases a rejected finalizer before quiescence and cleans the failed upload immediately', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
    const db = new MemoryDb()
    let now = 0
    const uploads = service(db, TEST_CONFIG, {
      // Advance beyond the drain deadline on its first condition check. This
      // makes the regression deterministic without waiting for the production
      // five-minute part timeout: the old code still skipped cleanup because
      // the failed finalizer remained registered while it drained itself.
      now: () => (now += TEST_CONFIG.partTimeoutMs + 1),
      finalize: async () => {
        throw new Error('synthetic finalizer rejection')
      },
    })
    const created = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'finalizer-rejection.bin',
      sizeBytes: 0,
      idempotencyKey: '33333333-3333-4333-8333-333333333333',
    })
    const uploadDirectory = join(tempRoot, '.uploads', created.session.uploadId)
    await mkdir(uploadDirectory, { recursive: true })
    await writeFile(join(uploadDirectory, 'finalizer-marker'), 'pending')

    const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))

    expect(outcome.kind).toBe('rejected')
    if (outcome.kind !== 'rejected')
      throw new Error(`unexpected completion outcome: ${outcome.kind}`)
    expect(outcome.error).toMatchObject({ message: 'synthetic finalizer rejection' })
    expect(db.sessions[0]).toMatchObject({
      state: 'failed',
      failure_code: 'finalization_failed',
      cleanup_at: expect.any(String),
    })
    expect(
      (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
    ).toBe(0)
    expect(db.parts).toHaveLength(0)
    await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })
  })

  it('releases a timed-out finalizer and preserves the timeout failure code during cleanup', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
    const db = new MemoryDb()
    let now = 0
    const config = { ...TEST_CONFIG, finalizeTimeoutMs: 5 }
    const uploads = service(db, config, {
      now: () => (now += config.partTimeoutMs + 1),
      finalize: async (_session, _parts, signal) =>
        new Promise((_resolve, reject) => {
          signal?.addEventListener('abort', () => reject(signal.reason), { once: true })
        }),
    })
    const created = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'finalizer-timeout.bin',
      sizeBytes: 0,
      idempotencyKey: '44444444-4444-4444-8444-444444444444',
    })
    const uploadDirectory = join(tempRoot, '.uploads', created.session.uploadId)
    await mkdir(uploadDirectory, { recursive: true })
    await writeFile(join(uploadDirectory, 'finalizer-marker'), 'pending')

    const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))

    expect(outcome.kind).toBe('rejected')
    if (outcome.kind !== 'rejected')
      throw new Error(`unexpected completion outcome: ${outcome.kind}`)
    expect(outcome.error).toMatchObject({ code: 'precondition_failed' })
    expect(db.sessions[0]).toMatchObject({
      state: 'failed',
      failure_code: 'precondition_failed',
      cleanup_at: expect.any(String),
    })
    expect(
      (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
    ).toBe(0)
    expect(db.parts).toHaveLength(0)
    await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })
  })

  it('leaves a failed finalizer cleanup eligible for the reconciler after the registry is released', async () => {
    tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
    const db = new MemoryDb()
    let now = 0
    const uploads = service(db, TEST_CONFIG, {
      now: () => (now += TEST_CONFIG.partTimeoutMs + 1),
      finalize: async () => {
        throw new Error('synthetic cleanup retry')
      },
    })
    const created = await uploads.create({
      ...principal,
      operation: 'create',
      parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
      name: 'finalizer-cleanup-retry.bin',
      sizeBytes: 0,
      idempotencyKey: '55555555-5555-4555-8555-555555555555',
    })
    const uploadDirectory = join(tempRoot, '.uploads', created.session.uploadId)
    await mkdir(join(tempRoot, '.uploads'), { recursive: true })
    // A non-directory at the private session path makes the strict direct
    // cleanup fail closed. The terminal row must remain retryable.
    await writeFile(uploadDirectory, 'not-a-directory')

    const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))
    expect(outcome.kind).toBe('rejected')
    expect(db.sessions[0]).toMatchObject({ state: 'failed', cleanup_at: null })
    expect(
      (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
    ).toBe(0)

    // The first reconciliation attempt must exercise the terminal-session
    // cancellation fence, then release it even though filesystem cleanup is
    // still blocked by the non-directory path. This catches a vacuous
    // assertion before reconcile() and proves the fence is bounded across
    // repeated cleanup cycles.
    await expect(uploads.reconcile()).resolves.toEqual({ staleParts: 0, expiredSessions: 0 })
    expect(
      (uploads as unknown as { canceledUploads: Set<string> }).canceledUploads.has(
        created.session.uploadId
      )
    ).toBe(false)

    await rm(uploadDirectory, { force: true })
    await mkdir(uploadDirectory, { recursive: true })
    await writeFile(join(uploadDirectory, 'finalizer-marker'), 'pending')
    await expect(uploads.reconcile()).resolves.toEqual({ staleParts: 0, expiredSessions: 0 })
    expect(db.sessions[0]).toMatchObject({ state: 'failed', cleanup_at: expect.any(String) })
    await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })
  })
})

function createSha(value: Buffer): string {
  return createHash('sha256').update(value).digest('hex')
}