Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

34.8 KB
Raw Blame
//! Segment file format: `.seg` (data) + `.sidx` (skip index)
//!
//! ## `.seg` file layout
//!
//! ```text
//! ┌─────────────────────────────────────────────────────────────────┐
//! │  Header  (64 bytes)                                              │
//! │    magic           [u8;4]   = "XAYA"                            │
//! │    format_version  u16      (currently 1)                       │
//! │    schema_version  u32                                           │
//! │    doc_count       u64                                           │
//! │    min_seq_no      u64                                           │
//! │    max_seq_no      u64                                           │
//! │    created_at_ms   u64      (unix epoch milliseconds)           │
//! │    flags           u16      (bit 0 = has_tombstones)            │
//! │    checksum_algo   u8       (0 = crc32c)                        │
//! │    _pad            [u8;9]                                        │
//! ├─────────────────────────────────────────────────────────────────┤
//! │  Section table length  u32                                       │
//! │  Section table entries (repeating, 25 bytes each)               │
//! │    section_type    u8                                            │
//! │    offset          u64                                           │
//! │    length          u64                                           │
//! │    crc32c          u32                                           │
//! │    _reserved       u4                                            │
//! ├─────────────────────────────────────────────────────────────────┤
//! │  Section data (arbitrary order, addressed by section table)      │
//! ├─────────────────────────────────────────────────────────────────┤
//! │  Footer  (32 bytes)                                              │
//! │    section_table_offset  u64                                     │
//! │    total_crc32c          u32  (over everything except footer)   │
//! │    _pad                  [u8;16]                                 │
//! │    magic_end             [u8;4]   = "BEEZ"                      │
//! └─────────────────────────────────────────────────────────────────┘
//! ```
//!
//! ## `.sidx` file layout
//!
//! The skip index is *intended* to map logical doc-position ranges to file
//! offsets for fast random access.  Its on-disk form is a sorted list of
//! (doc_ordinal, offset) pairs written as little-endian u64 pairs.
//!
//! **Status: written but not yet consumed on the read path.**  Every
//! flush/merge emits a `.sidx` side-car (see [`SegmentWriter::finish`]), but
//! no reader opens it — [`SegmentReader`] addresses sections purely through
//! the in-file section table, and random access to stored fields is already
//! served from an in-memory offset cache (the engine's `stored_slices_cache`).
//! The on-disk skip index is therefore currently redundant; it is retained
//! for format completeness and forward compatibility.  Wiring it into the
//! read path (so cold random access need not rebuild offsets in memory) is
//! future work.

use std::fs::File;
use std::io::{self, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;

use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use crc32fast::Hasher as Crc32Hasher;
use memmap2::Mmap;
use serde::{Deserialize, Serialize};
use tracing::{debug, instrument};
use uuid::Uuid;

use crate::{Result, SeqNo, StorageError};

// ── Constants ─────────────────────────────────────────────────────────────────

const SEG_MAGIC: &[u8; 4] = b"XAYA";
const SEG_MAGIC_END: &[u8; 4] = b"BEEZ";
const FORMAT_VERSION: u16 = 1;
const HEADER_LEN: u64 = 64;
const FOOTER_LEN: u64 = 32;
const SECTION_ENTRY_LEN: u64 = 25; // type(1)+offset(8)+length(8)+crc(4)+reserved(4)

// ── SegmentId ─────────────────────────────────────────────────────────────────

/// Unique identifier for a segment — a UUIDv4 string.
pub type SegmentId = String;

pub fn new_segment_id() -> SegmentId {
    Uuid::new_v4().to_string()
}

// ── SectionType ───────────────────────────────────────────────────────────────

/// Identifies the kind of data stored in a segment section.
///
/// Unknown section types are **skipped** (forward compatibility: a reader built
/// against an older version of the format will silently skip sections it does
/// not understand).
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum SectionType {
    /// Full-text search postings lists.
    Fts = 0x01,
    /// Dense float32 vectors for ANN search.
    Vectors = 0x02,
    /// Column-store (doc-values) for sorting / aggregations.
    Columns = 0x03,
    /// Stored fields: original source JSON.
    Stored = 0x04,
    /// Tombstone bitset (roaring bitmap of deleted doc ordinals).
    Tombstones = 0x05,
    /// Schema snapshot used when writing the segment.
    Schema = 0x06,
    /// Numeric point values (1-D BKD trees) for fast range queries.
    /// One BKD tree per indexed numeric field.
    Points = 0x07,
    /// Unknown — read by older versions; will be skipped.
    #[serde(other)]
    Unknown = 0xFF,
}

impl SectionType {
    fn from_u8(v: u8) -> Self {
        match v {
            0x01 => Self::Fts,
            0x02 => Self::Vectors,
            0x03 => Self::Columns,
            0x04 => Self::Stored,
            0x05 => Self::Tombstones,
            0x06 => Self::Schema,
            0x07 => Self::Points,
            _ => Self::Unknown,
        }
    }
}

// ── Seq-aware tombstone section codec (RC4 W2 #14) ───────────────────────────
//
// `SectionType::Tombstones` payload, version 2:
//
//   "ZTB2"   4 bytes magic
//   u32      pair count
//   lz4_flex::compress_prepend_size(body) where body repeats:
//     u64  delete seq_no (LE)
//     u16  id_len (LE)
//     id_len bytes UTF-8 doc_id
//
// V2 makes acked deletes SEGMENT-DURABLE: reopen applies these pairs
// max-seq-wins against the doc entries rebuilt from `.ids`/stored, so a
// delete survives restart without its WAL entry — which is what finally
// lets WAL maintenance unpin delete-bearing shards (the pre-fix pinning
// retained one WAL generation per shard FOREVER after a single plain
// delete).  The legacy payload (a bare JSON array of doc_id strings,
// written since the 2026-07 delete-durability fix but never read) has no
// seq information and is IGNORED by `decode_tombstones_v2` — those
// deletes remain protected by WAL pinning exactly as before.

/// Encode `(delete_seq_no, doc_id)` pairs as a ZTB2 tombstone section.
pub fn encode_tombstones_v2(pairs: &[(u64, &str)]) -> Vec<u8> {
    let mut body: Vec<u8> =
        Vec::with_capacity(pairs.iter().map(|(_, id)| 8 + 2 + id.len()).sum::<usize>());
    for (seq, id) in pairs {
        body.extend_from_slice(&seq.to_le_bytes());
        body.extend_from_slice(&(id.len() as u16).to_le_bytes());
        body.extend_from_slice(id.as_bytes());
    }
    let compressed = lz4_flex::compress_prepend_size(&body);
    let mut out = Vec::with_capacity(8 + compressed.len());
    out.extend_from_slice(b"ZTB2");
    out.extend_from_slice(&(pairs.len() as u32).to_le_bytes());
    out.extend_from_slice(&compressed);
    out
}

/// Decode a ZTB2 tombstone section.  Returns `None` for the legacy
/// id-only JSON payload (no seq info — cannot be applied safely) or any
/// malformed input.
pub fn decode_tombstones_v2(bytes: &[u8]) -> Option<Vec<(u64, String)>> {
    if bytes.len() < 8 || &bytes[..4] != b"ZTB2" {
        return None;
    }
    let count = u32::from_le_bytes(bytes[4..8].try_into().ok()?) as usize;
    let body = lz4_flex::decompress_size_prepended(&bytes[8..]).ok()?;
    let mut out = Vec::with_capacity(count);
    let mut pos = 0usize;
    for _ in 0..count {
        if pos + 10 > body.len() {
            return None;
        }
        let seq = u64::from_le_bytes(body[pos..pos + 8].try_into().ok()?);
        pos += 8;
        let id_len = u16::from_le_bytes(body[pos..pos + 2].try_into().ok()?) as usize;
        pos += 2;
        if pos + id_len > body.len() {
            return None;
        }
        let id = std::str::from_utf8(&body[pos..pos + id_len])
            .ok()?
            .to_string();
        pos += id_len;
        out.push((seq, id));
    }
    Some(out)
}

// ── SegmentHeader ─────────────────────────────────────────────────────────────

#[derive(Debug, Clone)]
pub struct SegmentHeader {
    pub format_version: u16,
    pub schema_version: u32,
    pub doc_count: u64,
    pub min_seq_no: SeqNo,
    pub max_seq_no: SeqNo,
    pub created_at_ms: u64,
    pub flags: u16,
    pub checksum_algo: u8,
}

impl SegmentHeader {
    fn write_to(&self, w: &mut impl Write) -> io::Result<()> {
        w.write_all(SEG_MAGIC)?;
        w.write_u16::<LittleEndian>(self.format_version)?;
        w.write_u32::<LittleEndian>(self.schema_version)?;
        w.write_u64::<LittleEndian>(self.doc_count)?;
        w.write_u64::<LittleEndian>(self.min_seq_no)?;
        w.write_u64::<LittleEndian>(self.max_seq_no)?;
        w.write_u64::<LittleEndian>(self.created_at_ms)?;
        w.write_u16::<LittleEndian>(self.flags)?;
        w.write_u8(self.checksum_algo)?;
        // Padding: magic(4)+ver(2)+schema_ver(4)+doc_count(8)+min_seq(8)+max_seq(8)
        //          +created_at(8)+flags(2)+checksum_algo(1) = 45 bytes → need 19 more for 64
        w.write_all(&[0u8; 19])?;
        Ok(())
    }

    fn read_from(data: &[u8]) -> Result<Self> {
        use std::io::Cursor;
        let mut r = Cursor::new(data);

        let mut magic = [0u8; 4];
        r.read_exact(&mut magic)?;
        if &magic != SEG_MAGIC {
            return Err(StorageError::InvalidMagic {
                expected: SEG_MAGIC,
                actual: magic.to_vec(),
            });
        }

        let format_version = r.read_u16::<LittleEndian>()?;
        if format_version != FORMAT_VERSION {
            return Err(StorageError::UnsupportedVersion(format_version));
        }

        let schema_version = r.read_u32::<LittleEndian>()?;
        let doc_count = r.read_u64::<LittleEndian>()?;
        let min_seq_no = r.read_u64::<LittleEndian>()?;
        let max_seq_no = r.read_u64::<LittleEndian>()?;
        let created_at_ms = r.read_u64::<LittleEndian>()?;
        let flags = r.read_u16::<LittleEndian>()?;
        let checksum_algo = r.read_u8()?;

        Ok(Self {
            format_version,
            schema_version,
            doc_count,
            min_seq_no,
            max_seq_no,
            created_at_ms,
            flags,
            checksum_algo,
        })
    }
}

// ── SectionEntry ──────────────────────────────────────────────────────────────

#[derive(Debug, Clone)]
struct SectionEntry {
    section_type: SectionType,
    offset: u64,
    length: u64,
    crc32c: u32,
}

impl SectionEntry {
    fn write_to(&self, w: &mut impl Write) -> io::Result<()> {
        w.write_u8(self.section_type as u8)?;
        w.write_u64::<LittleEndian>(self.offset)?;
        w.write_u64::<LittleEndian>(self.length)?;
        w.write_u32::<LittleEndian>(self.crc32c)?;
        w.write_u32::<LittleEndian>(0)?; // reserved
        Ok(())
    }

    fn read_from(r: &mut impl io::Read) -> io::Result<Self> {
        let t = r.read_u8()?;
        let offset = r.read_u64::<LittleEndian>()?;
        let length = r.read_u64::<LittleEndian>()?;
        let crc32c = r.read_u32::<LittleEndian>()?;
        let _reserved = r.read_u32::<LittleEndian>()?;
        Ok(Self {
            section_type: SectionType::from_u8(t),
            offset,
            length,
            crc32c,
        })
    }
}

// ── SegmentMeta ───────────────────────────────────────────────────────────────

/// Lightweight metadata kept in memory for a segment.
///
/// ## Upgrade hygiene (RC4 W3 #10)
///
/// This struct is serialized into the on-disk `snapshot.json` manifest that
/// makes flushed segments discoverable on restart. To keep that manifest
/// forward/backward compatible across xerj versions, every *descriptive*
/// field carries `#[serde(default)]` so that a newer field added in a later
/// release does not break an older snapshot open, and an older snapshot
/// missing a field written by a newer release still loads (the field takes
/// its default). serde already ignores unknown fields, so the reverse
/// direction (older binary, newer manifest) works too.
///
/// The identity/locator fields (`id`, `seg_path`) are deliberately left
/// REQUIRED: a segment entry that lacks them is meaningless, so a manifest
/// missing them is treated as corruption (deserialization fails) rather than
/// silently loading a broken entry.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct SegmentMeta {
    pub id: SegmentId,
    #[serde(default)]
    pub doc_count: u64,
    #[serde(default)]
    pub size_bytes: u64,
    #[serde(default)]
    pub min_seq_no: SeqNo,
    #[serde(default)]
    pub max_seq_no: SeqNo,
    #[serde(default)]
    pub created_at_ms: u64,
    #[serde(default)]
    pub has_tombstones: bool,
    /// Relative path of the `.seg` file from the index root.
    pub seg_path: String,
    /// Relative path of the `.sidx` file from the index root.
    #[serde(default)]
    pub sidx_path: String,
}

// ── SegmentWriter ─────────────────────────────────────────────────────────────

/// Builds a segment incrementally and writes `.seg` + `.sidx` atomically.
///
/// ## Usage
///
/// ```rust,no_run
/// # use xerj_storage::segment::{SegmentWriter, SectionType};
/// # let dir: std::path::PathBuf = "/tmp/idx".into();
/// let mut w = SegmentWriter::new(&dir, 0, 0, 1).unwrap();
/// w.add_section(SectionType::Stored, b"{\"title\":\"hello\"}").unwrap();
/// let meta = w.finish(1 /* doc_count */, 1 /* min_seq */, 1 /* max_seq */).unwrap();
/// ```
pub struct SegmentWriter {
    dir: PathBuf,
    id: SegmentId,
    schema_version: u32,
    sections: Vec<(SectionType, Vec<u8>)>,
    created_at_ms: u64,
}

impl SegmentWriter {
    /// Create a new writer that will produce files in `dir`.
    pub fn new(
        dir: impl AsRef<Path>,
        schema_version: u32,
        _generation: u64,
        _shard: u32,
    ) -> Result<Self> {
        let dir = dir.as_ref().to_path_buf();
        std::fs::create_dir_all(&dir)?;
        let created_at_ms = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap_or_default()
            .as_millis() as u64;
        Ok(Self {
            dir,
            id: new_segment_id(),
            schema_version,
            sections: Vec::new(),
            created_at_ms,
        })
    }

    /// Add a named section with raw bytes.
    pub fn add_section(&mut self, kind: SectionType, data: impl AsRef<[u8]>) -> Result<()> {
        self.sections.push((kind, data.as_ref().to_vec()));
        Ok(())
    }

    /// Finalise, write, and return metadata.
    ///
    /// Files are written to tmp paths then `rename`d so crashes leave no
    /// partial state.
    #[instrument(skip(self), fields(id = %self.id, doc_count, min_seq_no, max_seq_no))]
    pub fn finish(
        &self,
        doc_count: u64,
        min_seq_no: SeqNo,
        max_seq_no: SeqNo,
    ) -> Result<SegmentMeta> {
        let has_tombstones = self
            .sections
            .iter()
            .any(|(t, _)| *t == SectionType::Tombstones);
        let flags: u16 = if has_tombstones { 0x0001 } else { 0x0000 };

        let header = SegmentHeader {
            format_version: FORMAT_VERSION,
            schema_version: self.schema_version,
            doc_count,
            min_seq_no,
            max_seq_no,
            created_at_ms: self.created_at_ms,
            flags,
            checksum_algo: 0, // crc32c
        };

        // Build the section data buffer and section table
        let mut section_data: Vec<u8> = Vec::new();
        let mut section_entries: Vec<SectionEntry> = Vec::with_capacity(self.sections.len());

        // Section data starts after: header(64) + section_table_len(4) + N*25
        // We'll fix up the offset after we know N
        let section_table_len_bytes = 4u64 + self.sections.len() as u64 * SECTION_ENTRY_LEN;
        let data_start = HEADER_LEN + section_table_len_bytes;

        let mut cursor = data_start;
        for (kind, data) in &self.sections {
            let mut hasher = Crc32Hasher::new();
            hasher.update(data);
            let crc = hasher.finalize();
            section_entries.push(SectionEntry {
                section_type: *kind,
                offset: cursor,
                length: data.len() as u64,
                crc32c: crc,
            });
            section_data.extend_from_slice(data);
            cursor += data.len() as u64;
        }

        // Compute where the section table itself will sit (right after header)
        let section_table_offset = HEADER_LEN;
        let section_data_start = data_start;
        let _ = section_data_start; // used in offset calculation above

        // Compute footer position
        let _footer_offset = data_start + section_data.len() as u64;

        // Now assemble the full file in memory for CRC
        let mut buf: Vec<u8> = Vec::new();
        header.write_to(&mut buf)?;

        // Section table: count(u32) + entries
        let mut section_table_buf: Vec<u8> = Vec::new();
        section_table_buf.write_u32::<LittleEndian>(self.sections.len() as u32)?;
        for entry in &section_entries {
            entry.write_to(&mut section_table_buf)?;
        }
        buf.extend_from_slice(&section_table_buf);
        buf.extend_from_slice(&section_data);

        // Total CRC32C over everything except footer
        let mut total_hasher = Crc32Hasher::new();
        total_hasher.update(&buf);
        let total_crc = total_hasher.finalize();

        // Write footer (32 bytes)
        buf.write_u64::<LittleEndian>(section_table_offset)?;
        buf.write_u32::<LittleEndian>(total_crc)?;
        buf.write_all(&[0u8; 16])?; // padding
        buf.write_all(SEG_MAGIC_END)?;

        // Write .seg file
        let seg_filename = format!("{}.seg", self.id);
        let seg_path = self.dir.join(&seg_filename);
        let seg_tmp = seg_path.with_extension("tmp");
        {
            let mut f = File::create(&seg_tmp)?;
            f.write_all(&buf)?;
            f.sync_all()?;
        }
        std::fs::rename(&seg_tmp, &seg_path)?;

        // Write .sidx side-car (simple: N*(doc_ordinal:u64, offset:u64) pairs).
        // NOTE: emitted for format completeness only — the read path does NOT
        // consume it (see the module-level ".sidx file layout" note).
        // `SegmentReader` addresses sections via the in-file section table, and
        // random access to stored fields is served from the engine's in-memory
        // `stored_slices_cache`, so no reader ever opens this file.
        let sidx_filename = format!("{}.sidx", self.id);
        let sidx_path = self.dir.join(&sidx_filename);
        let sidx_tmp = sidx_path.with_extension("tmp");
        {
            let mut f = BufWriter::new(File::create(&sidx_tmp)?);
            // Minimal skip index: one entry pointing to the stored section
            if let Some(stored) = section_entries
                .iter()
                .find(|e| e.section_type == SectionType::Stored)
            {
                f.write_u64::<LittleEndian>(0)?; // first doc ordinal
                f.write_u64::<LittleEndian>(stored.offset)?; // file offset
            }
            f.flush()?;
            f.get_ref().sync_all()?;
        }
        std::fs::rename(&sidx_tmp, &sidx_path)?;

        // RC4 W1 #10 — make the renames themselves power-loss durable.
        // The file CONTENTS were fsynced above, but a rename only becomes
        // durable when the parent directory is fsynced; without this a
        // power loss could leave a fully-written segment invisible in the
        // directory while the WAL entries it covers were already pruned.
        xerj_common::fsio::fsync_dir(&self.dir)?;

        debug!(id = %self.id, size = buf.len(), doc_count, "segment written");

        Ok(SegmentMeta {
            id: self.id.clone(),
            doc_count,
            size_bytes: buf.len() as u64,
            min_seq_no,
            max_seq_no,
            created_at_ms: self.created_at_ms,
            has_tombstones,
            seg_path: seg_filename,
            sidx_path: sidx_filename,
        })
    }
}

// ── SegmentReader ─────────────────────────────────────────────────────────────

/// Memory-mapped segment reader.
///
/// The OS page cache backs the reads — no heap copies for the common path.
/// Section data is validated on first access (CRC32C).
pub struct SegmentReader {
    mmap: Arc<Mmap>,
    header: SegmentHeader,
    sections: Vec<SectionEntry>,
}

impl SegmentReader {
    /// Open the `.seg` file at `path` and validate the header + footer.
    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
        let file = File::open(path.as_ref())?;
        let mmap = unsafe { Mmap::map(&file) }?;
        let mmap = Arc::new(mmap);
        let reader = Self::from_mmap(mmap)?;
        // #819: `from_mmap`'s open-time whole-file CRC pass reads every
        // on-disk byte of the segment, so a validated open is the faithful
        // "bytes read from disk" site. The reader-cache hit path
        // (`from_mmap_arc`) reuses the mmap and reads nothing, so it
        // deliberately does not record.
        if let Some(m) = xerj_common::metrics::global_metrics() {
            m.record_bytes_read(reader.mmap.len() as u64);
        }
        Ok(reader)
    }

    /// Return the underlying Arc<Mmap>.  Used by the cached
    /// `open_segment_arc` path to clone SegmentReaders without
    /// reopening the file.
    pub fn mmap_arc(&self) -> &Arc<Mmap> {
        &self.mmap
    }

    /// Construct a SegmentReader from an already-validated Arc<Mmap>.
    /// Bypasses the full-file CRC check — only used from the
    /// `open_segment_arc` cache path where the same mmap has already
    /// been validated once at cache insertion time.
    pub fn from_mmap_arc(mmap: Arc<Mmap>) -> Result<Self> {
        // Re-parse the section table without redoing the total CRC.
        // The mmap came from a reader that already validated at open.
        let data = &mmap[..];
        let footer_start = data.len() - FOOTER_LEN as usize;
        let mut footer_cur = std::io::Cursor::new(&data[footer_start..]);
        let section_table_offset = footer_cur.read_u64::<LittleEndian>()?;
        let _stored_total_crc = footer_cur.read_u32::<LittleEndian>()?;
        let header = SegmentHeader::read_from(&data[..HEADER_LEN as usize])?;
        let st_start = section_table_offset as usize;
        let mut st_cur = std::io::Cursor::new(&data[st_start..]);
        let section_count = st_cur.read_u32::<LittleEndian>()? as usize;
        let mut sections = Vec::with_capacity(section_count);
        for _ in 0..section_count {
            sections.push(SectionEntry::read_from(&mut st_cur)?);
        }
        Ok(Self {
            mmap,
            header,
            sections,
        })
    }

    fn from_mmap(mmap: Arc<Mmap>) -> Result<Self> {
        let data = &mmap[..];
        if data.len() < (HEADER_LEN + FOOTER_LEN) as usize {
            return Err(StorageError::WalCorrupt(0, "segment file too small".into()));
        }

        // Validate footer magic
        let footer_start = data.len() - FOOTER_LEN as usize;
        let magic_end_offset = data.len() - 4;
        if &data[magic_end_offset..] != SEG_MAGIC_END {
            return Err(StorageError::InvalidMagic {
                expected: SEG_MAGIC_END,
                actual: data[magic_end_offset..].to_vec(),
            });
        }

        // Read footer
        let mut footer_cur = std::io::Cursor::new(&data[footer_start..]);
        let section_table_offset = footer_cur.read_u64::<LittleEndian>()?;
        let stored_total_crc = footer_cur.read_u32::<LittleEndian>()?;

        // Verify total CRC (everything except footer)
        let mut hasher = Crc32Hasher::new();
        hasher.update(&data[..footer_start]);
        let computed_crc = hasher.finalize();
        if stored_total_crc != computed_crc {
            return Err(StorageError::ChecksumMismatch {
                expected: stored_total_crc,
                actual: computed_crc,
            });
        }

        // Parse header
        let header = SegmentHeader::read_from(&data[..HEADER_LEN as usize])?;

        // Parse section table
        let st_start = section_table_offset as usize;
        let mut st_cur = std::io::Cursor::new(&data[st_start..]);
        let section_count = st_cur.read_u32::<LittleEndian>()? as usize;
        let mut sections = Vec::with_capacity(section_count);
        for _ in 0..section_count {
            sections.push(SectionEntry::read_from(&mut st_cur)?);
        }

        Ok(Self {
            mmap,
            header,
            sections,
        })
    }

    /// Return the segment header.
    pub fn header(&self) -> &SegmentHeader {
        &self.header
    }

    /// Return the raw bytes of a section, or `None` if that section type is
    /// not present in this segment.
    ///
    /// M5.20 — section CRC is NO LONGER re-validated on every call.
    /// The whole-file CRC is validated once at open time (in
    /// `from_mmap`); individual section CRCs were verifying the same
    /// immutable bytes on every query, which was a significant share
    /// of CPU time on concurrent-search benchmarks (197 segments ×
    /// ~500 KB stored = ~100 MB of CRC32 per query, 50 ms per query
    /// just in checksum work).  If you want strict per-section CRC
    /// checking, use `section_checked` below.
    pub fn section(&self, kind: SectionType) -> Result<Option<&[u8]>> {
        let entry = match self.sections.iter().find(|e| e.section_type == kind) {
            Some(e) => e,
            None => return Ok(None),
        };
        let start = entry.offset as usize;
        let end = start + entry.length as usize;
        Ok(Some(&self.mmap[start..end]))
    }

    /// Same as `section()` but re-validates the per-section CRC on
    /// every call.  Useful for `fsck`-style integrity checks; the
    /// fast path uses `section()` which relies on the open-time
    /// whole-file CRC.
    pub fn section_checked(&self, kind: SectionType) -> Result<Option<&[u8]>> {
        let entry = match self.sections.iter().find(|e| e.section_type == kind) {
            Some(e) => e,
            None => return Ok(None),
        };
        let start = entry.offset as usize;
        let end = start + entry.length as usize;
        let slice = &self.mmap[start..end];
        let mut hasher = Crc32Hasher::new();
        hasher.update(slice);
        let crc = hasher.finalize();
        if crc != entry.crc32c {
            return Err(StorageError::ChecksumMismatch {
                expected: entry.crc32c,
                actual: crc,
            });
        }
        Ok(Some(slice))
    }

    /// List all section types present in this segment.
    pub fn section_types(&self) -> Vec<SectionType> {
        self.sections.iter().map(|e| e.section_type).collect()
    }
}

// ── Tests ─────────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn write_and_read_segment() {
        let dir = tempfile::tempdir().unwrap();

        let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
        let stored_data = b"{\"title\":\"hello xerj\"}";
        writer
            .add_section(SectionType::Stored, stored_data)
            .unwrap();
        writer
            .add_section(SectionType::Schema, b"{\"fields\":{}}")
            .unwrap();

        let meta = writer.finish(1, 10, 10).unwrap();
        assert_eq!(meta.doc_count, 1);
        assert!(!meta.has_tombstones);

        let seg_path = dir.path().join(&meta.seg_path);
        let reader = SegmentReader::open(&seg_path).unwrap();

        assert_eq!(reader.header().doc_count, 1);
        assert_eq!(reader.header().min_seq_no, 10);

        let stored = reader.section(SectionType::Stored).unwrap().unwrap();
        assert_eq!(stored, stored_data);

        let schema = reader.section(SectionType::Schema).unwrap().unwrap();
        assert_eq!(schema, b"{\"fields\":{}}");

        // Missing section returns None
        assert!(reader.section(SectionType::Fts).unwrap().is_none());
    }

    /// #819: `xerj_bytes_read_total` was registered but never incremented,
    /// so it read a flat zero regardless of read volume. A validated
    /// `SegmentReader::open` now records the opened file's on-disk length
    /// through the process-global metrics handle.
    #[test]
    fn bytes_read_metric_increments_on_segment_open() {
        // Idempotent OnceLock: installs a handle if none yet, else reads the
        // existing one. Once installed, concurrent tests in this binary may
        // also open segments, so assert a lower bound, not an exact delta.
        xerj_common::metrics::set_global_metrics(std::sync::Arc::new(
            xerj_common::metrics::Metrics::new().unwrap(),
        ));
        let m = xerj_common::metrics::global_metrics().expect("global metrics installed");

        let dir = tempfile::tempdir().unwrap();
        let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
        writer
            .add_section(SectionType::Stored, b"{\"title\":\"bytes read 819\"}")
            .unwrap();
        let meta = writer.finish(1, 1, 1).unwrap();
        let seg_path = dir.path().join(&meta.seg_path);
        let seg_len = std::fs::metadata(&seg_path).unwrap().len();
        assert!(seg_len > 0);

        let before = m.bytes_read.get();
        let _reader = SegmentReader::open(&seg_path).unwrap();
        let after = m.bytes_read.get();
        assert!(
            after - before >= seg_len,
            "#819: bytes_read must grow by at least the segment's on-disk \
             length ({seg_len}) on a validated open, got {before} -> {after}"
        );
    }

    #[test]
    fn segment_with_tombstones_sets_flag() {
        let dir = tempfile::tempdir().unwrap();
        let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
        writer
            .add_section(SectionType::Tombstones, [0xFFu8; 4])
            .unwrap();
        let meta = writer.finish(10, 1, 10).unwrap();
        assert!(meta.has_tombstones);

        let reader = SegmentReader::open(dir.path().join(&meta.seg_path)).unwrap();
        assert_ne!(reader.header().flags & 0x0001, 0);
    }

    #[test]
    fn crc_mismatch_detected() {
        let dir = tempfile::tempdir().unwrap();
        let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
        writer.add_section(SectionType::Stored, b"data").unwrap();
        let meta = writer.finish(1, 1, 1).unwrap();

        // Corrupt the file
        let seg_path = dir.path().join(&meta.seg_path);
        let mut content = std::fs::read(&seg_path).unwrap();
        let mid = content.len() / 2;
        content[mid] ^= 0xFF;
        std::fs::write(&seg_path, &content).unwrap();

        let result = SegmentReader::open(&seg_path);
        assert!(matches!(result, Err(StorageError::ChecksumMismatch { .. })));
    }

    #[test]
    fn section_types_listed() {
        let dir = tempfile::tempdir().unwrap();
        let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
        writer.add_section(SectionType::Stored, b"s").unwrap();
        writer.add_section(SectionType::Fts, b"f").unwrap();
        let meta = writer.finish(1, 1, 1).unwrap();

        let reader = SegmentReader::open(dir.path().join(&meta.seg_path)).unwrap();
        let types = reader.section_types();
        assert!(types.contains(&SectionType::Stored));
        assert!(types.contains(&SectionType::Fts));
        assert!(!types.contains(&SectionType::Vectors));
    }

    /// Locks the `.sidx` disclosure: `finish()` emits the skip-index
    /// side-car, but the read path does NOT consume it.  `SegmentReader`
    /// addresses every section through the in-file section table, so a
    /// segment reads back fully even after its `.sidx` file is deleted.
    /// If a future change starts reading `.sidx`, this test will fail and
    /// force the module doc-comment to be updated in step.
    #[test]
    fn sidx_is_written_but_read_path_ignores_it() {
        let dir = tempfile::tempdir().unwrap();
        let mut writer = SegmentWriter::new(dir.path(), 1, 0, 0).unwrap();
        let stored_data = b"{\"title\":\"sidx disclosure\"}";
        writer
            .add_section(SectionType::Stored, stored_data)
            .unwrap();
        let meta = writer.finish(1, 1, 1).unwrap();

        // The side-car is written on finish.
        let sidx_path = dir.path().join(&meta.sidx_path);
        assert!(
            sidx_path.exists(),
            "finish() must emit the .sidx side-car for format completeness"
        );

        // Reads must not depend on it: remove the .sidx and re-open.
        std::fs::remove_file(&sidx_path).unwrap();
        let reader = SegmentReader::open(dir.path().join(&meta.seg_path)).unwrap();
        let stored = reader.section(SectionType::Stored).unwrap().unwrap();
        assert_eq!(
            stored, stored_data,
            "the read path must serve stored fields via the section table, \
             not the (absent) .sidx skip index"
        );
    }
}