Skip to content

web search done right

remove unused

Mikkel Denkere2fe43891230
diff --git a/crates/tantivy/src/columnar/column_index/mod.rs b/crates/tantivy/src/columnar/column_index/mod.rs
index 7a279735..ead2c47d 100644
--- a/crates/tantivy/src/columnar/column_index/mod.rs
+++ b/crates/tantivy/src/columnar/column_index/mod.rs
@@ -4,14 +4,11 @@
//! documents have exactly one element.
mod merge;
-mod multivalued_index;
-mod optional_index;
mod serialize;
use std::ops::Range;
pub use merge::merge_column_index;
-pub use optional_index::{OptionalIndex, Set};
pub use serialize::{open_column_index, serialize_column_index, SerializableColumnIndex};
use super::{Cardinality, DocId, RowId};
diff --git a/crates/tantivy/src/columnar/column_index/multivalued_index.rs b/crates/tantivy/src/columnar/column_index/multivalued_index.rs
deleted file mode 100644
index e8cd8cf5..00000000
--- a/crates/tantivy/src/columnar/column_index/multivalued_index.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-use std::io;
-use std::io::Write;
-use std::sync::Arc;
-
-use crate::common::OwnedBytes;
-
-use super::RowId;
-use crate::columnar::column_values::{
- load_u64_based_column_values, serialize_u64_based_column_values, CodecType, ColumnValues,
-};
-use crate::columnar::iterable::Iterable;
-
-pub fn serialize_multivalued_index(
- multivalued_index: &dyn Iterable<RowId>,
- output: &mut impl Write,
-) -> io::Result<()> {
- serialize_u64_based_column_values(
- multivalued_index,
- &[CodecType::Bitpacked, CodecType::Linear],
- output,
- )?;
- Ok(())
-}
-
-pub fn open_multivalued_index(bytes: OwnedBytes) -> io::Result<MultiValueIndex> {
- let start_index_column: Arc<dyn ColumnValues<RowId>> = load_u64_based_column_values(bytes)?;
- Ok(MultiValueIndex { start_index_column })
-}
-
-#[derive(Clone)]
-/// Index to resolve value range for given doc_id.
-/// Starts at 0.
-pub struct MultiValueIndex {
- pub start_index_column: Arc<dyn crate::columnar::ColumnValues<RowId>>,
-}
-
-impl std::fmt::Debug for MultiValueIndex {
- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
- f.debug_struct("MultiValuedIndex")
- .field("num_rows", &self.start_index_column.num_vals())
- .finish_non_exhaustive()
- }
-}
-
-impl From<Arc<dyn ColumnValues<RowId>>> for MultiValueIndex {
- fn from(start_index_column: Arc<dyn ColumnValues<RowId>>) -> Self {
- MultiValueIndex { start_index_column }
- }
-}
-
-impl MultiValueIndex {
- pub fn for_test(start_offsets: &[RowId]) -> MultiValueIndex {
- let mut buffer = Vec::new();
- serialize_multivalued_index(&start_offsets, &mut buffer).unwrap();
- let bytes = OwnedBytes::new(buffer);
- open_multivalued_index(bytes).unwrap()
- }
-
- /// Returns the number of documents in the index.
- #[inline]
- pub fn num_docs(&self) -> u32 {
- self.start_index_column.num_vals() - 1
- }
-}
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/mod.rs b/crates/tantivy/src/columnar/column_index/optional_index/mod.rs
deleted file mode 100644
index 5eafa50d..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/mod.rs
+++ /dev/null
@@ -1,530 +0,0 @@
-use std::io::{self, Write};
-use std::sync::Arc;
-
-mod set;
-mod set_block;
-
-use crate::common::{BinarySerializable, OwnedBytes, VInt};
-pub use set::{SelectCursor, Set, SetCodec};
-use set_block::{
- DenseBlock, DenseBlockCodec, SparseBlock, SparseBlockCodec, DENSE_BLOCK_NUM_BYTES,
-};
-
-use crate::columnar::iterable::Iterable;
-use crate::columnar::{DocId, InvalidData, RowId};
-
-/// The threshold for for number of elements after which we switch to dense block encoding.
-///
-/// We simply pick the value that minimize the size of the blocks.
-const DENSE_BLOCK_THRESHOLD: u32 =
- set_block::DENSE_BLOCK_NUM_BYTES / std::mem::size_of::<u16>() as u32; //< 5_120
-
-const ELEMENTS_PER_BLOCK: u32 = u16::MAX as u32 + 1;
-
-#[derive(Copy, Clone, Debug)]
-struct BlockMeta {
- non_null_rows_before_block: u32,
- start_byte_offset: u32,
- block_variant: BlockVariant,
-}
-
-#[derive(Clone, Copy, Debug)]
-enum BlockVariant {
- Dense,
- Sparse { num_vals: u16 },
-}
-
-impl BlockVariant {
- pub fn empty() -> Self {
- Self::Sparse { num_vals: 0 }
- }
- pub fn num_bytes_in_block(&self) -> u32 {
- match *self {
- BlockVariant::Dense => set_block::DENSE_BLOCK_NUM_BYTES,
- BlockVariant::Sparse { num_vals } => num_vals as u32 * 2,
- }
- }
-}
-
-/// This codec is inspired by roaring bitmaps.
-/// In the dense blocks, however, in order to accelerate `select`
-/// we interleave an offset over two bytes. (more on this lower)
-///
-/// The lower 16 bits of doc ids are stored as u16 while the upper 16 bits are given by the block
-/// id. Each block contains 1<<16 docids.
-///
-/// # Serialized Data Layout
-/// The data starts with the block data. Each block is either dense or sparse encoded, depending on
-/// the number of values in the block. A block is sparse when it contains less than
-/// DENSE_BLOCK_THRESHOLD (6144) values.
-/// [Sparse data block | dense data block, .. #repeat*; Desc: Either a sparse or dense encoded
-/// block]
-/// ### Sparse block data
-/// [u16 LE, .. #repeat*; Desc: Positions with values in a block]
-/// ### Dense block data
-/// [Dense codec for the whole block; Desc: Similar to a bitvec(0..ELEMENTS_PER_BLOCK) + Metadata
-/// for faster lookups. See dense.rs]
-///
-/// The data is followed by block metadata, to know which area of the raw block data belongs to
-/// which block. Only metadata for blocks with elements is recorded to
-/// keep the overhead low for scenarios with many very sparse columns. The block metadata consists
-/// of the block index and the number of values in the block. Since we don't store empty blocks
-/// num_vals is incremented by 1, e.g. 0 means 1 value.
-///
-/// The last u16 is storing the number of metadata blocks.
-/// [u16 LE, .. #repeat*; Desc: Positions with values in a block][(u16 LE, u16 LE), .. #repeat*;
-/// Desc: (Block Id u16, Num Elements u16)][u16 LE; Desc: num blocks with values u16]
-///
-/// # Opening
-/// When opening the data layout, the data is expanded to `Vec<SparseCodecBlockVariant>`, where the
-/// index is the block index. For each block `byte_start` and `offset` is computed.
-#[derive(Clone)]
-pub struct OptionalIndex {
- num_rows: RowId,
- num_non_null_rows: RowId,
- block_data: OwnedBytes,
- block_metas: Arc<[BlockMeta]>,
-}
-
-impl std::fmt::Debug for OptionalIndex {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("OptionalIndex")
- .field("num_rows", &self.num_rows)
- .field("num_non_null_rows", &self.num_non_null_rows)
- .finish_non_exhaustive()
- }
-}
-
-/// Splits a value address into lower and upper 16bits.
-/// The lower 16 bits are the value in the block
-/// The upper 16 bits are the block index
-#[derive(Copy, Debug, Clone)]
-struct RowAddr {
- block_id: u16,
- in_block_row_id: u16,
-}
-
-#[inline(always)]
-fn row_addr_from_row_id(row_id: RowId) -> RowAddr {
- RowAddr {
- block_id: (row_id / ELEMENTS_PER_BLOCK) as u16,
- in_block_row_id: (row_id % ELEMENTS_PER_BLOCK) as u16,
- }
-}
-
-enum BlockSelectCursor<'a> {
- Dense(<DenseBlock<'a> as Set<u16>>::SelectCursor<'a>),
- Sparse(<SparseBlock<'a> as Set<u16>>::SelectCursor<'a>),
-}
-
-impl<'a> BlockSelectCursor<'a> {
- fn select(&mut self, rank: u16) -> u16 {
- match self {
- BlockSelectCursor::Dense(dense_select_cursor) => dense_select_cursor.select(rank),
- BlockSelectCursor::Sparse(sparse_select_cursor) => sparse_select_cursor.select(rank),
- }
- }
-}
-pub struct OptionalIndexSelectCursor<'a> {
- current_block_cursor: BlockSelectCursor<'a>,
- current_block_id: u16,
- // The current block is guaranteed to contain ranks < end_rank.
- current_block_end_rank: RowId,
- optional_index: &'a OptionalIndex,
- block_doc_idx_start: RowId,
- num_null_rows_before_block: RowId,
-}
-
-impl<'a> OptionalIndexSelectCursor<'a> {
- fn search_and_load_block(&mut self, rank: RowId) {
- if rank < self.current_block_end_rank {
- // we are already in the right block
- return;
- }
- self.current_block_id = self.optional_index.find_block(rank, self.current_block_id);
- self.current_block_end_rank = self
- .optional_index
- .block_metas
- .get(self.current_block_id as usize + 1)
- .map(|block_meta| block_meta.non_null_rows_before_block)
- .unwrap_or(u32::MAX);
- self.block_doc_idx_start = (self.current_block_id as u32) * ELEMENTS_PER_BLOCK;
- let block_meta = self.optional_index.block_metas[self.current_block_id as usize];
- self.num_null_rows_before_block = block_meta.non_null_rows_before_block;
- let block: Block<'_> = self.optional_index.block(block_meta);
- self.current_block_cursor = match block {
- Block::Dense(dense_block) => BlockSelectCursor::Dense(dense_block.select_cursor()),
- Block::Sparse(sparse_block) => BlockSelectCursor::Sparse(sparse_block.select_cursor()),
- };
- }
-}
-
-impl<'a> SelectCursor<RowId> for OptionalIndexSelectCursor<'a> {
- fn select(&mut self, rank: RowId) -> RowId {
- self.search_and_load_block(rank);
- let index_in_block = (rank - self.num_null_rows_before_block) as u16;
- self.current_block_cursor.select(index_in_block) as RowId + self.block_doc_idx_start
- }
-}
-
-impl Set<RowId> for OptionalIndex {
- type SelectCursor<'b> = OptionalIndexSelectCursor<'b> where Self: 'b;
- // Check if value at position is not null.
- #[inline]
- fn contains(&self, row_id: RowId) -> bool {
- let RowAddr {
- block_id,
- in_block_row_id,
- } = row_addr_from_row_id(row_id);
- let block_meta = self.block_metas[block_id as usize];
- match self.block(block_meta) {
- Block::Dense(dense_block) => dense_block.contains(in_block_row_id),
- Block::Sparse(sparse_block) => sparse_block.contains(in_block_row_id),
- }
- }
-
- /// Any value doc_id is allowed.
- /// In particular, doc_id = num_rows.
- #[inline]
- fn rank(&self, doc_id: DocId) -> RowId {
- if doc_id >= self.num_docs() {
- return self.num_non_nulls();
- }
- let RowAddr {
- block_id,
- in_block_row_id,
- } = row_addr_from_row_id(doc_id);
- let block_meta = self.block_metas[block_id as usize];
- let block = self.block(block_meta);
-
- let block_offset_row_id = match block {
- Block::Dense(dense_block) => dense_block.rank(in_block_row_id),
- Block::Sparse(sparse_block) => sparse_block.rank(in_block_row_id),
- } as u32;
- block_meta.non_null_rows_before_block + block_offset_row_id
- }
-
- /// Any value doc_id is allowed.
- /// In particular, doc_id = num_rows.
- #[inline]
- fn rank_if_exists(&self, doc_id: DocId) -> Option<RowId> {
- let RowAddr {
- block_id,
- in_block_row_id,
- } = row_addr_from_row_id(doc_id);
- let block_meta = *self.block_metas.get(block_id as usize)?;
- let block = self.block(block_meta);
- let block_offset_row_id = match block {
- Block::Dense(dense_block) => dense_block.rank_if_exists(in_block_row_id),
- Block::Sparse(sparse_block) => sparse_block.rank_if_exists(in_block_row_id),
- }? as u32;
- Some(block_meta.non_null_rows_before_block + block_offset_row_id)
- }
-
- #[inline]
- fn select(&self, rank: RowId) -> RowId {
- let block_pos = self.find_block(rank, 0);
- let block_doc_idx_start = (block_pos as u32) * ELEMENTS_PER_BLOCK;
- let block_meta = self.block_metas[block_pos as usize];
- let block: Block<'_> = self.block(block_meta);
- let index_in_block = (rank - block_meta.non_null_rows_before_block) as u16;
- let in_block_rank = match block {
- Block::Dense(dense_block) => dense_block.select(index_in_block),
- Block::Sparse(sparse_block) => sparse_block.select(index_in_block),
- };
- block_doc_idx_start + in_block_rank as u32
- }
-
- fn select_cursor(&self) -> OptionalIndexSelectCursor<'_> {
- OptionalIndexSelectCursor {
- current_block_cursor: BlockSelectCursor::Sparse(
- SparseBlockCodec::open(b"").select_cursor(),
- ),
- current_block_id: 0u16,
- current_block_end_rank: 0u32, //< this is sufficient to force the first load
- optional_index: self,
- block_doc_idx_start: 0u32,
- num_null_rows_before_block: 0u32,
- }
- }
-}
-
-impl OptionalIndex {
- pub fn for_test(num_rows: RowId, row_ids: &[RowId]) -> OptionalIndex {
- assert!(row_ids
- .last()
- .copied()
- .map(|last_row_id| last_row_id < num_rows)
- .unwrap_or(true));
- let mut buffer = Vec::new();
- serialize_optional_index(&row_ids, num_rows, &mut buffer).unwrap();
- let bytes = OwnedBytes::new(buffer);
- open_optional_index(bytes).unwrap()
- }
-
- pub fn num_docs(&self) -> RowId {
- self.num_rows
- }
-
- pub fn num_non_nulls(&self) -> RowId {
- self.num_non_null_rows
- }
-
- pub fn iter_rows(&self) -> impl Iterator<Item = RowId> + '_ {
- // TODO optimize
- let mut select_batch = self.select_cursor();
- (0..self.num_non_null_rows).map(move |rank| select_batch.select(rank))
- }
- pub fn select_batch(&self, ranks: &mut [RowId]) {
- let mut select_cursor = self.select_cursor();
- for rank in ranks.iter_mut() {
- *rank = select_cursor.select(*rank);
- }
- }
-
- #[inline]
- fn block(&self, block_meta: BlockMeta) -> Block<'_> {
- let BlockMeta {
- start_byte_offset,
- block_variant,
- ..
- } = block_meta;
- let start_byte_offset = start_byte_offset as usize;
- let bytes = self.block_data.as_slice();
- match block_variant {
- BlockVariant::Dense => Block::Dense(DenseBlockCodec::open(
- &bytes[start_byte_offset..start_byte_offset + DENSE_BLOCK_NUM_BYTES as usize],
- )),
- BlockVariant::Sparse { num_vals } => {
- let end_byte_offset = start_byte_offset + num_vals as usize * 2;
- let sparse_bytes = &bytes[start_byte_offset..end_byte_offset];
- Block::Sparse(SparseBlockCodec::open(sparse_bytes))
- }
- }
- }
-
- #[inline]
- fn find_block(&self, dense_idx: u32, start_block_pos: u16) -> u16 {
- for block_pos in start_block_pos..self.block_metas.len() as u16 {
- let offset = self.block_metas[block_pos as usize].non_null_rows_before_block;
- if offset > dense_idx {
- return block_pos - 1u16;
- }
- }
- self.block_metas.len() as u16 - 1u16
- }
-
- // TODO Add a good API for the codec_idx to original_idx translation.
- // The Iterator API is a probably a bad idea
-}
-
-#[derive(Copy, Clone)]
-enum Block<'a> {
- Dense(DenseBlock<'a>),
- Sparse(SparseBlock<'a>),
-}
-
-#[derive(Debug, Copy, Clone)]
-enum OptionalIndexCodec {
- Dense = 0,
- Sparse = 1,
-}
-
-impl OptionalIndexCodec {
- fn to_code(self) -> u8 {
- self as u8
- }
-
- fn try_from_code(code: u8) -> Result<Self, InvalidData> {
- match code {
- 0 => Ok(Self::Dense),
- 1 => Ok(Self::Sparse),
- _ => Err(InvalidData),
- }
- }
-}
-
-impl BinarySerializable for OptionalIndexCodec {
- fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
- writer.write_all(&[self.to_code()])
- }
-
- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
- let optional_codec_code = u8::deserialize(reader)?;
- let optional_codec = Self::try_from_code(optional_codec_code)?;
- Ok(optional_codec)
- }
-}
-
-fn serialize_optional_index_block(block_els: &[u16], out: &mut impl io::Write) -> io::Result<()> {
- let is_sparse = is_sparse(block_els.len() as u32);
- if is_sparse {
- SparseBlockCodec::serialize(block_els.iter().copied(), out)?;
- } else {
- DenseBlockCodec::serialize(block_els.iter().copied(), out)?;
- }
- Ok(())
-}
-
-pub fn serialize_optional_index<W: io::Write>(
- non_null_rows: &dyn Iterable<RowId>,
- num_rows: RowId,
- output: &mut W,
-) -> io::Result<()> {
- VInt(num_rows as u64).serialize(output)?;
-
- let mut rows_it = non_null_rows.boxed_iter();
- let mut block_metadata: Vec<SerializedBlockMeta> = Vec::new();
- let mut current_block = Vec::new();
-
- // This if-statement for the first element ensures that
- // `block_metadata` is not empty in the loop below.
- let Some(idx) = rows_it.next() else {
- output.write_all(&0u16.to_le_bytes())?;
- return Ok(());
- };
-
- let row_addr = row_addr_from_row_id(idx);
-
- let mut current_block_id = row_addr.block_id;
- current_block.push(row_addr.in_block_row_id);
-
- for idx in rows_it {
- let value_addr = row_addr_from_row_id(idx);
- if current_block_id != value_addr.block_id {
- serialize_optional_index_block(&current_block[..], output)?;
- block_metadata.push(SerializedBlockMeta {
- block_id: current_block_id,
- num_non_null_rows: current_block.len() as u32,
- });
- current_block.clear();
- current_block_id = value_addr.block_id;
- }
- current_block.push(value_addr.in_block_row_id);
- }
-
- // handle last block
- serialize_optional_index_block(&current_block[..], output)?;
-
- block_metadata.push(SerializedBlockMeta {
- block_id: current_block_id,
- num_non_null_rows: current_block.len() as u32,
- });
-
- for block in &block_metadata {
- output.write_all(&block.to_bytes())?;
- }
-
- output.write_all((block_metadata.len() as u16).to_le_bytes().as_ref())?;
-
- Ok(())
-}
-
-const SERIALIZED_BLOCK_META_NUM_BYTES: usize = 4;
-
-#[derive(Clone, Copy, Debug)]
-struct SerializedBlockMeta {
- block_id: u16,
- num_non_null_rows: u32, //< takes values in 1..=u16::MAX
-}
-
-// TODO unit tests
-impl SerializedBlockMeta {
- #[inline]
- fn from_bytes(bytes: [u8; SERIALIZED_BLOCK_META_NUM_BYTES]) -> SerializedBlockMeta {
- let block_id = u16::from_le_bytes(bytes[0..2].try_into().unwrap());
- let num_non_null_rows: u32 =
- u16::from_le_bytes(bytes[2..4].try_into().unwrap()) as u32 + 1u32;
- SerializedBlockMeta {
- block_id,
- num_non_null_rows,
- }
- }
-
- #[inline]
- fn to_bytes(self) -> [u8; SERIALIZED_BLOCK_META_NUM_BYTES] {
- assert!(self.num_non_null_rows > 0);
- let mut bytes = [0u8; SERIALIZED_BLOCK_META_NUM_BYTES];
- bytes[0..2].copy_from_slice(&self.block_id.to_le_bytes());
- // We don't store empty blocks, therefore we can subtract 1.
- // This way we will be able to use u16 when the number of elements is 1 << 16 or u16::MAX+1
- bytes[2..4].copy_from_slice(&((self.num_non_null_rows - 1u32) as u16).to_le_bytes());
- bytes
- }
-}
-
-#[inline]
-fn is_sparse(num_rows_in_block: u32) -> bool {
- num_rows_in_block < DENSE_BLOCK_THRESHOLD
-}
-
-fn deserialize_optional_index_block_metadatas(
- data: &[u8],
- num_rows: u32,
-) -> (Box<[BlockMeta]>, u32) {
- let num_blocks = data.len() / SERIALIZED_BLOCK_META_NUM_BYTES;
- let mut block_metas = Vec::with_capacity(num_blocks + 1);
- let mut start_byte_offset = 0;
- let mut non_null_rows_before_block = 0;
- for block_meta_bytes in data.chunks_exact(SERIALIZED_BLOCK_META_NUM_BYTES) {
- let block_meta_bytes: [u8; SERIALIZED_BLOCK_META_NUM_BYTES] =
- block_meta_bytes.try_into().unwrap();
- let SerializedBlockMeta {
- block_id,
- num_non_null_rows,
- } = SerializedBlockMeta::from_bytes(block_meta_bytes);
- block_metas.resize(
- block_id as usize,
- BlockMeta {
- non_null_rows_before_block,
- start_byte_offset,
- block_variant: BlockVariant::empty(),
- },
- );
- let block_variant = if is_sparse(num_non_null_rows) {
- BlockVariant::Sparse {
- num_vals: num_non_null_rows as u16,
- }
- } else {
- BlockVariant::Dense
- };
- block_metas.push(BlockMeta {
- non_null_rows_before_block,
- start_byte_offset,
- block_variant,
- });
- start_byte_offset += block_variant.num_bytes_in_block();
- non_null_rows_before_block += num_non_null_rows;
- }
- block_metas.resize(
- ((num_rows + ELEMENTS_PER_BLOCK - 1) / ELEMENTS_PER_BLOCK) as usize,
- BlockMeta {
- non_null_rows_before_block,
- start_byte_offset,
- block_variant: BlockVariant::empty(),
- },
- );
- (block_metas.into_boxed_slice(), non_null_rows_before_block)
-}
-
-pub fn open_optional_index(bytes: OwnedBytes) -> io::Result<OptionalIndex> {
- let (mut bytes, num_non_empty_blocks_bytes) = bytes.rsplit(2);
- let num_non_empty_block_bytes =
- u16::from_le_bytes(num_non_empty_blocks_bytes.as_slice().try_into().unwrap());
- let num_rows = VInt::deserialize_u64(&mut bytes)? as u32;
- let block_metas_num_bytes =
- num_non_empty_block_bytes as usize * SERIALIZED_BLOCK_META_NUM_BYTES;
- let (block_data, block_metas) = bytes.rsplit(block_metas_num_bytes);
- let (block_metas, num_non_null_rows) =
- deserialize_optional_index_block_metadatas(block_metas.as_slice(), num_rows);
- let optional_index = OptionalIndex {
- num_rows,
- num_non_null_rows,
- block_data,
- block_metas: block_metas.into(),
- };
- Ok(optional_index)
-}
-
-#[cfg(test)]
-mod tests;
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/set.rs b/crates/tantivy/src/columnar/column_index/optional_index/set.rs
deleted file mode 100644
index b2bf9cbe..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/set.rs
+++ /dev/null
@@ -1,48 +0,0 @@
-use std::io;
-
-/// A codec makes it possible to serialize a set of
-/// elements, and open the resulting Set representation.
-pub trait SetCodec {
- type Item: Copy + TryFrom<usize> + Eq + std::hash::Hash + std::fmt::Debug;
- type Reader<'a>: Set<Self::Item>;
-
- /// Serializes a set of unique sorted u16 elements.
- ///
- /// May panic if the elements are not sorted.
- fn serialize(els: impl Iterator<Item = Self::Item>, wrt: impl io::Write) -> io::Result<()>;
- fn open(data: &[u8]) -> Self::Reader<'_>;
-}
-
-/// Stateful object that makes it possible to compute several select in a row,
-/// provided the rank passed as argument are increasing.
-pub trait SelectCursor<T> {
- // May panic if rank is greater than the number of elements in the Set,
- // or if rank is < than value provided in the previous call.
- fn select(&mut self, rank: T) -> T;
-}
-
-pub trait Set<T> {
- type SelectCursor<'b>: SelectCursor<T>
- where Self: 'b;
-
- /// Returns true if the elements is contained in the Set
- fn contains(&self, el: T) -> bool;
-
- /// Returns the number of rows in the set that are < `el`
- fn rank(&self, el: T) -> T;
-
- /// If the set contains `el` returns the element rank.
- /// If the set does not contain the element, it returns `None`.
- fn rank_if_exists(&self, el: T) -> Option<T>;
-
- /// Return the rank-th value stored in this bitmap.
- ///
- /// # Panics
- ///
- /// May panic if rank is greater or equal to the number of
- /// elements in the Set.
- fn select(&self, rank: T) -> T;
-
- /// Creates a brand new select cursor.
- fn select_cursor(&self) -> Self::SelectCursor<'_>;
-}
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/set_block/dense.rs b/crates/tantivy/src/columnar/column_index/optional_index/set_block/dense.rs
deleted file mode 100644
index 4c582b1f..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/set_block/dense.rs
+++ /dev/null
@@ -1,279 +0,0 @@
-use std::io::{self, Write};
-
-use crate::common::BinarySerializable;
-
-use crate::columnar::column_index::optional_index::{
- SelectCursor, Set, SetCodec, ELEMENTS_PER_BLOCK,
-};
-
-#[inline(always)]
-fn get_bit_at(input: u64, n: u16) -> bool {
- input & (1 << n) != 0
-}
-
-#[inline]
-fn set_bit_at(input: &mut u64, n: u16) {
- *input |= 1 << n;
-}
-
-/// For the `DenseCodec`, `data` which contains the encoded blocks.
-/// Each block consists of [u8; 12]. The first 8 bytes is a bitvec for 64 elements.
-/// The last 4 bytes are the offset, the number of set bits so far.
-///
-/// When translating the original index to a dense index, the correct block can be computed
-/// directly `orig_idx/64`. Inside the block the position is `orig_idx%64`.
-///
-/// When translating a dense index to the original index, we can use the offset to find the correct
-/// block. Direct computation is not possible, but we can employ a linear or binary search.
-
-const ELEMENTS_PER_MINI_BLOCK: u16 = 64;
-const MINI_BLOCK_BITVEC_NUM_BYTES: usize = 8;
-const MINI_BLOCK_OFFSET_NUM_BYTES: usize = 2;
-pub const MINI_BLOCK_NUM_BYTES: usize = MINI_BLOCK_BITVEC_NUM_BYTES + MINI_BLOCK_OFFSET_NUM_BYTES;
-
-/// Number of bytes in a dense block.
-pub const DENSE_BLOCK_NUM_BYTES: u32 =
- (ELEMENTS_PER_BLOCK / ELEMENTS_PER_MINI_BLOCK as u32) * MINI_BLOCK_NUM_BYTES as u32;
-
-pub struct DenseBlockCodec;
-
-impl SetCodec for DenseBlockCodec {
- type Item = u16;
- type Reader<'a> = DenseBlock<'a>;
-
- fn serialize(els: impl Iterator<Item = u16>, wrt: impl io::Write) -> io::Result<()> {
- serialize_dense_codec(els, wrt)
- }
-
- #[inline]
- fn open(data: &[u8]) -> Self::Reader<'_> {
- assert_eq!(data.len(), DENSE_BLOCK_NUM_BYTES as usize);
- DenseBlock(data)
- }
-}
-
-/// Interpreting the bitvec as a set of integer within 0..=63
-/// and given an element, returns the number of elements in the
-/// set lesser than the element.
-///
-/// # Panics
-///
-/// May panic or return a wrong result if el <= 64.
-#[inline(always)]
-fn rank_u64(bitvec: u64, el: u16) -> u16 {
- debug_assert!(el < 64);
- let mask = (1u64 << el) - 1;
- let masked_bitvec = bitvec & mask;
- masked_bitvec.count_ones() as u16
-}
-
-#[inline(always)]
-fn select_u64(mut bitvec: u64, rank: u16) -> u16 {
- for _ in 0..rank {
- bitvec &= bitvec - 1;
- }
- bitvec.trailing_zeros() as u16
-}
-
-// TODO test the following solution on Intel... on Ryzen Zen <3 it is a catastrophy.
-// #[target_feature(enable = "bmi2")]
-// unsafe fn select_bitvec_unsafe(bitvec: u64, rank: u16) -> u16 {
-// let pdep = _pdep_u64(1u64 << rank, bitvec);
-// pdep.trailing_zeros() as u16
-// }
-
-#[derive(Clone, Copy, Debug)]
-struct DenseMiniBlock {
- bitvec: u64,
- rank: u16,
-}
-
-impl DenseMiniBlock {
- fn from_bytes(data: [u8; MINI_BLOCK_NUM_BYTES]) -> Self {
- let bitvec = u64::from_le_bytes(data[..MINI_BLOCK_BITVEC_NUM_BYTES].try_into().unwrap());
- let rank = u16::from_le_bytes(data[MINI_BLOCK_BITVEC_NUM_BYTES..].try_into().unwrap());
- Self { bitvec, rank }
- }
-
- fn to_bytes(self) -> [u8; MINI_BLOCK_NUM_BYTES] {
- let mut bytes = [0u8; MINI_BLOCK_NUM_BYTES];
- bytes[..MINI_BLOCK_BITVEC_NUM_BYTES].copy_from_slice(&self.bitvec.to_le_bytes());
- bytes[MINI_BLOCK_BITVEC_NUM_BYTES..].copy_from_slice(&self.rank.to_le_bytes());
- bytes
- }
-}
-
-#[derive(Copy, Clone)]
-pub struct DenseBlock<'a>(&'a [u8]);
-
-pub struct DenseBlockSelectCursor<'a> {
- block_id: u16,
- dense_block: DenseBlock<'a>,
-}
-
-impl<'a> SelectCursor<u16> for DenseBlockSelectCursor<'a> {
- #[inline]
- fn select(&mut self, rank: u16) -> u16 {
- self.block_id = self
- .dense_block
- .find_miniblock_containing_rank(rank, self.block_id)
- .unwrap();
- let index_block = self.dense_block.mini_block(self.block_id);
- let in_block_rank = rank - index_block.rank;
- self.block_id * ELEMENTS_PER_MINI_BLOCK + select_u64(index_block.bitvec, in_block_rank)
- }
-}
-
-impl<'a> Set<u16> for DenseBlock<'a> {
- type SelectCursor<'b> = DenseBlockSelectCursor<'a> where Self: 'b;
-
- #[inline(always)]
- fn contains(&self, el: u16) -> bool {
- let mini_block_id = el / ELEMENTS_PER_MINI_BLOCK;
- let bitvec = self.mini_block(mini_block_id).bitvec;
- let pos_in_bitvec = el % ELEMENTS_PER_MINI_BLOCK;
- get_bit_at(bitvec, pos_in_bitvec)
- }
-
- #[inline(always)]
- fn rank_if_exists(&self, el: u16) -> Option<u16> {
- let block_pos = el / ELEMENTS_PER_MINI_BLOCK;
- let index_block = self.mini_block(block_pos);
- let pos_in_block_bit_vec = el % ELEMENTS_PER_MINI_BLOCK;
- let ones_in_block = rank_u64(index_block.bitvec, pos_in_block_bit_vec);
- let rank = index_block.rank + ones_in_block;
- if get_bit_at(index_block.bitvec, pos_in_block_bit_vec) {
- Some(rank)
- } else {
- None
- }
- }
-
- #[inline(always)]
- fn rank(&self, el: u16) -> u16 {
- let block_pos = el / ELEMENTS_PER_MINI_BLOCK;
- let index_block = self.mini_block(block_pos);
- let pos_in_block_bit_vec = el % ELEMENTS_PER_MINI_BLOCK;
- let ones_in_block = rank_u64(index_block.bitvec, pos_in_block_bit_vec);
- index_block.rank + ones_in_block
- }
-
- #[inline(always)]
- fn select(&self, rank: u16) -> u16 {
- let block_id = self.find_miniblock_containing_rank(rank, 0).unwrap();
- let index_block = self.mini_block(block_id);
- let in_block_rank = rank - index_block.rank;
- block_id * ELEMENTS_PER_MINI_BLOCK + select_u64(index_block.bitvec, in_block_rank)
- }
-
- #[inline(always)]
- fn select_cursor(&self) -> Self::SelectCursor<'_> {
- DenseBlockSelectCursor {
- block_id: 0,
- dense_block: *self,
- }
- }
-}
-
-impl<'a> DenseBlock<'a> {
- #[inline]
- fn mini_block(&self, mini_block_id: u16) -> DenseMiniBlock {
- let data_start_pos = mini_block_id as usize * MINI_BLOCK_NUM_BYTES;
- DenseMiniBlock::from_bytes(
- self.0[data_start_pos..data_start_pos + MINI_BLOCK_NUM_BYTES]
- .try_into()
- .unwrap(),
- )
- }
-
- #[inline]
- fn iter_miniblocks(
- &self,
- from_block_id: u16,
- ) -> impl Iterator<Item = (u16, DenseMiniBlock)> + '_ {
- self.0
- .chunks_exact(MINI_BLOCK_NUM_BYTES)
- .enumerate()
- .skip(from_block_id as usize)
- .map(|(block_id, bytes)| {
- let mini_block = DenseMiniBlock::from_bytes(bytes.try_into().unwrap());
- (block_id as u16, mini_block)
- })
- }
-
- /// Finds the block position containing the dense_idx.
- ///
- /// # Correctness
- /// dense_idx needs to be smaller than the number of values in the index
- ///
- /// The last offset number is equal to the number of values in the index.
- #[inline]
- fn find_miniblock_containing_rank(&self, rank: u16, from_block_id: u16) -> Option<u16> {
- self.iter_miniblocks(from_block_id)
- .take_while(|(_, block)| block.rank <= rank)
- .map(|(block_id, _)| block_id)
- .last()
- }
-}
-
-/// Iterator over all values, true if set, otherwise false
-pub fn serialize_dense_codec(
- els: impl Iterator<Item = u16>,
- mut output: impl Write,
-) -> io::Result<()> {
- let mut non_null_rows_before: u16 = 0u16;
- let mut block = 0u64;
- let mut current_block_id = 0u16;
- for el in els {
- let block_id = el / ELEMENTS_PER_MINI_BLOCK;
- let in_offset = el % ELEMENTS_PER_MINI_BLOCK;
- while block_id > current_block_id {
- let dense_mini_block = DenseMiniBlock {
- bitvec: block,
- rank: non_null_rows_before,
- };
- output.write_all(&dense_mini_block.to_bytes())?;
- non_null_rows_before += block.count_ones() as u16;
- block = 0u64;
- current_block_id += 1u16;
- }
- set_bit_at(&mut block, in_offset);
- }
- while current_block_id <= u16::MAX / ELEMENTS_PER_MINI_BLOCK {
- block.serialize(&mut output)?;
- non_null_rows_before.serialize(&mut output)?;
- // This will overflow to 0 exactly if all bits are set.
- // This is however not problem as we won't use this last value.
- non_null_rows_before = non_null_rows_before.wrapping_add(block.count_ones() as u16);
- block = 0u64;
- current_block_id += 1u16;
- }
- Ok(())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_select_bitvec() {
- assert_eq!(select_u64(1u64, 0), 0);
- assert_eq!(select_u64(2u64, 0), 1);
- assert_eq!(select_u64(4u64, 0), 2);
- assert_eq!(select_u64(8u64, 0), 3);
- assert_eq!(select_u64(1 | 8u64, 0), 0);
- assert_eq!(select_u64(1 | 8u64, 1), 3);
- }
-
- #[test]
- fn test_count_ones() {
- for i in 0..=63 {
- assert_eq!(rank_u64(u64::MAX, i), i);
- }
- }
-
- #[test]
- fn test_dense() {
- assert_eq!(DENSE_BLOCK_NUM_BYTES, 10_240);
- }
-}
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/set_block/mod.rs b/crates/tantivy/src/columnar/column_index/optional_index/set_block/mod.rs
deleted file mode 100644
index fdfc0062..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/set_block/mod.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-mod dense;
-mod sparse;
-
-pub use dense::{DenseBlock, DenseBlockCodec, DENSE_BLOCK_NUM_BYTES};
-pub use sparse::{SparseBlock, SparseBlockCodec};
-
-#[cfg(test)]
-mod tests;
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/set_block/sparse.rs b/crates/tantivy/src/columnar/column_index/optional_index/set_block/sparse.rs
deleted file mode 100644
index e656e6f1..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/set_block/sparse.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-use crate::columnar::column_index::optional_index::{SelectCursor, Set, SetCodec};
-
-pub struct SparseBlockCodec;
-
-impl SetCodec for SparseBlockCodec {
- type Item = u16;
- type Reader<'a> = SparseBlock<'a>;
-
- fn serialize(
- els: impl Iterator<Item = u16>,
- mut wrt: impl std::io::Write,
- ) -> std::io::Result<()> {
- for el in els {
- wrt.write_all(&el.to_le_bytes())?;
- }
- Ok(())
- }
-
- fn open(data: &[u8]) -> Self::Reader<'_> {
- SparseBlock(data)
- }
-}
-
-#[derive(Copy, Clone)]
-pub struct SparseBlock<'a>(&'a [u8]);
-
-impl<'a> SelectCursor<u16> for SparseBlock<'a> {
- #[inline]
- fn select(&mut self, rank: u16) -> u16 {
- <SparseBlock<'a> as Set<u16>>::select(self, rank)
- }
-}
-
-impl<'a> Set<u16> for SparseBlock<'a> {
- type SelectCursor<'b> = Self where Self: 'b;
-
- #[inline(always)]
- fn contains(&self, el: u16) -> bool {
- self.binary_search(el).is_ok()
- }
-
- #[inline(always)]
- fn rank_if_exists(&self, el: u16) -> Option<u16> {
- self.binary_search(el).ok()
- }
-
- #[inline(always)]
- fn rank(&self, el: u16) -> u16 {
- self.binary_search(el).unwrap_or_else(|el| el)
- }
-
- #[inline(always)]
- fn select(&self, rank: u16) -> u16 {
- let offset = rank as usize * 2;
- u16::from_le_bytes(self.0[offset..offset + 2].try_into().unwrap())
- }
-
- #[inline(always)]
- fn select_cursor(&self) -> Self::SelectCursor<'_> {
- *self
- }
-}
-
-#[inline(always)]
-fn get_u16(data: &[u8], byte_position: usize) -> u16 {
- let bytes: [u8; 2] = data[byte_position..byte_position + 2].try_into().unwrap();
- u16::from_le_bytes(bytes)
-}
-
-impl<'a> SparseBlock<'a> {
- #[inline(always)]
- fn value_at_idx(&self, data: &[u8], idx: u16) -> u16 {
- let start_offset: usize = idx as usize * 2;
- get_u16(data, start_offset)
- }
-
- #[inline]
- fn num_vals(&self) -> u16 {
- (self.0.len() / 2) as u16
- }
-
- #[inline]
- #[allow(clippy::comparison_chain)]
- // Looks for the element in the block. Returns the positions if found.
- fn binary_search(&self, target: u16) -> Result<u16, u16> {
- let data = &self.0;
- let mut size = self.num_vals();
- let mut left = 0;
- let mut right = size;
- // TODO try different implem.
- // e.g. exponential search into binary search
- while left < right {
- let mid = left + size / 2;
-
- // TODO do boundary check only once, and then use an
- // unsafe `value_at_idx`
- let mid_val = self.value_at_idx(data, mid);
-
- if target > mid_val {
- left = mid + 1;
- } else if target < mid_val {
- right = mid;
- } else {
- return Ok(mid);
- }
-
- size = right - left;
- }
- Err(left)
- }
-}
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/set_block/tests.rs b/crates/tantivy/src/columnar/column_index/optional_index/set_block/tests.rs
deleted file mode 100644
index f62d794a..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/set_block/tests.rs
+++ /dev/null
@@ -1,109 +0,0 @@
-use std::collections::HashMap;
-
-use crate::columnar::column_index::optional_index::set_block::dense::DENSE_BLOCK_NUM_BYTES;
-use crate::columnar::column_index::optional_index::set_block::{DenseBlockCodec, SparseBlockCodec};
-use crate::columnar::column_index::optional_index::{SelectCursor, Set, SetCodec};
-
-fn test_set_helper<C: SetCodec<Item = u16>>(vals: &[u16]) -> usize {
- let mut buffer = Vec::new();
- C::serialize(vals.iter().copied(), &mut buffer).unwrap();
- let tested_set = C::open(buffer.as_slice());
- let hash_set: HashMap<C::Item, C::Item> = vals
- .iter()
- .copied()
- .enumerate()
- .map(|(ord, val)| (val, C::Item::try_from(ord).ok().unwrap()))
- .collect();
- for val in 0u16..=u16::MAX {
- assert_eq!(tested_set.contains(val), hash_set.contains_key(&val));
- assert_eq!(tested_set.rank_if_exists(val), hash_set.get(&val).copied());
- assert_eq!(
- tested_set.rank(val),
- vals.iter().cloned().take_while(|v| *v < val).count() as u16
- );
- }
- for rank in 0..vals.len() {
- assert_eq!(tested_set.select(rank as u16), vals[rank]);
- }
- buffer.len()
-}
-
-#[test]
-fn test_dense_block_set_u16_empty() {
- let buffer_len = test_set_helper::<DenseBlockCodec>(&[]);
- assert_eq!(buffer_len, DENSE_BLOCK_NUM_BYTES as usize);
-}
-
-#[test]
-fn test_dense_block_set_u16_max() {
- let buffer_len = test_set_helper::<DenseBlockCodec>(&[u16::MAX]);
- assert_eq!(buffer_len, DENSE_BLOCK_NUM_BYTES as usize);
-}
-
-#[test]
-fn test_sparse_block_set_u16_empty() {
- let buffer_len = test_set_helper::<SparseBlockCodec>(&[]);
- assert_eq!(buffer_len, 0);
-}
-
-#[test]
-fn test_sparse_block_set_u16_max() {
- let buffer_len = test_set_helper::<SparseBlockCodec>(&[u16::MAX]);
- assert_eq!(buffer_len, 2);
-}
-
-use proptest::prelude::*;
-
-proptest! {
- #![proptest_config(ProptestConfig::with_cases(1))]
- #[test]
- fn test_prop_test_dense(els in proptest::collection::btree_set(0..=u16::MAX, 0..=u16::MAX as usize)) {
- let vals: Vec<u16> = els.into_iter().collect();
- let buffer_len = test_set_helper::<DenseBlockCodec>(&vals);
- assert_eq!(buffer_len, DENSE_BLOCK_NUM_BYTES as usize);
- }
-
- #[test]
- fn test_prop_test_sparse(els in proptest::collection::btree_set(0..=u16::MAX, 0..=u16::MAX as usize)) {
- let vals: Vec<u16> = els.into_iter().collect();
- let buffer_len = test_set_helper::<SparseBlockCodec>(&vals);
- assert_eq!(buffer_len, vals.len() * 2);
- }
-}
-
-#[test]
-fn test_simple_translate_codec_codec_idx_to_original_idx_dense() {
- let mut buffer = Vec::new();
- DenseBlockCodec::serialize([1, 3, 17, 32, 30_000, 30_001].iter().copied(), &mut buffer)
- .unwrap();
- let tested_set = DenseBlockCodec::open(buffer.as_slice());
- assert!(tested_set.contains(1));
- let mut select_cursor = tested_set.select_cursor();
- assert_eq!(select_cursor.select(0), 1);
- assert_eq!(select_cursor.select(1), 3);
- assert_eq!(select_cursor.select(2), 17);
-}
-
-#[test]
-fn test_simple_translate_codec_idx_to_original_idx_sparse() {
- let mut buffer = Vec::new();
- SparseBlockCodec::serialize([1, 3, 17].iter().copied(), &mut buffer).unwrap();
- let tested_set = SparseBlockCodec::open(buffer.as_slice());
- assert!(tested_set.contains(1));
- let mut select_cursor = tested_set.select_cursor();
- assert_eq!(SelectCursor::select(&mut select_cursor, 0), 1);
- assert_eq!(SelectCursor::select(&mut select_cursor, 1), 3);
- assert_eq!(SelectCursor::select(&mut select_cursor, 2), 17);
-}
-
-#[test]
-fn test_simple_translate_codec_idx_to_original_idx_dense() {
- let mut buffer = Vec::new();
- DenseBlockCodec::serialize(0u16..150u16, &mut buffer).unwrap();
- let tested_set = DenseBlockCodec::open(buffer.as_slice());
- assert!(tested_set.contains(1));
- let mut select_cursor = tested_set.select_cursor();
- for i in 0..150 {
- assert_eq!(i, select_cursor.select(i));
- }
-}
diff --git a/crates/tantivy/src/columnar/column_index/optional_index/tests.rs b/crates/tantivy/src/columnar/column_index/optional_index/tests.rs
deleted file mode 100644
index 65205f9d..00000000
--- a/crates/tantivy/src/columnar/column_index/optional_index/tests.rs
+++ /dev/null
@@ -1,223 +0,0 @@
-use proptest::prelude::*;
-use proptest::{prop_oneof, proptest};
-
-use super::*;
-use crate::columnar::{ColumnarReader, ColumnarWriter, DynamicColumnHandle};
-
-#[test]
-fn test_optional_index_bug_2293() {
- // tests for panic in docid_range_to_rowids for docid == num_docs
- test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK - 1);
- test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK);
- test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK + 1);
-}
-fn test_optional_index_with_num_docs(num_docs: u32) {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_numerical(100, "score", 80i64);
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer
- .serialize(num_docs, None, &mut buffer)
- .unwrap();
- let columnar = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar.num_columns(), 1);
- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("score").unwrap();
- assert_eq!(cols.len(), 1);
-
- let col = cols[0].open().unwrap();
- col.column_index().docid_range_to_rowids(0..num_docs);
-}
-
-#[test]
-fn test_dense_block_threshold() {
- assert_eq!(super::DENSE_BLOCK_THRESHOLD, 5_120);
-}
-
-fn random_bitvec() -> BoxedStrategy<Vec<bool>> {
- prop_oneof![
- 1 => prop::collection::vec(proptest::bool::weighted(1.0), 0..100),
- 1 => prop::collection::vec(proptest::bool::weighted(0.00), 0..(ELEMENTS_PER_BLOCK as usize * 3)), // empty blocks
- 1 => prop::collection::vec(proptest::bool::weighted(1.00), 0..(ELEMENTS_PER_BLOCK as usize + 10)), // full block
- 1 => prop::collection::vec(proptest::bool::weighted(0.01), 0..100),
- 1 => prop::collection::vec(proptest::bool::weighted(0.01), 0..u16::MAX as usize),
- 8 => vec![any::<bool>()],
- ]
- .boxed()
-}
-
-proptest! {
- #![proptest_config(ProptestConfig::with_cases(50))]
- #[test]
- fn test_with_random_bitvecs(bitvec1 in random_bitvec(), bitvec2 in random_bitvec(), bitvec3 in random_bitvec()) {
- let mut bitvec = Vec::new();
- bitvec.extend_from_slice(&bitvec1);
- bitvec.extend_from_slice(&bitvec2);
- bitvec.extend_from_slice(&bitvec3);
- test_null_index(&bitvec[..]);
- }
-}
-
-#[test]
-fn test_with_random_sets_simple() {
- let vals = 10..ELEMENTS_PER_BLOCK * 2;
- let mut out: Vec<u8> = Vec::new();
- serialize_optional_index(&vals, 100, &mut out).unwrap();
- let null_index = open_optional_index(OwnedBytes::new(out)).unwrap();
- let ranks: Vec<u32> = (65_472u32..65_473u32).collect();
- let els: Vec<u32> = ranks.iter().copied().map(|rank| rank + 10).collect();
- let mut select_cursor = null_index.select_cursor();
- for (rank, el) in ranks.iter().copied().zip(els.iter().copied()) {
- assert_eq!(select_cursor.select(rank), el);
- }
-}
-
-#[test]
-fn test_optional_index_trailing_empty_blocks() {
- test_null_index(&[false]);
-}
-
-#[test]
-fn test_optional_index_one_block_false() {
- let mut iter = vec![false; ELEMENTS_PER_BLOCK as usize];
- iter.push(true);
- test_null_index(&iter[..]);
-}
-
-#[test]
-fn test_optional_index_one_block_true() {
- let mut iter = vec![true; ELEMENTS_PER_BLOCK as usize];
- iter.push(true);
- test_null_index(&iter[..]);
-}
-
-impl<'a> Iterable<RowId> for &'a [bool] {
- fn boxed_iter(&self) -> Box<dyn Iterator<Item = RowId> + 'a> {
- Box::new(
- self.iter()
- .cloned()
- .enumerate()
- .filter(|(_pos, val)| *val)
- .map(|(pos, _val)| pos as u32),
- )
- }
-}
-
-fn test_null_index(data: &[bool]) {
- let mut out: Vec<u8> = Vec::new();
- serialize_optional_index(&data, data.len() as RowId, &mut out).unwrap();
- let null_index = open_optional_index(OwnedBytes::new(out)).unwrap();
- let orig_idx_with_value: Vec<u32> = data
- .iter()
- .enumerate()
- .filter(|(_pos, val)| **val)
- .map(|(pos, _val)| pos as u32)
- .collect();
- let mut select_iter = null_index.select_cursor();
- for i in 0..orig_idx_with_value.len() {
- assert_eq!(select_iter.select(i as u32), orig_idx_with_value[i]);
- }
-
- let step_size = (orig_idx_with_value.len() / 100).max(1);
- for (dense_idx, orig_idx) in orig_idx_with_value.iter().enumerate().step_by(step_size) {
- assert_eq!(null_index.rank_if_exists(*orig_idx), Some(dense_idx as u32));
- }
-
- // 100 samples
- let step_size = (data.len() / 100).max(1);
- for (pos, value) in data.iter().enumerate().step_by(step_size) {
- assert_eq!(null_index.contains(pos as u32), *value);
- }
-}
-
-#[test]
-fn test_optional_index_test_translation() {
- let optional_index = OptionalIndex::for_test(4, &[0, 2]);
- let mut select_cursor = optional_index.select_cursor();
- assert_eq!(select_cursor.select(0), 0);
- assert_eq!(select_cursor.select(1), 2);
-}
-
-#[test]
-fn test_optional_index_translate() {
- let optional_index = OptionalIndex::for_test(4, &[0, 2]);
- assert_eq!(optional_index.rank_if_exists(0), Some(0));
- assert_eq!(optional_index.rank_if_exists(2), Some(1));
-}
-
-#[test]
-fn test_optional_index_small() {
- let optional_index = OptionalIndex::for_test(4, &[0, 2]);
- assert!(optional_index.contains(0));
- assert!(!optional_index.contains(1));
- assert!(optional_index.contains(2));
- assert!(!optional_index.contains(3));
-}
-
-#[test]
-fn test_optional_index_large() {
- let row_ids = &[ELEMENTS_PER_BLOCK, ELEMENTS_PER_BLOCK + 1];
- let optional_index = OptionalIndex::for_test(ELEMENTS_PER_BLOCK + 2, row_ids);
- assert!(!optional_index.contains(0));
- assert!(!optional_index.contains(100));
- assert!(!optional_index.contains(ELEMENTS_PER_BLOCK - 1));
- assert!(optional_index.contains(ELEMENTS_PER_BLOCK));
- assert!(optional_index.contains(ELEMENTS_PER_BLOCK + 1));
-}
-
-fn test_optional_index_iter_aux(row_ids: &[RowId], num_rows: RowId) {
- let optional_index = OptionalIndex::for_test(num_rows, row_ids);
- assert_eq!(optional_index.num_docs(), num_rows);
- assert!(optional_index.iter_rows().eq(row_ids.iter().copied()));
-}
-
-#[test]
-fn test_optional_index_iter_empty() {
- test_optional_index_iter_aux(&[], 0u32);
-}
-
-fn test_optional_index_rank_aux(row_ids: &[RowId]) {
- let num_rows = row_ids.last().copied().unwrap_or(0u32) + 1;
- let null_index = OptionalIndex::for_test(num_rows, row_ids);
- assert_eq!(null_index.num_docs(), num_rows);
- for (row_id, row_val) in row_ids.iter().copied().enumerate() {
- assert_eq!(null_index.rank(row_val), row_id as u32);
- assert_eq!(null_index.rank_if_exists(row_val), Some(row_id as u32));
- if row_val > 0 && !null_index.contains(&row_val - 1) {
- assert_eq!(null_index.rank(row_val - 1), row_id as u32);
- }
- assert_eq!(null_index.rank(row_val + 1), row_id as u32 + 1);
- }
-}
-
-#[test]
-fn test_optional_index_rank() {
- test_optional_index_rank_aux(&[1u32]);
- test_optional_index_rank_aux(&[0u32, 1u32]);
- let mut block = Vec::new();
- block.push(3u32);
- block.extend((0..ELEMENTS_PER_BLOCK).map(|i| i + ELEMENTS_PER_BLOCK + 1));
- test_optional_index_rank_aux(&block);
-}
-
-#[test]
-fn test_optional_index_iter_empty_one() {
- test_optional_index_iter_aux(&[1], 2u32);
- test_optional_index_iter_aux(&[100_000], 200_000u32);
-}
-
-#[test]
-fn test_optional_index_iter_dense_block() {
- let mut block = Vec::new();
- block.push(3u32);
- block.extend((0..ELEMENTS_PER_BLOCK).map(|i| i + ELEMENTS_PER_BLOCK + 1));
- test_optional_index_iter_aux(&block, 3 * ELEMENTS_PER_BLOCK);
-}
-
-#[test]
-fn test_optional_index_for_tests() {
- let optional_index = OptionalIndex::for_test(4, &[1, 2]);
- assert!(!optional_index.contains(0));
- assert!(optional_index.contains(1));
- assert!(optional_index.contains(2));
- assert!(!optional_index.contains(3));
- assert_eq!(optional_index.num_docs(), 4);
-}