Skip to content

web search done right

fix clippy warnings

Mikkel Denker43065867632a
diff --git a/crates/tantivy/src/bitpacker/blocked_bitpacker.rs b/crates/tantivy/src/bitpacker/blocked_bitpacker.rs
deleted file mode 100644
index d28ef9e0..00000000
--- a/crates/tantivy/src/bitpacker/blocked_bitpacker.rs
+++ /dev/null
@@ -1,180 +0,0 @@
-use super::bitpacker::BitPacker;
-use super::compute_num_bits;
-use super::{minmax, BitUnpacker};
-
-const BLOCK_SIZE: usize = 128;
-
-/// `BlockedBitpacker` compresses data in blocks of
-/// 128 elements, while keeping an index on it
-#[derive(Debug, Clone)]
-pub struct BlockedBitpacker {
- // bitpacked blocks
- compressed_blocks: Vec<u8>,
- // uncompressed data, collected until BLOCK_SIZE
- buffer: Vec<u64>,
- offset_and_bits: Vec<BlockedBitpackerEntryMetaData>,
-}
-impl Default for BlockedBitpacker {
- fn default() -> Self {
- BlockedBitpacker::new()
- }
-}
-
-/// `BlockedBitpackerEntryMetaData` encodes the
-/// offset and bit_width into a u64 bit field
-///
-/// This saves some space, since 7byte is more
-/// than enough and also keeps the access fast
-/// because of alignment
-#[derive(Debug, Clone, Default)]
-struct BlockedBitpackerEntryMetaData {
- encoded: u64,
- base_value: u64,
-}
-
-impl BlockedBitpackerEntryMetaData {
- fn new(offset: u64, num_bits: u8, base_value: u64) -> Self {
- let encoded = offset | (num_bits as u64) << (64 - 8);
- Self {
- encoded,
- base_value,
- }
- }
- fn offset(&self) -> u64 {
- (self.encoded << 8) >> 8
- }
- fn num_bits(&self) -> u8 {
- (self.encoded >> 56) as u8
- }
- fn base_value(&self) -> u64 {
- self.base_value
- }
-}
-
-#[test]
-fn metadata_test() {
- let meta = BlockedBitpackerEntryMetaData::new(50000, 6, 40000);
- assert_eq!(meta.offset(), 50000);
- assert_eq!(meta.num_bits(), 6);
-}
-
-fn mem_usage<T>(items: &Vec<T>) -> usize {
- items.capacity() * std::mem::size_of::<T>()
-}
-
-impl BlockedBitpacker {
- pub fn new() -> Self {
- Self {
- compressed_blocks: vec![0; 8],
- buffer: vec![],
- offset_and_bits: vec![],
- }
- }
-
- /// The memory used (inclusive childs)
- pub fn mem_usage(&self) -> usize {
- std::mem::size_of::<BlockedBitpacker>()
- + self.compressed_blocks.capacity()
- + mem_usage(&self.offset_and_bits)
- + mem_usage(&self.buffer)
- }
-
- #[inline]
- pub fn add(&mut self, val: u64) {
- self.buffer.push(val);
- if self.buffer.len() == BLOCK_SIZE {
- self.flush();
- }
- }
-
- pub fn flush(&mut self) {
- if let Some((min_value, max_value)) = minmax(self.buffer.iter()) {
- let mut bit_packer = BitPacker::new();
- let num_bits_block = compute_num_bits(*max_value - min_value);
- // todo performance: the padding handling could be done better, e.g. use a slice and
- // return num_bytes written from bitpacker
- self.compressed_blocks
- .resize(self.compressed_blocks.len() - 8, 0); // remove padding for bitpacker
- let offset = self.compressed_blocks.len() as u64;
- // todo performance: for some bit_width we
- // can encode multiple vals into the
- // mini_buffer before checking to flush
- // (to be done in BitPacker)
- for val in self.buffer.iter() {
- bit_packer
- .write(
- *val - min_value,
- num_bits_block,
- &mut self.compressed_blocks,
- )
- .expect("cannot write bitpacking to output"); // write to in memory can't fail
- }
- bit_packer.flush(&mut self.compressed_blocks).unwrap();
- self.offset_and_bits
- .push(BlockedBitpackerEntryMetaData::new(
- offset,
- num_bits_block,
- *min_value,
- ));
-
- self.buffer.clear();
- self.compressed_blocks
- .resize(self.compressed_blocks.len() + 8, 0); // add padding for bitpacker
- }
- }
- #[inline]
- pub fn get(&self, idx: usize) -> u64 {
- let metadata_pos = idx / BLOCK_SIZE;
- let pos_in_block = idx % BLOCK_SIZE;
- if let Some(metadata) = self.offset_and_bits.get(metadata_pos) {
- let unpacked = BitUnpacker::new(metadata.num_bits()).get(
- pos_in_block as u32,
- &self.compressed_blocks[metadata.offset() as usize..],
- );
- unpacked + metadata.base_value()
- } else {
- self.buffer[pos_in_block]
- }
- }
-
- pub fn iter(&self) -> impl Iterator<Item = u64> + '_ {
- // todo performance: we could decompress a whole block and cache it instead
- let bitpacked_elems = self.offset_and_bits.len() * BLOCK_SIZE;
- let iter = (0..bitpacked_elems)
- .map(move |idx| self.get(idx))
- .chain(self.buffer.iter().cloned());
- iter
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- #[test]
- fn blocked_bitpacker_empty() {
- let blocked_bitpacker = BlockedBitpacker::new();
- assert_eq!(
- blocked_bitpacker.iter().collect::<Vec<u64>>(),
- Vec::<u64>::new()
- );
- }
- #[test]
- fn blocked_bitpacker_one() {
- let mut blocked_bitpacker = BlockedBitpacker::new();
- blocked_bitpacker.add(50000);
- assert_eq!(blocked_bitpacker.get(0), 50000);
- assert_eq!(blocked_bitpacker.iter().collect::<Vec<u64>>(), vec![50000]);
- }
- #[test]
- fn blocked_bitpacker_test() {
- let mut blocked_bitpacker = BlockedBitpacker::new();
- for val in 0..21500 {
- blocked_bitpacker.add(val);
- }
- for val in 0..21500 {
- assert_eq!(blocked_bitpacker.get(val as usize), val);
- }
- assert_eq!(blocked_bitpacker.iter().count(), 21500);
- assert_eq!(blocked_bitpacker.iter().last().unwrap(), 21499);
- }
-}
diff --git a/crates/tantivy/src/bitpacker/mod.rs b/crates/tantivy/src/bitpacker/mod.rs
index ef91ade0..7a1d6319 100644
--- a/crates/tantivy/src/bitpacker/mod.rs
+++ b/crates/tantivy/src/bitpacker/mod.rs
@@ -1,5 +1,4 @@
mod bitpacker;
-mod blocked_bitpacker;
mod filter_vec;
use std::cmp::Ordering;
diff --git a/crates/tantivy/src/stacker/arena_hashmap.rs b/crates/tantivy/src/stacker/arena_hashmap.rs
index d2b938ec..954790db 100644
--- a/crates/tantivy/src/stacker/arena_hashmap.rs
+++ b/crates/tantivy/src/stacker/arena_hashmap.rs
@@ -45,11 +45,6 @@ impl ArenaHashMap {
self.shared_arena_hashmap.mem_usage() + self.memory_arena.mem_usage()
}
- #[inline]
- pub fn is_empty(&self) -> bool {
- self.shared_arena_hashmap.is_empty()
- }
-
#[inline]
pub fn len(&self) -> usize {
self.shared_arena_hashmap.len()
diff --git a/crates/tantivy/src/stacker/memory_arena.rs b/crates/tantivy/src/stacker/memory_arena.rs
index 95f669bf..10ede89a 100644
--- a/crates/tantivy/src/stacker/memory_arena.rs
+++ b/crates/tantivy/src/stacker/memory_arena.rs
@@ -113,15 +113,6 @@ impl MemoryArena {
self.pages.len() * PAGE_SIZE
}
- /// Returns the number of bytes allocated in the arena.
- pub fn len(&self) -> usize {
- self.pages.len().saturating_sub(1) * PAGE_SIZE + self.pages.last().unwrap().len
- }
-
- pub fn is_empty(&self) -> bool {
- self.len() == 0
- }
-
#[inline]
pub fn write_at<Item: Copy + 'static>(&mut self, addr: Addr, val: Item) {
let dest = self.slice_mut(addr, std::mem::size_of::<Item>());
@@ -157,11 +148,6 @@ impl MemoryArena {
self.get_page(addr.page_id())
.slice_from(addr.page_local_addr())
}
- #[inline]
- pub fn slice_from_mut(&mut self, addr: Addr) -> &mut [u8] {
- self.get_page_mut(addr.page_id())
- .slice_from_mut(addr.page_local_addr())
- }
#[inline]
pub fn slice_mut(&mut self, addr: Addr, len: usize) -> &mut [u8] {
@@ -225,10 +211,6 @@ impl Page {
fn slice_from(&self, local_addr: usize) -> &[u8] {
&self.data[local_addr..]
}
- #[inline]
- fn slice_from_mut(&mut self, local_addr: usize) -> &mut [u8] {
- &mut self.data[local_addr..]
- }
#[inline]
fn slice_mut(&mut self, local_addr: usize, len: usize) -> &mut [u8] {
diff --git a/crates/tantivy/src/stacker/shared_arena_hashmap.rs b/crates/tantivy/src/stacker/shared_arena_hashmap.rs
index 0ec4a6a6..dff09cda 100644
--- a/crates/tantivy/src/stacker/shared_arena_hashmap.rs
+++ b/crates/tantivy/src/stacker/shared_arena_hashmap.rs
@@ -187,11 +187,6 @@ impl SharedArenaHashMap {
};
}
- #[inline]
- pub fn is_empty(&self) -> bool {
- self.len() == 0
- }
-
#[inline]
pub fn len(&self) -> usize {
self.len