web search done right
fix clippy warnings
Mikkel Denker43065867632a
diff --git a/crates/tantivy/src/bitpacker/blocked_bitpacker.rs b/crates/tantivy/src/bitpacker/blocked_bitpacker.rsdeleted file mode 100644index d28ef9e0..00000000--- a/crates/tantivy/src/bitpacker/blocked_bitpacker.rs+++ /dev/null@@ -1,180 +0,0 @@-use super::bitpacker::BitPacker;-use super::compute_num_bits;-use super::{minmax, BitUnpacker};--const BLOCK_SIZE: usize = 128;--/// `BlockedBitpacker` compresses data in blocks of-/// 128 elements, while keeping an index on it-#[derive(Debug, Clone)]-pub struct BlockedBitpacker {- // bitpacked blocks- compressed_blocks: Vec<u8>,- // uncompressed data, collected until BLOCK_SIZE- buffer: Vec<u64>,- offset_and_bits: Vec<BlockedBitpackerEntryMetaData>,-}-impl Default for BlockedBitpacker {- fn default() -> Self {- BlockedBitpacker::new()- }-}--/// `BlockedBitpackerEntryMetaData` encodes the-/// offset and bit_width into a u64 bit field-///-/// This saves some space, since 7byte is more-/// than enough and also keeps the access fast-/// because of alignment-#[derive(Debug, Clone, Default)]-struct BlockedBitpackerEntryMetaData {- encoded: u64,- base_value: u64,-}--impl BlockedBitpackerEntryMetaData {- fn new(offset: u64, num_bits: u8, base_value: u64) -> Self {- let encoded = offset | (num_bits as u64) << (64 - 8);- Self {- encoded,- base_value,- }- }- fn offset(&self) -> u64 {- (self.encoded << 8) >> 8- }- fn num_bits(&self) -> u8 {- (self.encoded >> 56) as u8- }- fn base_value(&self) -> u64 {- self.base_value- }-}--#[test]-fn metadata_test() {- let meta = BlockedBitpackerEntryMetaData::new(50000, 6, 40000);- assert_eq!(meta.offset(), 50000);- assert_eq!(meta.num_bits(), 6);-}--fn mem_usage<T>(items: &Vec<T>) -> usize {- items.capacity() * std::mem::size_of::<T>()-}--impl BlockedBitpacker {- pub fn new() -> Self {- Self {- compressed_blocks: vec![0; 8],- buffer: vec![],- offset_and_bits: vec![],- }- }-- /// The memory used (inclusive childs)- pub fn mem_usage(&self) -> usize {- std::mem::size_of::<BlockedBitpacker>()- + self.compressed_blocks.capacity()- + mem_usage(&self.offset_and_bits)- + mem_usage(&self.buffer)- }-- #[inline]- pub fn add(&mut self, val: u64) {- self.buffer.push(val);- if self.buffer.len() == BLOCK_SIZE {- self.flush();- }- }-- pub fn flush(&mut self) {- if let Some((min_value, max_value)) = minmax(self.buffer.iter()) {- let mut bit_packer = BitPacker::new();- let num_bits_block = compute_num_bits(*max_value - min_value);- // todo performance: the padding handling could be done better, e.g. use a slice and- // return num_bytes written from bitpacker- self.compressed_blocks- .resize(self.compressed_blocks.len() - 8, 0); // remove padding for bitpacker- let offset = self.compressed_blocks.len() as u64;- // todo performance: for some bit_width we- // can encode multiple vals into the- // mini_buffer before checking to flush- // (to be done in BitPacker)- for val in self.buffer.iter() {- bit_packer- .write(- *val - min_value,- num_bits_block,- &mut self.compressed_blocks,- )- .expect("cannot write bitpacking to output"); // write to in memory can't fail- }- bit_packer.flush(&mut self.compressed_blocks).unwrap();- self.offset_and_bits- .push(BlockedBitpackerEntryMetaData::new(- offset,- num_bits_block,- *min_value,- ));-- self.buffer.clear();- self.compressed_blocks- .resize(self.compressed_blocks.len() + 8, 0); // add padding for bitpacker- }- }- #[inline]- pub fn get(&self, idx: usize) -> u64 {- let metadata_pos = idx / BLOCK_SIZE;- let pos_in_block = idx % BLOCK_SIZE;- if let Some(metadata) = self.offset_and_bits.get(metadata_pos) {- let unpacked = BitUnpacker::new(metadata.num_bits()).get(- pos_in_block as u32,- &self.compressed_blocks[metadata.offset() as usize..],- );- unpacked + metadata.base_value()- } else {- self.buffer[pos_in_block]- }- }-- pub fn iter(&self) -> impl Iterator<Item = u64> + '_ {- // todo performance: we could decompress a whole block and cache it instead- let bitpacked_elems = self.offset_and_bits.len() * BLOCK_SIZE;- let iter = (0..bitpacked_elems)- .map(move |idx| self.get(idx))- .chain(self.buffer.iter().cloned());- iter- }-}--#[cfg(test)]-mod tests {- use super::*;- #[test]- fn blocked_bitpacker_empty() {- let blocked_bitpacker = BlockedBitpacker::new();- assert_eq!(- blocked_bitpacker.iter().collect::<Vec<u64>>(),- Vec::<u64>::new()- );- }- #[test]- fn blocked_bitpacker_one() {- let mut blocked_bitpacker = BlockedBitpacker::new();- blocked_bitpacker.add(50000);- assert_eq!(blocked_bitpacker.get(0), 50000);- assert_eq!(blocked_bitpacker.iter().collect::<Vec<u64>>(), vec![50000]);- }- #[test]- fn blocked_bitpacker_test() {- let mut blocked_bitpacker = BlockedBitpacker::new();- for val in 0..21500 {- blocked_bitpacker.add(val);- }- for val in 0..21500 {- assert_eq!(blocked_bitpacker.get(val as usize), val);- }- assert_eq!(blocked_bitpacker.iter().count(), 21500);- assert_eq!(blocked_bitpacker.iter().last().unwrap(), 21499);- }-}diff --git a/crates/tantivy/src/bitpacker/mod.rs b/crates/tantivy/src/bitpacker/mod.rsindex ef91ade0..7a1d6319 100644--- a/crates/tantivy/src/bitpacker/mod.rs+++ b/crates/tantivy/src/bitpacker/mod.rs@@ -1,5 +1,4 @@mod bitpacker;-mod blocked_bitpacker;mod filter_vec;use std::cmp::Ordering;diff --git a/crates/tantivy/src/stacker/arena_hashmap.rs b/crates/tantivy/src/stacker/arena_hashmap.rsindex d2b938ec..954790db 100644--- a/crates/tantivy/src/stacker/arena_hashmap.rs+++ b/crates/tantivy/src/stacker/arena_hashmap.rs@@ -45,11 +45,6 @@ impl ArenaHashMap {self.shared_arena_hashmap.mem_usage() + self.memory_arena.mem_usage()}- #[inline]- pub fn is_empty(&self) -> bool {- self.shared_arena_hashmap.is_empty()- }-#[inline]pub fn len(&self) -> usize {self.shared_arena_hashmap.len()diff --git a/crates/tantivy/src/stacker/memory_arena.rs b/crates/tantivy/src/stacker/memory_arena.rsindex 95f669bf..10ede89a 100644--- a/crates/tantivy/src/stacker/memory_arena.rs+++ b/crates/tantivy/src/stacker/memory_arena.rs@@ -113,15 +113,6 @@ impl MemoryArena {self.pages.len() * PAGE_SIZE}- /// Returns the number of bytes allocated in the arena.- pub fn len(&self) -> usize {- self.pages.len().saturating_sub(1) * PAGE_SIZE + self.pages.last().unwrap().len- }-- pub fn is_empty(&self) -> bool {- self.len() == 0- }-#[inline]pub fn write_at<Item: Copy + 'static>(&mut self, addr: Addr, val: Item) {let dest = self.slice_mut(addr, std::mem::size_of::<Item>());@@ -157,11 +148,6 @@ impl MemoryArena {self.get_page(addr.page_id()).slice_from(addr.page_local_addr())}- #[inline]- pub fn slice_from_mut(&mut self, addr: Addr) -> &mut [u8] {- self.get_page_mut(addr.page_id())- .slice_from_mut(addr.page_local_addr())- }#[inline]pub fn slice_mut(&mut self, addr: Addr, len: usize) -> &mut [u8] {@@ -225,10 +211,6 @@ impl Page {fn slice_from(&self, local_addr: usize) -> &[u8] {&self.data[local_addr..]}- #[inline]- fn slice_from_mut(&mut self, local_addr: usize) -> &mut [u8] {- &mut self.data[local_addr..]- }#[inline]fn slice_mut(&mut self, local_addr: usize, len: usize) -> &mut [u8] {diff --git a/crates/tantivy/src/stacker/shared_arena_hashmap.rs b/crates/tantivy/src/stacker/shared_arena_hashmap.rsindex 0ec4a6a6..dff09cda 100644--- a/crates/tantivy/src/stacker/shared_arena_hashmap.rs+++ b/crates/tantivy/src/stacker/shared_arena_hashmap.rs@@ -187,11 +187,6 @@ impl SharedArenaHashMap {};}- #[inline]- pub fn is_empty(&self) -> bool {- self.len() == 0- }-#[inline]pub fn len(&self) -> usize {self.len