web search done right
remove optional and multivalued columns from tantivy as we only use full columnar indices
Mikkel Denkerba14aaab68f4
diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rsindex 1e1019dc..0af963d3 100644--- a/crates/core/examples/indexer.rs+++ b/crates/core/examples/indexer.rs@@ -40,7 +40,6 @@ fn main() -> anyhow::Result<()> {.to_string(),}),host_centrality_threshold: None,- topics_path: None,host_centrality_store_path: data_path.join("centrality/").to_str().unwrap().to_string(),page_centrality_store_path: Some(data_pathdiff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex f1ba4798..83036daf 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -33,7 +33,6 @@ pub struct IndexerConfig {pub warc_source: WarcSource,pub page_webgraph: Option<IndexerGraphConfig>,pub host_centrality_threshold: Option<f64>,- pub topics_path: Option<String>,pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,pub safety_classifier_path: Option<String>,diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 0594da26..a0e408f9 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -23,8 +23,8 @@ use crate::config::{defaults, IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LocalConfig,WebSpellConfig,};+use crate::entrypoint::indexer;use crate::entrypoint::indexer::JobSettings;-use crate::entrypoint::{dmoz_parser, indexer};use crate::Result;use std::fs::{self};use std::path::Path;@@ -44,7 +44,6 @@ fn download_files() {"queries_us.csv","sample.warc.gz","bangs.json",- "content.rdf.u8.gz","english-wordnet-2022-subset.ttl","lambdamart.txt","test.zim",@@ -188,13 +187,6 @@ fn create_inverted_index() -> Result<()> {page_webgraph: Some(IndexerGraphConfig::Local {path: webgraph_path.to_str().unwrap().to_string(),}),- topics_path: Some(- Path::new(DATA_PATH)- .join("human_annotations")- .to_str()- .unwrap()- .to_string(),- ),output_path: out_path.to_str().unwrap().to_string(),limit_warc_files: None,skip_warc_files: None,@@ -232,17 +224,9 @@ fn create_entity_index() -> Result<()> {Ok(())}-fn parse_topics() -> Result<()> {- let dmoz_path = Path::new(DATA_PATH).join("content.rdf.u8.gz");-- let topics = dmoz_parser::parse(dmoz_path)?;- topics.save(Path::new(DATA_PATH).join("human_annotations"))-}-fn index_files() -> Result<()> {create_webgraph()?;calculate_centrality();- parse_topics()?;create_inverted_index()?;create_entity_index()?;build_spellchecker()?;diff --git a/crates/core/src/entrypoint/dmoz_parser.rs b/crates/core/src/entrypoint/dmoz_parser.rsdeleted file mode 100644index e54a35e4..00000000--- a/crates/core/src/entrypoint/dmoz_parser.rs+++ /dev/null@@ -1,163 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use flate2::read::MultiGzDecoder;-use quick_xml::events::Event;-use url::Url;--use crate::{- human_website_annotations::{Info, Topic},- Result,-};-use std::{- collections::HashMap,- fs::File,- io::{BufRead, BufReader},- path::Path,-};--use crate::human_website_annotations;--struct PageIterator<R: BufRead> {- reader: quick_xml::Reader<R>,- buf: Vec<u8>,-}--impl<R: BufRead> From<R> for PageIterator<R> {- fn from(reader: R) -> Self {- Self {- reader: quick_xml::Reader::from_reader(reader),- buf: Vec::new(),- }- }-}--impl<R: BufRead> Iterator for PageIterator<R> {- type Item = Page;-- fn next(&mut self) -> Option<Self::Item> {- let mut current_page = None;- let mut inside_topic = false;- let mut inside_desc = false;-- while let Ok(event) = self.reader.read_event_into(&mut self.buf) {- match event {- Event::Start(ref e) if e.name().as_ref() == b"ExternalPage" => {- let url_attr = e- .attributes()- .filter(std::result::Result::is_ok)- .find(|attr| attr.as_ref().unwrap().key.as_ref() == b"about")- .unwrap()- .unwrap();- let url = url_attr.decode_and_unescape_value(&self.reader).unwrap();- current_page = Some(Page::new(url.into_owned()));- inside_desc = false;- inside_topic = false;- }- Event::End(ref e) if e.name().as_ref() == b"ExternalPage" => break,- Event::Start(ref e) if e.name().as_ref() == b"topic" && current_page.is_some() => {- inside_topic = true;- }- Event::End(ref e) if e.name().as_ref() == b"topic" && current_page.is_some() => {- inside_topic = false;- }- Event::Start(ref e)- if e.name().as_ref() == b"d:Description" && current_page.is_some() =>- {- inside_desc = true;- }- Event::End(ref e)- if e.name().as_ref() == b"d:Description" && current_page.is_some() =>- {- inside_desc = false;- }- Event::Text(ref e) => {- if inside_topic {- if let Some(page) = &mut current_page {- let topic = e.unescape().unwrap();- page.topic.push_str(topic.as_ref());- }- } else if inside_desc {- if let Some(page) = &mut current_page {- let desc = e.unescape().unwrap();- page.description.push_str(desc.as_ref());- }- }- }- Event::Eof => break,- _ => (),- }- }-- current_page- }-}--#[derive(Debug, Default)]-struct Page {- url: String,- description: String,- topic: String,-}--impl Page {- fn new(url: String) -> Self {- Self {- url,- ..Default::default()- }- }-}--pub fn parse<P: AsRef<Path>>(dmoz_file: P) -> Result<human_website_annotations::Mapper> {- let file = File::open(dmoz_file)?;- let reader = BufReader::new(file);- let reader = BufReader::new(MultiGzDecoder::new(reader));- let mut map = HashMap::new();-- for page in PageIterator::from(reader) {- let url = match Url::parse(&page.url) {- Ok(url) => url,- Err(_) => continue,- };-- if url.path() != "/" {- continue;- }-- if page.topic.contains("World") {- continue;- }-- let topic = Topic::from_string("/".to_string() + page.topic.as_str());-- let info = Info {- description: page.description,- topic,- };-- map.insert(url.host_str().unwrap().to_string(), info);- }-- Ok(map.into())-}--pub fn run<P: AsRef<Path>>(dmoz_file: P, output_path: P) -> Result<()> {- let mapper = parse(dmoz_file)?;- mapper.save(output_path)?;-- Ok(())-}diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 67df1e4d..60a018bd 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -31,7 +31,6 @@ use crate::models::dual_encoder::DualEncoder as DualEncoderModel;use crate::webgraph::remote::RemoteWebgraph;use crate::Result;-use crate::human_website_annotations;use crate::index::Index;use crate::rake::RakeModel;use crate::ranking::SignalComputer;@@ -42,7 +41,6 @@ pub struct Config {pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,pub page_webgraph: Option<IndexerGraphConfig>,- pub topics_path: Option<String>,pub safety_classifier_path: Option<String>,pub dual_encoder: Option<IndexerDualEncoderConfig>,}@@ -53,7 +51,6 @@ impl From<IndexerConfig> for Config {host_centrality_store_path: config.host_centrality_store_path,page_centrality_store_path: config.page_centrality_store_path,page_webgraph: config.page_webgraph,- topics_path: config.topics_path,safety_classifier_path: config.safety_classifier_path,dual_encoder: config.dual_encoder,}@@ -66,7 +63,6 @@ impl From<LiveIndexConfig> for Config {host_centrality_store_path: config.host_centrality_store_path,page_centrality_store_path: config.page_centrality_store_path,page_webgraph: config.page_webgraph,- topics_path: None,safety_classifier_path: config.safety_classifier_path,dual_encoder: None,}@@ -166,7 +162,6 @@ pub struct IndexingWorker {page_centrality_store: Option<speedy_kv::Db<NodeID, f64>>,page_centrality_rank_store: Option<speedy_kv::Db<NodeID, u64>>,page_webgraph: Option<Webgraph>,- topics: Option<human_website_annotations::Mapper>,safety_classifier: Option<safety_classifier::Model>,job_settings: Option<JobSettings>,rake: RakeModel,@@ -198,10 +193,6 @@ impl IndexingWorker {speedy_kv::Db::open_or_create(Path::new(&p).join("harmonic_rank")).unwrap()}),page_webgraph: config.page_webgraph.as_ref().map(Webgraph::new),- topics: config- .topics_path- .as_ref()- .map(|path| human_website_annotations::Mapper::open(path).unwrap()),safety_classifier: config.safety_classifier_path.as_ref()@@ -355,16 +346,6 @@ impl IndexingWorker {}}- fn set_dmoz_description(&self, page: &mut Webpage) {- if let Some(mapper) = self.topics.as_ref() {- if let Some(info) =- mapper.get(&page.html.url().host_str().unwrap_or_default().to_string())- {- page.dmoz_description = Some(info.description.clone())- }- }- }-fn set_keywords(&self, page: &mut Webpage) {page.keywords = page.html.keywords(&self.rake);}@@ -468,7 +449,6 @@ impl IndexingWorker {}self.set_page_centralities(&mut prepared);- self.set_dmoz_description(&mut prepared);self.set_keywords(&mut prepared);self.set_safety_classification(&mut prepared);@@ -516,7 +496,6 @@ mod tests {host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),page_centrality_store_path: None,page_webgraph: None,- topics_path: None,safety_classifier_path: None,dual_encoder: Some(IndexerDualEncoderConfig {model_path: data_path.to_str().unwrap().to_string(),diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rsindex e2f6e73a..59969878 100644--- a/crates/core/src/entrypoint/mod.rs+++ b/crates/core/src/entrypoint/mod.rs@@ -23,7 +23,6 @@ mod centrality;#[cfg(feature = "dev")]pub mod configure;pub mod crawler;-pub mod dmoz_parser;mod entity;pub mod entity_search_server;pub mod feed_indexer;diff --git a/crates/core/src/human_website_annotations.rs b/crates/core/src/human_website_annotations.rsdeleted file mode 100644index 416af3ab..00000000--- a/crates/core/src/human_website_annotations.rs+++ /dev/null@@ -1,151 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use itertools::Itertools;--use tantivy::schema::Facet;--use crate::Result;-use std::{- collections::{HashMap, HashSet},- fs::File,- io::{BufReader, Write},- path::Path,-};--#[derive(Debug, Hash, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)]-pub struct Topic<const N: usize = 2> {- detailed_topics: Vec<String>,-}--/// bincode seems to not like default const generics, therefore manual impl-impl<const N: usize> bincode::Encode for Topic<N> {- fn encode<E: bincode::enc::Encoder>(- &self,- encoder: &mut E,- ) -> Result<(), bincode::error::EncodeError> {- self.detailed_topics.encode(encoder)- }-}--impl<const N: usize> bincode::Decode for Topic<N> {- fn decode<D: bincode::de::Decoder>(- decoder: &mut D,- ) -> Result<Self, bincode::error::DecodeError> {- Ok(Self {- detailed_topics: Vec::decode(decoder)?,- })- }-}--impl<'de, const N: usize> bincode::BorrowDecode<'de> for Topic<N> {- fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(- decoder: &mut D,- ) -> Result<Self, bincode::error::DecodeError> {- Ok(Self {- detailed_topics: Vec::borrow_decode(decoder)?,- })- }-}--impl<const N: usize> From<Facet> for Topic<N> {- fn from(facet: Facet) -> Self {- Self {- detailed_topics: facet- .to_path()- .into_iter()- .take(N)- .map(String::from)- .collect(),- }- }-}--impl<const N: usize> Topic<N> {- pub fn from_string(s: String) -> Self {- Self {- detailed_topics: s- .split('/')- .skip(2)- .take(N)- .map(String::from)- .collect::<Vec<_>>(),- }- }-}--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct Info {- pub description: String,- pub topic: Topic,-}--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct Mapper(HashMap<String, Info>);--impl From<HashMap<String, Info>> for Mapper {- fn from(map: HashMap<String, Info>) -> Self {- Self(map)- }-}--impl Mapper {- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {- let mut file = File::options()- .create(true)- .truncate(true)- .write(true)- .open(path)?;-- let bytes = bincode::encode_to_vec(self, bincode::config::standard())?;- file.write_all(&bytes)?;-- Ok(())- }-- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut reader = BufReader::new(File::open(path)?);-- Ok(bincode::decode_from_std_read(- &mut reader,- bincode::config::standard(),- )?)- }-- pub fn get(&self, host: &String) -> Option<&Info> {- self.0.get(host)- }-- pub fn all_topics(&self) -> HashSet<Topic> {- self.0.values().map(|info| info.topic.clone()).collect()- }-- pub fn top_topics(&self, top_n: usize) -> Vec<Topic> {- let mut topics: HashMap<Topic, usize> = HashMap::new();-- for topic in self.0.values().map(|info| info.topic.clone()) {- *topics.entry(topic).or_default() += 1;- }-- topics- .into_iter()- .sorted_by_key(|(_, count)| *count)- .rev()- .map(|(topic, _)| topic)- .take(top_n)- .collect()- }-}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 5b349225..18162b30 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -57,7 +57,6 @@ mod external_sort;mod fastfield_reader;pub mod feed;mod highlighted;-mod human_website_annotations;pub mod hyperloglog;pub mod image_store;mod improvement;diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 1341cab5..496f4e36 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -59,13 +59,6 @@ enum Commands {mode: CentralityMode,},- /// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories.- /// It can be used to calculate the topic centrality for websites or augments website descriptions during indexing.- DmozParser {- dmoz_file: String,- output_path: String,- },-/// Webgraph specific commands.Webgraph {#[clap(subcommand)]@@ -394,10 +387,6 @@ fn main() -> Result<()> {Commands::Configure { skip_download } => {configure::run(skip_download)?;}- Commands::DmozParser {- dmoz_file,- output_path,- } => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),Commands::Crawler { options } => match options {Crawler::Worker { config_path } => {let config: config::CrawlerConfig = load_toml_config(config_path);diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 1f28b9ad..0fc1d7d1 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -748,7 +748,6 @@ mod tests {host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),page_centrality_store_path: None,page_webgraph: None,- topics_path: None,safety_classifier_path: None,dual_encoder: Some(IndexerDualEncoderConfig {model_path: data_path.to_str().unwrap().to_string(),diff --git a/crates/tantivy/src/collector/facet_collector.rs b/crates/tantivy/src/collector/facet_collector.rsdeleted file mode 100644index 0c706faa..00000000--- a/crates/tantivy/src/collector/facet_collector.rs+++ /dev/null@@ -1,827 +0,0 @@-use std::cmp::Ordering;-use std::collections::{btree_map, BTreeMap, BTreeSet, BinaryHeap};-use std::io;-use std::ops::Bound;--use crate::collector::{Collector, SegmentCollector};-use crate::fastfield::FacetReader;-use crate::schema::Facet;-use crate::{DocId, Score, SegmentOrdinal, SegmentReader};--struct Hit<'a> {- count: u64,- facet: &'a Facet,-}--impl<'a> Eq for Hit<'a> {}--impl<'a> PartialEq<Hit<'a>> for Hit<'a> {- fn eq(&self, other: &Hit<'_>) -> bool {- self.count == other.count- }-}--impl<'a> PartialOrd<Hit<'a>> for Hit<'a> {- fn partial_cmp(&self, other: &Hit<'_>) -> Option<Ordering> {- Some(self.cmp(other))- }-}--impl<'a> Ord for Hit<'a> {- fn cmp(&self, other: &Self) -> Ordering {- other- .count- .cmp(&self.count)- .then(self.facet.cmp(other.facet))- }-}--fn facet_depth(facet_bytes: &[u8]) -> usize {- if facet_bytes.is_empty() {- 0- } else {- facet_bytes.iter().cloned().filter(|b| *b == 0u8).count() + 1- }-}--/// Collector for faceting-///-/// The collector collects all facets. You need to configure it-/// beforehand with the facet you want to extract.-///-/// This is done by calling `.add_facet(...)` with the root of the-/// facet you want to extract as argument.-///-/// Facet counts will only be computed for the facet that are direct children-/// of such a root facet.-///-/// For instance, if your index represents books, your hierarchy of facets-/// may contain `category`, `language`.-///-/// The category facet may include `subcategories`. For instance, a book-/// could belong to `/category/fiction/fantasy`.-///-/// If you request the facet counts for `/category`, the result will be-/// the breakdown of counts for the direct children of `/category`-/// (e.g. `/category/fiction`, `/category/biography`, `/category/personal_development`).-///-/// Once collection is finished, you can harvest its results in the form-/// of a [`FacetCounts`] object, and extract your facet counts from it.-///-/// This implementation assumes you are working with a number of facets that-/// is many hundreds of times smaller than your number of documents.-///-///-/// ```rust-/// use tantivy::collector::FacetCollector;-/// use tantivy::query::AllQuery;-/// use tantivy::schema::{Facet, Schema, FacetOptions, TEXT};-/// use tantivy::{doc, Index};-///-/// fn example() -> tantivy::Result<()> {-/// let mut schema_builder = Schema::builder();-///-/// // Facet have their own specific type.-/// // It is not a bad practise to put all of your-/// // facet information in the same field.-/// let facet = schema_builder.add_facet_field("facet", FacetOptions::default());-/// let title = schema_builder.add_text_field("title", TEXT);-/// let schema = schema_builder.build();-/// let index = Index::create_in_ram(schema);-/// {-/// let mut index_writer = index.writer(15_000_000)?;-/// // a document can be associated with any number of facets-/// index_writer.add_document(doc!(-/// title => "The Name of the Wind",-/// facet => Facet::from("/lang/en"),-/// facet => Facet::from("/category/fiction/fantasy")-/// ))?;-/// index_writer.add_document(doc!(-/// title => "Dune",-/// facet => Facet::from("/lang/en"),-/// facet => Facet::from("/category/fiction/sci-fi")-/// ))?;-/// index_writer.add_document(doc!(-/// title => "La Vénus d'Ille",-/// facet => Facet::from("/lang/fr"),-/// facet => Facet::from("/category/fiction/fantasy"),-/// facet => Facet::from("/category/fiction/horror")-/// ))?;-/// index_writer.add_document(doc!(-/// title => "The Diary of a Young Girl",-/// facet => Facet::from("/lang/en"),-/// facet => Facet::from("/category/biography")-/// ))?;-/// index_writer.commit()?;-/// }-/// let reader = index.reader()?;-/// let searcher = reader.searcher();-///-/// {-/// let mut facet_collector = FacetCollector::for_field("facet");-/// facet_collector.add_facet("/lang");-/// facet_collector.add_facet("/category");-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;-///-/// // This lists all of the facet counts-/// let facets: Vec<(&Facet, u64)> = facet_counts-/// .get("/category")-/// .collect();-/// assert_eq!(facets, vec![-/// (&Facet::from("/category/biography"), 1),-/// (&Facet::from("/category/fiction"), 3)-/// ]);-/// }-///-/// {-/// let mut facet_collector = FacetCollector::for_field("facet");-/// facet_collector.add_facet("/category/fiction");-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;-///-/// // This lists all of the facet counts-/// let facets: Vec<(&Facet, u64)> = facet_counts-/// .get("/category/fiction")-/// .collect();-/// assert_eq!(facets, vec![-/// (&Facet::from("/category/fiction/fantasy"), 2),-/// (&Facet::from("/category/fiction/horror"), 1),-/// (&Facet::from("/category/fiction/sci-fi"), 1)-/// ]);-/// }-///-/// {-/// let mut facet_collector = FacetCollector::for_field("facet");-/// facet_collector.add_facet("/category/fiction");-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;-///-/// // This lists all of the facet counts-/// let facets: Vec<(&Facet, u64)> = facet_counts.top_k("/category/fiction", 1);-/// assert_eq!(facets, vec![-/// (&Facet::from("/category/fiction/fantasy"), 2)-/// ]);-/// }-///-/// {-/// let mut facet_collector = FacetCollector::for_field("facet");-/// facet_collector.add_facet("/");-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;-///-/// // This lists all of the facet counts-/// let facets: Vec<(&Facet, u64)> = facet_counts-/// .get("/")-/// .collect();-/// assert_eq!(facets, vec![-/// (&Facet::from("/category"), 4),-/// (&Facet::from("/lang"), 4)-/// ]);-/// }-///-/// Ok(())-/// }-/// # assert!(example().is_ok());-/// ```-pub struct FacetCollector {- field_name: String,- facets: BTreeSet<Facet>,-}--pub struct FacetSegmentCollector {- reader: FacetReader,- // collapse facet_id -> count- counts: Vec<u64>,- // facet_ord -> compressed collapse facet_id- compressed_collapse_mapping: Vec<usize>,- // compressed collapse facet_id -> facet_ord- unique_facet_ords: Vec<(u64, usize)>,-}--impl FacetCollector {- /// Create a facet collector to collect the facets- /// from a specific facet `Field`.- ///- /// This function does not check whether the field- /// is of the proper type.- pub fn for_field(field_name: impl ToString) -> FacetCollector {- FacetCollector {- field_name: field_name.to_string(),- facets: BTreeSet::default(),- }- }-- /// Adds a facet that we want to record counts- ///- /// Adding facet `Facet::from("/country")` for instance,- /// will record the counts of all of the direct children of the facet country- /// (e.g. `/country/FR`, `/country/UK`).- ///- /// Adding two facets within which one is the prefix of the other is forbidden.- /// If you need the correct number of unique documents for two such facets,- /// just add them in a separate `FacetCollector`.- pub fn add_facet<T>(&mut self, facet_from: T)- where- Facet: From<T>,- {- let facet = Facet::from(facet_from);- for old_facet in &self.facets {- assert!(- !old_facet.is_prefix_of(&facet),- "Tried to add a facet which is a descendant of an already added facet."- );- assert!(- !facet.is_prefix_of(old_facet),- "Tried to add a facet which is an ancestor of an already added facet."- );- }- self.facets.insert(facet);- }-}--fn compress_mapping(mapping: &[(u64, usize)]) -> (Vec<usize>, Vec<(u64, usize)>) {- // facet_ord -> collapse facet_id- let mut compressed_collapse_mapping: Vec<usize> = Vec::with_capacity(mapping.len());- // collapse facet_id -> facet_ord- let mut unique_facet_ords: Vec<(u64, usize)> = Vec::new();- if mapping.is_empty() {- return (Vec::new(), Vec::new());- }- compressed_collapse_mapping.push(0);- unique_facet_ords.push(mapping[0]);- let mut last_facet_ord = mapping[0];- let mut last_facet_id = 0;- for &facet_ord in &mapping[1..] {- if facet_ord != last_facet_ord {- last_facet_id += 1;- last_facet_ord = facet_ord;- unique_facet_ords.push(facet_ord);- }- compressed_collapse_mapping.push(last_facet_id);- }- (compressed_collapse_mapping, unique_facet_ords)-}--impl Collector for FacetCollector {- type Fruit = FacetCounts;-- type Child = FacetSegmentCollector;-- fn for_segment(- &self,- _: SegmentOrdinal,- reader: &SegmentReader,- ) -> crate::Result<FacetSegmentCollector> {- let facet_reader = reader.facet_reader(&self.field_name)?;- let facet_dict = facet_reader.facet_dict();- let collapse_mapping: Vec<(u64, usize)> =- compute_collapse_mapping(facet_dict, &self.facets)?;- let (compressed_collapse_mapping, unique_facet_ords) = compress_mapping(&collapse_mapping);- let counts = vec![0u64; unique_facet_ords.len()];- Ok(FacetSegmentCollector {- reader: facet_reader,- compressed_collapse_mapping,- counts,- unique_facet_ords,- })- }-- fn requires_scoring(&self) -> bool {- false- }-- fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> crate::Result<FacetCounts> {- let mut facet_counts: BTreeMap<Facet, u64> = BTreeMap::new();- for segment_facet_counts in segments_facet_counts {- for (facet, count) in segment_facet_counts.facet_counts {- *(facet_counts.entry(facet).or_insert(0)) += count;- }- }- Ok(FacetCounts { facet_counts })- }-}--fn is_child_facet(parent_facet: &[u8], possible_child_facet: &[u8]) -> bool {- if !possible_child_facet.starts_with(parent_facet) {- return false;- }- if parent_facet.is_empty() {- return true;- }- possible_child_facet.get(parent_facet.len()).copied() == Some(0u8)-}--fn compute_collapse_mapping_one(- facet_terms: &mut crate::columnar::Streamer,- facet_bytes: &[u8],- collapsed: &mut [(u64, usize)],-) -> io::Result<bool> {- let mut facet_child: Vec<u8> = Vec::new();- let mut term_ord = 0;- let offset = facet_bytes.len() + 1;- let depth = facet_depth(facet_bytes);- loop {- match facet_terms.key().cmp(facet_bytes) {- Ordering::Less | Ordering::Equal => {}- Ordering::Greater => {- if !is_child_facet(facet_bytes, facet_terms.key()) {- return Ok(true);- }- let suffix = &facet_terms.key()[offset..];- if facet_child.is_empty() || !is_child_facet(&facet_child, suffix) {- facet_child.clear();- term_ord = facet_terms.term_ord();- let end = suffix- .iter()- .position(|b| *b == 0u8)- .unwrap_or(suffix.len());- facet_child.extend(&suffix[..end]);- }- collapsed[facet_terms.term_ord() as usize] = (term_ord, depth);- }- }- if !facet_terms.advance() {- return Ok(false);- }- }-}--fn compute_collapse_mapping(- facet_dict: &crate::columnar::Dictionary,- facets: &BTreeSet<Facet>,-) -> io::Result<Vec<(u64, usize)>> {- let mut collapsed = vec![(u64::MAX, 0); facet_dict.num_terms()];- if facets.is_empty() {- return Ok(collapsed);- }- let mut facet_terms: crate::columnar::Streamer = facet_dict.range().into_stream()?;- if !facet_terms.advance() {- return Ok(collapsed);- }- let mut facet_bytes = Vec::new();- for facet in facets {- facet_bytes.clear();- facet_bytes.extend(facet.encoded_str().as_bytes());- if !compute_collapse_mapping_one(&mut facet_terms, &facet_bytes, &mut collapsed[..])? {- break;- }- }- Ok(collapsed)-}--impl SegmentCollector for FacetSegmentCollector {- type Fruit = FacetCounts;-- fn collect(&mut self, doc: DocId, _: Score) {- let mut previous_collapsed_ord: usize = usize::MAX;- for facet_ord in self.reader.facet_ords(doc) {- let collapsed_ord = self.compressed_collapse_mapping[facet_ord as usize];- self.counts[collapsed_ord] += u64::from(collapsed_ord != previous_collapsed_ord);- previous_collapsed_ord = collapsed_ord;- }- }-- /// Returns the results of the collection.- ///- /// This method does not just return the counters,- /// it also translates the facet ordinals of the last segment.- fn harvest(self) -> FacetCounts {- let mut facet_counts = BTreeMap::new();- let facet_dict = self.reader.facet_dict();- for (collapsed_facet_ord, count) in self.counts.iter().cloned().enumerate() {- if count == 0 {- continue;- }- let mut facet = vec![];- let (facet_ord, facet_depth) = self.unique_facet_ords[collapsed_facet_ord];- // TODO handle errors.- if facet_dict.ord_to_term(facet_ord, &mut facet).is_ok() {- if let Some((end_collapsed_facet, _)) = facet- .iter()- .enumerate()- .filter(|(_pos, &b)| b == 0u8)- .nth(facet_depth)- {- facet.truncate(end_collapsed_facet);- }- if let Ok(facet) = Facet::from_encoded(facet) {- facet_counts.insert(facet, count);- }- }- }- FacetCounts { facet_counts }- }-}--/// Intermediary result of the `FacetCollector` that stores-/// the facet counts for all the segments.-#[derive(Default, Clone)]-pub struct FacetCounts {- facet_counts: BTreeMap<Facet, u64>,-}--pub struct FacetChildIterator<'a> {- underlying: btree_map::Range<'a, Facet, u64>,-}--impl<'a> Iterator for FacetChildIterator<'a> {- type Item = (&'a Facet, u64);-- fn next(&mut self) -> Option<Self::Item> {- self.underlying.next().map(|(facet, count)| (facet, *count))- }-}--impl FacetCounts {- /// Returns an iterator over all of the facet count pairs inside this result.- /// See the documentation for [`FacetCollector`] for a usage example.- pub fn get<T>(&self, facet_from: T) -> FacetChildIterator<'_>- where- Facet: From<T>,- {- let facet = Facet::from(facet_from);- let lower_bound = Bound::Excluded(facet.clone());- let upper_bound = if facet.is_root() {- Bound::Unbounded- } else {- let mut facet_after_bytes: String = facet.encoded_str().to_owned();- facet_after_bytes.push('\u{1}');- let facet_after = Facet::from_encoded_string(facet_after_bytes);- Bound::Excluded(facet_after)- };- let underlying: btree_map::Range<'_, _, _> =- self.facet_counts.range((lower_bound, upper_bound));- FacetChildIterator { underlying }- }-- /// Returns a vector of top `k` facets with their counts, sorted highest-to-lowest by counts.- /// See the documentation for [`FacetCollector`] for a usage example.- pub fn top_k<T>(&self, facet: T, k: usize) -> Vec<(&Facet, u64)>- where- Facet: From<T>,- {- let mut heap = BinaryHeap::with_capacity(k);- let mut it = self.get(facet);-- // push the first k elements to first bring the heap- // to capacity- for (facet, count) in (&mut it).take(k) {- heap.push(Hit { count, facet });- }-- let mut lowest_count: u64 = heap.peek().map(|hit| hit.count).unwrap_or(u64::MIN); //< the `unwrap_or` case may be triggered but the value- // is never used in that case.-- for (facet, count) in it {- if count > lowest_count {- if let Some(mut head) = heap.peek_mut() {- *head = Hit { count, facet };- }- // the heap gets reconstructed at this point- if let Some(head) = heap.peek() {- lowest_count = head.count;- }- }- }- heap.into_sorted_vec()- .into_iter()- .map(|hit| (hit.facet, hit.count))- .collect::<Vec<_>>()- }-}--#[cfg(test)]-mod tests {- use std::collections::BTreeSet;- use std::iter;-- use crate::columnar::Dictionary;- use rand::distributions::Uniform;- use rand::prelude::SliceRandom;- use rand::{thread_rng, Rng};-- use super::{FacetCollector, FacetCounts};- use crate::collector::facet_collector::compress_mapping;- use crate::collector::Count;- use crate::index::Index;- use crate::query::{AllQuery, QueryParser, TermQuery};- use crate::schema::{Facet, FacetOptions, IndexRecordOption, Schema, TantivyDocument};- use crate::{IndexWriter, Term};-- fn test_collapse_mapping_aux(- facet_terms: &[&str],- facet_params: &[&str],- expected_collapsed_mapping: &[(u64, usize)],- ) {- let mut facets: Vec<Facet> = facet_terms.iter().map(Facet::from).collect();- facets.sort();- let facet_terms: Vec<&str> = facets.iter().map(|facet| facet.encoded_str()).collect();- let dictionary = Dictionary::build_for_tests(&facet_terms);- let facet_params: BTreeSet<Facet> = facet_params.iter().map(Facet::from).collect();- let collapse_mapping = super::compute_collapse_mapping(&dictionary, &facet_params).unwrap();- assert_eq!(&collapse_mapping[..], expected_collapsed_mapping);- }-- #[test]- fn test_collapse_simple() {- test_collapse_mapping_aux(&["/facet/a", "/facet/b"], &["/facet"], &[(0, 1), (1, 1)]);- test_collapse_mapping_aux(- &["/facet/a", "/facet/a2", "/facet/b"],- &["/facet"],- &[(0, 1), (1, 1), (2, 1)],- );- test_collapse_mapping_aux(&["/facet/a", "/facet/a/2"], &["/facet"], &[(0, 1), (0, 1)]);- test_collapse_mapping_aux(- &["/facet/a", "/facet/a/2", "/facet/b"],- &["/facet"],- &[(0, 1), (0, 1), (2, 1)],- );- }-- fn test_compress_mapping_aux(- collapsed_mapping: &[(u64, usize)],- expected_compressed_collapsed_mapping: &[usize],- expected_unique_facet_ords: &[(u64, usize)],- ) {- let (compressed_collapsed_mapping, unique_facet_ords) = compress_mapping(collapsed_mapping);- assert_eq!(- compressed_collapsed_mapping,- expected_compressed_collapsed_mapping- );- assert_eq!(unique_facet_ords, expected_unique_facet_ords);- }-- #[test]- fn test_compress_mapping() {- test_compress_mapping_aux(&[], &[], &[]);- test_compress_mapping_aux(&[(1, 2)], &[0], &[(1, 2)]);- test_compress_mapping_aux(&[(1, 2), (1, 2)], &[0, 0], &[(1, 2)]);- test_compress_mapping_aux(- &[(1, 2), (5, 2), (5, 2), (6, 3), (8, 3)],- &[0, 1, 1, 2, 3],- &[(1, 2), (5, 2), (6, 3), (8, 3)],- );- }-- #[test]- fn test_facet_collector_simple() {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(facet_field=>Facet::from("/facet/a")))- .unwrap();- index_writer- .add_document(doc!(facet_field=>Facet::from("/facet/b")))- .unwrap();- index_writer- .add_document(doc!(facet_field=>Facet::from("/facet/b")))- .unwrap();- index_writer- .add_document(doc!(facet_field=>Facet::from("/facet/c")))- .unwrap();- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet("/facet");- let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector).unwrap();- let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 1);- assert_eq!(facets, vec![(&Facet::from("/facet/b"), 2)]);- }-- #[test]- fn test_facet_collector_drilldown() {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- let num_facets: usize = 3 * 4 * 5;- let facets: Vec<Facet> = (0..num_facets)- .map(|mut n| {- let top = n % 3;- n /= 3;- let mid = n % 4;- n /= 4;- let leaf = n % 5;- Facet::from(&format!("/top{top}/mid{mid}/leaf{leaf}"))- })- .collect();- for i in 0..num_facets * 10 {- let mut doc = TantivyDocument::new();- doc.add_facet(facet_field, facets[i % num_facets].clone());- index_writer.add_document(doc).unwrap();- }- index_writer.commit().unwrap();- let reader = index.reader().unwrap();- let searcher = reader.searcher();- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet(Facet::from("/top1"));- let counts = searcher.search(&AllQuery, &facet_collector).unwrap();-- {- let facets: Vec<(String, u64)> = counts- .get("/top1")- .map(|(facet, count)| (facet.to_string(), count))- .collect();- assert_eq!(- facets,- [- ("/top1/mid0", 50),- ("/top1/mid1", 50),- ("/top1/mid2", 50),- ("/top1/mid3", 50),- ]- .iter()- .map(|&(facet_str, count)| (String::from(facet_str), count))- .collect::<Vec<_>>()- );- }- }-- #[test]- #[should_panic(- expected = "Tried to add a facet which is a descendant of an already added facet."- )]- fn test_misused_facet_collector() {- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet(Facet::from("/country"));- facet_collector.add_facet(Facet::from("/country/europe"));- }-- #[test]- fn test_doc_unsorted_multifacet() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facets", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer = index.writer_for_tests()?;- index_writer.add_document(doc!(- facet_field => Facet::from_text(&"/subjects/A/a").unwrap(),- facet_field => Facet::from_text(&"/subjects/B/a").unwrap(),- facet_field => Facet::from_text(&"/subjects/A/b").unwrap(),- facet_field => Facet::from_text(&"/subjects/B/b").unwrap(),- ))?;- index_writer.commit()?;- let reader = index.reader()?;- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 1);- let mut facet_collector = FacetCollector::for_field("facets");- facet_collector.add_facet("/subjects");- let counts = searcher.search(&AllQuery, &facet_collector)?;- let facets: Vec<(&Facet, u64)> = counts.get("/subjects").collect();- assert_eq!(facets[0].1, 1);- Ok(())- }-- #[test]- fn test_doc_search_by_facet() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer = index.writer_for_tests()?;- index_writer.add_document(doc!(- facet_field => Facet::from_text(&"/A/A").unwrap(),- ))?;- index_writer.add_document(doc!(- facet_field => Facet::from_text(&"/A/B").unwrap(),- ))?;- index_writer.add_document(doc!(- facet_field => Facet::from_text(&"/A/C/A").unwrap(),- ))?;- index_writer.add_document(doc!(- facet_field => Facet::from_text(&"/D/C/A").unwrap(),- ))?;- index_writer.commit()?;- let reader = index.reader()?;- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 4);-- let count_facet = |facet_str: &str| {- let term = Term::from_facet(facet_field, &Facet::from_text(facet_str).unwrap());- searcher- .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)- .unwrap()- };-- assert_eq!(count_facet("/"), 4);- assert_eq!(count_facet("/A"), 3);- assert_eq!(count_facet("/A/B"), 1);- assert_eq!(count_facet("/A/C"), 1);- assert_eq!(count_facet("/A/C/A"), 1);- assert_eq!(count_facet("/C/A"), 0);-- let query_parser = QueryParser::for_index(&index, vec![]);- {- let query = query_parser.parse_query("facet:/A/B")?;- assert_eq!(1, searcher.search(&query, &Count).unwrap());- }- {- let query = query_parser.parse_query("facet:/A")?;- assert_eq!(3, searcher.search(&query, &Count)?);- }- Ok(())- }-- #[test]- fn test_non_used_facet_collector() {- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet(Facet::from("/country"));- facet_collector.add_facet(Facet::from("/countryeurope"));- }-- #[test]- fn test_facet_collector_topk() {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- let uniform = Uniform::new_inclusive(1, 100_000);- let mut docs: Vec<TantivyDocument> =- vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]- .into_iter()- .flat_map(|(c, count)| {- let facet = Facet::from(&format!("/facet/{c}"));- let doc = doc!(facet_field => facet);- iter::repeat(doc).take(count)- })- .map(|mut doc| {- doc.add_facet(- facet_field,- &format!("/facet/{}", thread_rng().sample(uniform)),- );- doc- })- .collect();- docs[..].shuffle(&mut thread_rng());-- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- for doc in docs {- index_writer.add_document(doc).unwrap();- }- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();-- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet("/facet");- let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector).unwrap();-- {- let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 3);- assert_eq!(- facets,- vec![- (&Facet::from("/facet/b"), 100),- (&Facet::from("/facet/e"), 21),- (&Facet::from("/facet/d"), 12),- ]- );- }- }-- #[test]- fn test_facet_collector_topk_tie_break() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]- .into_iter()- .flat_map(|(c, count)| {- let facet = Facet::from(&format!("/facet/{c}"));- let doc = doc!(facet_field => facet);- iter::repeat(doc).take(count)- })- .collect();-- let mut index_writer = index.writer_for_tests()?;- for doc in docs {- index_writer.add_document(doc)?;- }- index_writer.commit()?;-- let searcher = index.reader()?.searcher();- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet("/facet");- let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector)?;-- let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 2);- assert_eq!(- facets,- vec![(&Facet::from("/facet/c"), 4), (&Facet::from("/facet/a"), 2)]- );- Ok(())- }-- #[test]- fn is_child_facet() {- assert!(super::is_child_facet(&b"foo"[..], &b"foo\0bar"[..]));- assert!(super::is_child_facet(&b""[..], &b"foo\0bar"[..]));- assert!(super::is_child_facet(&b""[..], &b"foo"[..]));- assert!(!super::is_child_facet(&b"foo\0bar"[..], &b"foo"[..]));- assert!(!super::is_child_facet(&b"foo"[..], &b"foobar\0baz"[..]));- }-}diff --git a/crates/tantivy/src/collector/filter_collector_wrapper.rs b/crates/tantivy/src/collector/filter_collector_wrapper.rsindex df121222..e68ffe37 100644--- a/crates/tantivy/src/collector/filter_collector_wrapper.rs+++ b/crates/tantivy/src/collector/filter_collector_wrapper.rs@@ -112,7 +112,7 @@ where#[inline]fn accept_document(&self, doc_id: DocId) -> bool {if let Some(column) = &self.column_opt {- for val in column.values_for_doc(doc_id) {+ if let Some(val) = column.first(doc_id) {if (self.predicate)(val) {return true;}diff --git a/crates/tantivy/src/collector/histogram_collector.rs b/crates/tantivy/src/collector/histogram_collector.rsdeleted file mode 100644index 2d99cca4..00000000--- a/crates/tantivy/src/collector/histogram_collector.rs+++ /dev/null@@ -1,305 +0,0 @@-use std::sync::Arc;--use crate::columnar::ColumnValues;-use fastdivide::DividerU64;--use crate::collector::{Collector, SegmentCollector};-use crate::fastfield::{FastFieldNotAvailableError, FastValue};-use crate::schema::Type;-use crate::{DocId, Score};--/// Histogram builds an histogram of the values of a fastfield for the-/// collected DocSet.-///-/// At construction, it is given parameters that define a partition of an interval-/// [min_val, max_val) into N buckets with the same width.-/// The ith bucket is then defined by `[min_val + i * bucket_width, min_val + (i+1) * bucket_width)`-///-/// An histogram is then defined as a `Vec<u64>` of length `num_buckets`, that contains a count of-/// documents for each value bucket.-///-/// See also [`HistogramCollector::new()`].-///-/// # Warning-///-/// f64 fields are not supported.-#[derive(Clone)]-pub struct HistogramCollector {- min_value: u64,- num_buckets: usize,- divider: DividerU64,- field: String,-}--impl HistogramCollector {- /// Builds a new HistogramCollector.- ///- /// The scale/range of the histogram is not dynamic. It is required to- /// define it by supplying following parameter:- /// - `min_value`: the minimum value that can be recorded in the histogram.- /// - `bucket_width`: the length of the interval that is associated with each buckets.- /// - `num_buckets`: The overall number of buckets.- ///- /// Together, this parameters define a partition of `[min_value, min_value + num_buckets *- /// bucket_width)` into `num_buckets` intervals of width bucket that we call `bucket`.- ///- /// # Disclaimer- /// This function panics if the field given is of type f64.- pub fn new<TFastValue: FastValue>(- field: String,- min_value: TFastValue,- bucket_width: u64,- num_buckets: usize,- ) -> HistogramCollector {- let fast_type = TFastValue::to_type();- assert!(fast_type == Type::U64 || fast_type == Type::I64 || fast_type == Type::Date);- HistogramCollector {- min_value: min_value.to_u64(),- num_buckets,- field,- divider: DividerU64::divide_by(bucket_width),- }- }-}--struct HistogramComputer {- counts: Vec<u64>,- min_value: u64,- divider: DividerU64,-}--impl HistogramComputer {- #[inline]- pub(crate) fn add_value(&mut self, value: u64) {- if value < self.min_value {- return;- }- let delta = value - self.min_value;- let bucket_id: usize = self.divider.divide(delta) as usize;- if bucket_id < self.counts.len() {- self.counts[bucket_id] += 1;- }- }-- fn harvest(self) -> Vec<u64> {- self.counts- }-}-pub struct SegmentHistogramCollector {- histogram_computer: HistogramComputer,- column_u64: Arc<dyn ColumnValues<u64>>,-}--impl SegmentCollector for SegmentHistogramCollector {- type Fruit = Vec<u64>;-- fn collect(&mut self, doc: DocId, _score: Score) {- let value = self.column_u64.get_val(doc);- self.histogram_computer.add_value(value);- }-- fn harvest(self) -> Self::Fruit {- self.histogram_computer.harvest()- }-}--impl Collector for HistogramCollector {- type Fruit = Vec<u64>;- type Child = SegmentHistogramCollector;-- fn for_segment(- &self,- _segment_local_id: crate::SegmentOrdinal,- segment: &crate::SegmentReader,- ) -> crate::Result<Self::Child> {- let column_opt = segment.fast_fields().u64_lenient(&self.field)?;- let (column, _column_type) = column_opt.ok_or_else(|| FastFieldNotAvailableError {- field_name: self.field.clone(),- })?;- let column_u64 = column.first_or_default_col(0u64);- Ok(SegmentHistogramCollector {- histogram_computer: HistogramComputer {- counts: vec![0; self.num_buckets],- min_value: self.min_value,- divider: self.divider,- },- column_u64,- })- }-- fn requires_scoring(&self) -> bool {- false- }-- fn merge_fruits(&self, child_histograms: Vec<Vec<u64>>) -> crate::Result<Vec<u64>> {- Ok(add_vecs(child_histograms, self.num_buckets))- }-}--pub fn add_arrays_into(acc: &mut [u64], add: &[u64]) {- assert_eq!(acc.len(), add.len());- for (dest_bucket, bucket_count) in acc.iter_mut().zip(add) {- *dest_bucket += bucket_count;- }-}--fn add_vecs(mut vals_list: Vec<Vec<u64>>, len: usize) -> Vec<u64> {- let mut acc = vals_list.pop().unwrap_or_else(|| vec![0u64; len]);- assert_eq!(acc.len(), len);- for vals in vals_list {- add_arrays_into(&mut acc, &vals);- }- acc-}--#[cfg(test)]-mod tests {- use fastdivide::DividerU64;- use query::AllQuery;-- use super::{add_vecs, HistogramCollector, HistogramComputer};- use crate::schema::{Schema, FAST};- use crate::time::{Date, Month};- use crate::{query, DateTime, Index};-- #[test]- fn test_add_histograms_simple() {- assert_eq!(- add_vecs(vec![vec![1, 0, 3], vec![11, 2, 3], vec![0, 0, 1]], 3),- vec![12, 2, 7]- )- }-- #[test]- fn test_add_histograms_empty() {- assert_eq!(add_vecs(vec![], 3), vec![0, 0, 0])- }-- #[test]- fn test_histogram_builder_simple() {- // [1..3)- // [3..5)- // ..- // [9..11)- let mut histogram_computer = HistogramComputer {- counts: vec![0; 5],- min_value: 1,- divider: DividerU64::divide_by(2),- };- histogram_computer.add_value(1);- histogram_computer.add_value(7);- assert_eq!(histogram_computer.harvest(), vec![1, 0, 0, 1, 0]);- }-- #[test]- fn test_histogram_too_low_is_ignored() {- let mut histogram_computer = HistogramComputer {- counts: vec![0; 5],- min_value: 2,- divider: DividerU64::divide_by(2),- };- histogram_computer.add_value(0);- assert_eq!(histogram_computer.harvest(), vec![0, 0, 0, 0, 0]);- }-- #[test]- fn test_histogram_too_high_is_ignored() {- let mut histogram_computer = HistogramComputer {- counts: vec![0u64; 5],- min_value: 0,- divider: DividerU64::divide_by(2),- };- histogram_computer.add_value(10);- assert_eq!(histogram_computer.harvest(), vec![0, 0, 0, 0, 0]);- }- #[test]- fn test_no_segments() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- schema_builder.add_u64_field("val_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let reader = index.reader()?;- let searcher = reader.searcher();- let all_query = AllQuery;- let histogram_collector = HistogramCollector::new("val_field".to_string(), 0u64, 2, 5);- let histogram = searcher.search(&all_query, &histogram_collector)?;- assert_eq!(histogram, vec![0; 5]);- Ok(())- }-- #[test]- fn test_histogram_i64() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let val_field = schema_builder.add_i64_field("val_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer = index.writer_for_tests()?;- writer.add_document(doc!(val_field=>12i64))?;- writer.add_document(doc!(val_field=>-30i64))?;- writer.add_document(doc!(val_field=>-12i64))?;- writer.add_document(doc!(val_field=>-10i64))?;- writer.commit()?;- let reader = index.reader()?;- let searcher = reader.searcher();- let all_query = AllQuery;- let histogram_collector =- HistogramCollector::new("val_field".to_string(), -20i64, 10u64, 4);- let histogram = searcher.search(&all_query, &histogram_collector)?;- assert_eq!(histogram, vec![1, 1, 0, 1]);- Ok(())- }-- #[test]- fn test_histogram_merge() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let val_field = schema_builder.add_i64_field("val_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer = index.writer_for_tests()?;- writer.add_document(doc!(val_field=>12i64))?;- writer.commit()?;- writer.add_document(doc!(val_field=>-30i64))?;- writer.commit()?;- writer.add_document(doc!(val_field=>-12i64))?;- writer.commit()?;- writer.add_document(doc!(val_field=>-10i64))?;- writer.commit()?;- let reader = index.reader()?;- let searcher = reader.searcher();- let all_query = AllQuery;- let histogram_collector =- HistogramCollector::new("val_field".to_string(), -20i64, 10u64, 4);- let histogram = searcher.search(&all_query, &histogram_collector)?;- assert_eq!(histogram, vec![1, 1, 0, 1]);- Ok(())- }-- #[test]- fn test_histogram_dates() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let date_field = schema_builder.add_date_field("date_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer = index.writer_for_tests()?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;- writer.add_document(- doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1986, Month::March, 9)?.with_hms(0, 0, 0)?)),- )?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;- writer.commit()?;- let reader = index.reader()?;- let searcher = reader.searcher();- let all_query = AllQuery;- let week_histogram_collector = HistogramCollector::new(- "date_field".to_string(),- DateTime::from_primitive(- Date::from_calendar_date(1980, Month::January, 1)?.with_hms(0, 0, 0)?,- ),- 3_600_000_000_000 * 24 * 365, // it is just for a unit test... sorry leap years.- 10,- );- let week_histogram = searcher.search(&all_query, &week_histogram_collector)?;- assert_eq!(week_histogram, vec![0, 0, 1, 1, 0, 0, 1, 0, 0, 0]);- Ok(())- }-}diff --git a/crates/tantivy/src/collector/mod.rs b/crates/tantivy/src/collector/mod.rsindex 1325f43e..ca5b1962 100644--- a/crates/tantivy/src/collector/mod.rs+++ b/crates/tantivy/src/collector/mod.rs@@ -59,9 +59,6 @@ use crate::{DocId, Score, SegmentOrdinal, SegmentReader};mod count_collector;pub use self::count_collector::Count;-mod histogram_collector;-pub use histogram_collector::HistogramCollector;-mod multi_collector;pub use self::multi_collector::{FruitHandle, MultiCollector, MultiFruit};@@ -76,8 +73,6 @@ pub use self::custom_score_top_collector::{CustomScorer, CustomSegmentScorer};mod tweak_score_top_collector;pub use self::tweak_score_top_collector::{ScoreSegmentTweaker, ScoreTweaker};-mod facet_collector;-pub use self::facet_collector::{FacetCollector, FacetCounts};use crate::query::Weight;mod docset_collector;diff --git a/crates/tantivy/src/collector/tests.rs b/crates/tantivy/src/collector/tests.rsindex 4b2c9c0b..ee3aa8bd 100644--- a/crates/tantivy/src/collector/tests.rs+++ b/crates/tantivy/src/collector/tests.rs@@ -202,7 +202,7 @@ impl SegmentCollector for FastFieldSegmentCollector {type Fruit = Vec<u64>;fn collect(&mut self, doc: DocId, _score: Score) {- self.vals.extend(self.reader.values_for_doc(doc));+ self.vals.extend(self.reader.first(doc));}fn harvest(self) -> Vec<u64> {diff --git a/crates/tantivy/src/collector/top_score_collector.rs b/crates/tantivy/src/collector/top_score_collector.rsindex a3eb05c1..632f4194 100644--- a/crates/tantivy/src/collector/top_score_collector.rs+++ b/crates/tantivy/src/collector/top_score_collector.rs@@ -143,12 +143,8 @@ impl CustomScorer<u64> for ScorerByField {sort_column_opt.ok_or_else(|| FastFieldNotAvailableError {field_name: self.field.clone(),})?;- let mut default_value = 0u64;- if self.order.is_asc() {- default_value = u64::MAX;- }Ok(ScorerByFastFieldReader {- sort_column: sort_column.first_or_default_col(default_value),+ sort_column: sort_column.values,order: self.order.clone(),})}@@ -1083,6 +1079,7 @@ mod tests {index_writer.add_document(doc!(title => "empty beer",+ size => 18446744073709551615u64,)).unwrap();});diff --git a/crates/tantivy/src/columnar/block_accessor.rs b/crates/tantivy/src/columnar/block_accessor.rsindex f5687cc5..0807b2df 100644--- a/crates/tantivy/src/columnar/block_accessor.rs+++ b/crates/tantivy/src/columnar/block_accessor.rs@@ -1,13 +1,8 @@-use std::cmp::Ordering;--use super::{Column, DocId, RowId};+use super::{Column, DocId};#[derive(Debug, Default, Clone)]pub struct ColumnBlockAccessor<T> {val_cache: Vec<T>,- docid_cache: Vec<DocId>,- missing_docids_cache: Vec<DocId>,- row_id_cache: Vec<RowId>,}impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>@@ -15,39 +10,12 @@ impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>{#[inline]pub fn fetch_block<'a>(&'a mut self, docs: &'a [u32], accessor: &Column<T>) {- if accessor.index.get_cardinality().is_full() {- self.val_cache.resize(docs.len(), T::default());- accessor.values.get_vals(docs, &mut self.val_cache);- } else {- self.docid_cache.clear();- self.row_id_cache.clear();- accessor.row_ids_for_docs(docs, &mut self.docid_cache, &mut self.row_id_cache);- self.val_cache.resize(self.row_id_cache.len(), T::default());- accessor- .values- .get_vals(&self.row_id_cache, &mut self.val_cache);- }+ self.val_cache.resize(docs.len(), T::default());+ accessor.values.get_vals(docs, &mut self.val_cache);}#[inline]- pub fn fetch_block_with_missing(&mut self, docs: &[u32], accessor: &Column<T>, missing: T) {+ pub fn fetch_block_with_missing(&mut self, docs: &[u32], accessor: &Column<T>) {self.fetch_block(docs, accessor);- // no missing values- if accessor.index.get_cardinality().is_full() {- return;- }-- // We can compare docid_cache length with docs to find missing docs- // For multi value columns we can't rely on the length and always need to scan- if accessor.index.get_cardinality().is_multivalue() || docs.len() != self.docid_cache.len()- {- self.missing_docids_cache.clear();- find_missing_docs(docs, &self.docid_cache, |doc| {- self.missing_docids_cache.push(doc);- self.val_cache.push(missing);- });- self.docid_cache- .extend_from_slice(&self.missing_docids_cache);- }}#[inline]@@ -62,99 +30,7 @@ impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>////// The docs is used if the column is full (each docs has exactly one value), otherwise the/// internal docid vec is used for the iterator, which e.g. may contain duplicate docs.- pub fn iter_docid_vals<'a>(- &'a self,- docs: &'a [u32],- accessor: &Column<T>,- ) -> impl Iterator<Item = (DocId, T)> + '_ {- if accessor.index.get_cardinality().is_full() {- docs.iter().cloned().zip(self.val_cache.iter().cloned())- } else {- self.docid_cache- .iter()- .cloned()- .zip(self.val_cache.iter().cloned())- }- }-}--/// Given two sorted lists of docids `docs` and `hits`, hits is a subset of `docs`.-/// Return all docs that are not in `hits`.-fn find_missing_docs<F>(docs: &[u32], hits: &[u32], mut callback: F)-where- F: FnMut(u32),-{- let mut docs_iter = docs.iter();- let mut hits_iter = hits.iter();-- let mut doc = docs_iter.next();- let mut hit = hits_iter.next();-- while let (Some(¤t_doc), Some(¤t_hit)) = (doc, hit) {- match current_doc.cmp(¤t_hit) {- Ordering::Less => {- callback(current_doc);- doc = docs_iter.next();- }- Ordering::Equal => {- doc = docs_iter.next();- hit = hits_iter.next();- }- Ordering::Greater => {- hit = hits_iter.next();- }- }- }-- while let Some(¤t_doc) = doc {- callback(current_doc);- doc = docs_iter.next();- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_find_missing_docs() {- let docs: Vec<u32> = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];- let hits: Vec<u32> = vec![2, 4, 6, 8, 10];-- let mut missing_docs: Vec<u32> = Vec::new();-- find_missing_docs(&docs, &hits, |missing_doc| {- missing_docs.push(missing_doc);- });-- assert_eq!(missing_docs, vec![1, 3, 5, 7, 9]);- }-- #[test]- fn test_find_missing_docs_empty() {- let docs: Vec<u32> = Vec::new();- let hits: Vec<u32> = vec![2, 4, 6, 8, 10];-- let mut missing_docs: Vec<u32> = Vec::new();-- find_missing_docs(&docs, &hits, |missing_doc| {- missing_docs.push(missing_doc);- });-- assert_eq!(missing_docs, Vec::<u32>::new());- }-- #[test]- fn test_find_missing_docs_all_missing() {- let docs: Vec<u32> = vec![1, 2, 3, 4, 5];- let hits: Vec<u32> = Vec::new();-- let mut missing_docs: Vec<u32> = Vec::new();-- find_missing_docs(&docs, &hits, |missing_doc| {- missing_docs.push(missing_doc);- });-- assert_eq!(missing_docs, vec![1, 2, 3, 4, 5]);+ pub fn iter_docid_vals<'a>(&'a self, docs: &'a [u32]) -> impl Iterator<Item = (DocId, T)> + '_ {+ docs.iter().cloned().zip(self.val_cache.iter().cloned())}}diff --git a/crates/tantivy/src/columnar/column/dictionary_encoded.rs b/crates/tantivy/src/columnar/column/dictionary_encoded.rsindex 28847329..bbe4c1d1 100644--- a/crates/tantivy/src/columnar/column/dictionary_encoded.rs+++ b/crates/tantivy/src/columnar/column/dictionary_encoded.rs@@ -1,4 +1,3 @@-use std::ops::Deref;use std::sync::Arc;use std::{fmt, io};@@ -30,13 +29,6 @@ impl fmt::Debug for BytesColumn {}impl BytesColumn {- pub fn empty(num_docs: u32) -> BytesColumn {- BytesColumn {- dictionary: Arc::new(Dictionary::empty()),- term_ord_column: Column::build_empty_column(num_docs),- }- }-/// Fills the given `output` buffer with the term associated to the ordinal `ord`.////// Returns `false` if the term does not exist (e.g. `term_ord` is greater or equal to the@@ -51,7 +43,7 @@ impl BytesColumn {}pub fn term_ords(&self, row_id: RowId) -> impl Iterator<Item = u64> + '_ {- self.term_ord_column.values_for_doc(row_id)+ self.term_ord_column.first(row_id).into_iter()}/// Returns the column of ordinals@@ -67,55 +59,3 @@ impl BytesColumn {self.dictionary.as_ref()}}--#[derive(Clone)]-pub struct StrColumn(BytesColumn);--impl fmt::Debug for StrColumn {- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {- write!(f, "{:?}", self.term_ord_column)- }-}--impl From<StrColumn> for BytesColumn {- fn from(str_column: StrColumn) -> BytesColumn {- str_column.0- }-}--impl StrColumn {- pub fn wrap(bytes_column: BytesColumn) -> StrColumn {- StrColumn(bytes_column)- }-- pub fn dictionary(&self) -> &Dictionary<VoidSSTable> {- self.0.dictionary.as_ref()- }-- /// Fills the buffer- pub fn ord_to_str(&self, term_ord: u64, output: &mut String) -> io::Result<bool> {- unsafe {- let buf = output.as_mut_vec();- if !self.0.dictionary.ord_to_term(term_ord, buf)? {- return Ok(false);- }- // TODO consider remove checks if it hurts performance.- if std::str::from_utf8(buf.as_slice()).is_err() {- buf.clear();- return Err(io::Error::new(- io::ErrorKind::InvalidData,- "Not valid utf-8",- ));- }- }- Ok(true)- }-}--impl Deref for StrColumn {- type Target = BytesColumn;-- fn deref(&self) -> &Self::Target {- &self.0- }-}diff --git a/crates/tantivy/src/columnar/column/mod.rs b/crates/tantivy/src/columnar/column/mod.rsindex d4158dd4..1a6a32d2 100644--- a/crates/tantivy/src/columnar/column/mod.rs+++ b/crates/tantivy/src/columnar/column/mod.rs@@ -7,16 +7,13 @@ use std::ops::{Range, RangeInclusive};use std::sync::Arc;use crate::common::BinarySerializable;-pub use dictionary_encoded::{BytesColumn, StrColumn};-pub use serialize::{- open_column_bytes, open_column_str, open_column_u128, open_column_u128_as_compact_u64,- open_column_u64, serialize_column_mappable_to_u128, serialize_column_mappable_to_u64,-};+pub use dictionary_encoded::BytesColumn;+pub use serialize::{open_column_bytes, open_column_u64, serialize_column_mappable_to_u64};-use super::column_index::{ColumnIndex, Set};+use super::column_index::ColumnIndex;use super::column_values::monotonic_mapping::StrictlyMonotonicMappingToInternal;use super::column_values::{monotonic_map_column, ColumnValues};-use super::{Cardinality, DocId, EmptyColumnValues, MonotonicallyMappableToU64, RowId};+use super::{Cardinality, DocId, MonotonicallyMappableToU64, RowId};#[derive(Clone)]pub struct Column<T = u64> {@@ -28,21 +25,12 @@ impl<T: Debug + PartialOrd + Send + Sync + Copy + 'static> Debug for Column<T> {fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {let num_docs = self.num_docs();let entries = (0..num_docs)- .map(|i| (i, self.values_for_doc(i).collect::<Vec<_>>()))- .filter(|(_, vals)| !vals.is_empty());+ .map(|i| (i, self.first(i)))+ .collect::<Vec<_>>();f.debug_map().entries(entries).finish()}}-impl<T: PartialOrd + Default> Column<T> {- pub fn build_empty_column(num_docs: u32) -> Column<T> {- Column {- index: ColumnIndex::Empty { num_docs },- values: Arc::new(EmptyColumnValues),- }- }-}-impl<T: MonotonicallyMappableToU64> Column<T> {pub fn to_u64_monotonic(self) -> Column<u64> {let values = Arc::new(monotonic_map_column(@@ -57,21 +45,9 @@ impl<T: MonotonicallyMappableToU64> Column<T> {}impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {- #[inline]- pub fn get_cardinality(&self) -> Cardinality {- self.index.get_cardinality()- }-pub fn num_docs(&self) -> RowId {match &self.index {- ColumnIndex::Empty { num_docs } => *num_docs,ColumnIndex::Full => self.values.num_vals(),- ColumnIndex::Optional(optional_index) => optional_index.num_docs(),- ColumnIndex::Multivalued(col_index) => {- // The multivalued index contains all value start row_id,- // and one extra value at the end with the overall number of rows.- col_index.num_docs()- }}}@@ -85,31 +61,14 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {#[inline]pub fn first(&self, row_id: RowId) -> Option<T> {- self.values_for_doc(row_id).next()+ Some(self.values.get_val(row_id))}/// Load the first value for each docid in the provided slice.#[inline]pub fn first_vals(&self, docids: &[DocId], output: &mut [Option<T>]) {match &self.index {- ColumnIndex::Empty { .. } => {}ColumnIndex::Full => self.values.get_vals_opt(docids, output),- ColumnIndex::Optional(optional_index) => {- for (i, docid) in docids.iter().enumerate() {- output[i] = optional_index- .rank_if_exists(*docid)- .map(|rowid| self.values.get_val(rowid));- }- }- ColumnIndex::Multivalued(multivalued_index) => {- for (i, docid) in docids.iter().enumerate() {- let range = multivalued_index.range(*docid);- let is_empty = range.start == range.end;- if !is_empty {- output[i] = Some(self.values.get_val(range.start));- }- }- }}}@@ -130,12 +89,6 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {self.index.docids_to_rowids(doc_ids, doc_ids_out, row_ids)}- pub fn values_for_doc(&self, doc_id: DocId) -> impl Iterator<Item = T> + '_ {- self.index- .value_row_ids(doc_id)- .map(|value_row_id: RowId| self.values.get_val(value_row_id))- }-/// Get the docids of values which are in the provided value range.#[inline]pub fn get_docids_for_value_range(@@ -152,25 +105,6 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {// Load rowsself.values.get_row_ids_for_value_range(value_range, rowid_range, doc_ids);- // Convert rows to docids- self.index- .select_batch_in_place(selected_docid_range.start, doc_ids);- }-- /// Fills the output vector with the (possibly multiple values that are associated_with- /// `row_id`.- ///- /// This method clears the `output` vector.- pub fn fill_vals(&self, row_id: RowId, output: &mut Vec<T>) {- output.clear();- output.extend(self.values_for_doc(row_id));- }-- pub fn first_or_default_col(self, default_value: T) -> Arc<dyn ColumnValues<T>> {- Arc::new(FirstValueWithDefault {- column: self,- default_value,- })}}@@ -185,35 +119,3 @@ impl BinarySerializable for Cardinality {Ok(cardinality)}}--// TODO simplify or optimize-struct FirstValueWithDefault<T: Copy> {- column: Column<T>,- default_value: T,-}--impl<T: PartialOrd + Debug + Send + Sync + Copy + 'static> ColumnValues<T>- for FirstValueWithDefault<T>-{- #[inline(always)]- fn get_val(&self, idx: u32) -> T {- self.column.first(idx).unwrap_or(self.default_value)- }-- fn min_value(&self) -> T {- self.column.values.min_value()- }-- fn max_value(&self) -> T {- self.column.values.max_value()- }-- fn num_vals(&self) -> u32 {- match &self.column.index {- ColumnIndex::Empty { .. } => 0u32,- ColumnIndex::Full => self.column.values.num_vals(),- ColumnIndex::Optional(optional_idx) => optional_idx.num_docs(),- ColumnIndex::Multivalued(multivalue_idx) => multivalue_idx.num_docs(),- }- }-}diff --git a/crates/tantivy/src/columnar/column/serialize.rs b/crates/tantivy/src/columnar/column/serialize.rsindex eca3fb08..211432d7 100644--- a/crates/tantivy/src/columnar/column/serialize.rs+++ b/crates/tantivy/src/columnar/column/serialize.rs@@ -5,28 +5,16 @@ use std::sync::Arc;use crate::common::OwnedBytes;use crate::sstable::Dictionary;-use super::StrColumn;use crate::columnar::column::{BytesColumn, Column};use crate::columnar::column_index::{serialize_column_index, SerializableColumnIndex};use crate::columnar::column_values::{- load_u64_based_column_values, serialize_column_values_u128, serialize_u64_based_column_values,- CodecType, MonotonicallyMappableToU128, MonotonicallyMappableToU64,+ load_u64_based_column_values, serialize_u64_based_column_values, CodecType,+ MonotonicallyMappableToU64,};use crate::columnar::iterable::Iterable;-pub fn serialize_column_mappable_to_u128<T: MonotonicallyMappableToU128>(- column_index: SerializableColumnIndex<'_>,- iterable: &dyn Iterable<T>,- output: &mut impl Write,-) -> io::Result<()> {- let column_index_num_bytes = serialize_column_index(column_index, output)?;- serialize_column_values_u128(iterable, output)?;- output.write_all(&column_index_num_bytes.to_le_bytes())?;- Ok(())-}-pub fn serialize_column_mappable_to_u64<T: MonotonicallyMappableToU64>(- column_index: SerializableColumnIndex<'_>,+ column_index: SerializableColumnIndex,column_values: &impl Iterable<T>,output: &mut impl Write,) -> io::Result<()> {@@ -57,46 +45,6 @@ pub fn open_column_u64<T: MonotonicallyMappableToU64>(bytes: OwnedBytes) -> io::})}-pub fn open_column_u128<T: MonotonicallyMappableToU128>(- bytes: OwnedBytes,-) -> io::Result<Column<T>> {- let (body, column_index_num_bytes_payload) = bytes.rsplit(4);- let column_index_num_bytes = u32::from_le_bytes(- column_index_num_bytes_payload- .as_slice()- .try_into()- .unwrap(),- );- let (column_index_data, column_values_data) = body.split(column_index_num_bytes as usize);- let column_index = crate::columnar::column_index::open_column_index(column_index_data)?;- let column_values = crate::columnar::column_values::open_u128_mapped(column_values_data)?;- Ok(Column {- index: column_index,- values: column_values,- })-}--/// Open the column as u64.-///-/// See [`open_u128_as_compact_u64`] for more details.-pub fn open_column_u128_as_compact_u64(bytes: OwnedBytes) -> io::Result<Column<u64>> {- let (body, column_index_num_bytes_payload) = bytes.rsplit(4);- let column_index_num_bytes = u32::from_le_bytes(- column_index_num_bytes_payload- .as_slice()- .try_into()- .unwrap(),- );- let (column_index_data, column_values_data) = body.split(column_index_num_bytes as usize);- let column_index = crate::columnar::column_index::open_column_index(column_index_data)?;- let column_values =- crate::columnar::column_values::open_u128_as_compact_u64(column_values_data)?;- Ok(Column {- index: column_index,- values: column_values,- })-}-pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {let (body, dictionary_len_bytes) = data.rsplit(8);let dictionary_len = u64::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());@@ -108,8 +56,3 @@ pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {term_ord_column,})}--pub fn open_column_str(data: OwnedBytes) -> io::Result<StrColumn> {- let bytes_column = open_column_bytes(data)?;- Ok(StrColumn::wrap(bytes_column))-}diff --git a/crates/tantivy/src/columnar/column_index/merge/mod.rs b/crates/tantivy/src/columnar/column_index/merge/mod.rsindex 22149951..199f3107 100644--- a/crates/tantivy/src/columnar/column_index/merge/mod.rs+++ b/crates/tantivy/src/columnar/column_index/merge/mod.rs@@ -1,89 +1,17 @@mod shuffled;mod stacked;-use crate::common::ReadOnlyBitSet;use shuffled::merge_column_index_shuffled;use stacked::merge_column_index_stacked;use crate::columnar::column_index::SerializableColumnIndex;-use crate::columnar::{Cardinality, ColumnIndex, MergeRowOrder};+use crate::columnar::{Cardinality, MergeRowOrder};-fn detect_cardinality_single_column_index(- column_index: &ColumnIndex,- alive_bitset_opt: &Option<ReadOnlyBitSet>,-) -> Cardinality {- let Some(alive_bitset) = alive_bitset_opt else {- return column_index.get_cardinality();- };- let cardinality_before_deletes = column_index.get_cardinality();- if cardinality_before_deletes == Cardinality::Full {- // The columnar cardinality can only become more restrictive in the presence of deletes- // (where cardinality sorted from the more restrictive to the least restrictive are Full,- // Optional, Multivalued)- //- // If we are already "Full", we are guaranteed to stay "Full" after deletes.- return Cardinality::Full;- }- let mut cardinality_so_far = Cardinality::Full;- for doc_id in alive_bitset.iter() {- let num_values = column_index.value_row_ids(doc_id).len();- let row_cardinality = match num_values {- 0 => Cardinality::Optional,- 1 => Cardinality::Full,- _ => Cardinality::Multivalued,- };- cardinality_so_far = cardinality_so_far.max(row_cardinality);- if cardinality_so_far >= cardinality_before_deletes {- // There won't be any improvement in the cardinality.- // We can early exit.- return cardinality_before_deletes;- }- }- cardinality_so_far-}--fn detect_cardinality(- column_indexes: &[ColumnIndex],- merge_row_order: &MergeRowOrder,-) -> Cardinality {- match merge_row_order {- MergeRowOrder::Stack(_) => column_indexes- .iter()- .map(ColumnIndex::get_cardinality)- .max()- .unwrap_or(Cardinality::Full),- MergeRowOrder::Shuffled(shuffle_merge_order) => {- let mut merged_cardinality = Cardinality::Full;- for (column_index, alive_bitset_opt) in column_indexes- .iter()- .zip(shuffle_merge_order.alive_bitsets.iter())- {- let cardinality: Cardinality =- detect_cardinality_single_column_index(column_index, alive_bitset_opt);- if cardinality == Cardinality::Multivalued {- return cardinality;- }- merged_cardinality = merged_cardinality.max(cardinality);- }- merged_cardinality- }- }-}--pub fn merge_column_index<'a>(- columns: &'a [ColumnIndex],- merge_row_order: &'a MergeRowOrder,-) -> SerializableColumnIndex<'a> {- // For simplification, we do not try to detect whether the cardinality could be- // downgraded thanks to deletes.- let cardinality_after_merge = detect_cardinality(columns, merge_row_order);+pub fn merge_column_index(merge_row_order: &MergeRowOrder) -> SerializableColumnIndex {+ let cardinality_after_merge = Cardinality::Full;match merge_row_order {- MergeRowOrder::Stack(stack_merge_order) => {- merge_column_index_stacked(columns, cardinality_after_merge, stack_merge_order)- }- MergeRowOrder::Shuffled(complex_merge_order) => {- merge_column_index_shuffled(columns, cardinality_after_merge, complex_merge_order)- }+ MergeRowOrder::Stack(_) => merge_column_index_stacked(cardinality_after_merge),+ MergeRowOrder::Shuffled(_) => merge_column_index_shuffled(cardinality_after_merge),}}@@ -92,121 +20,3 @@ pub fn merge_column_index<'a>(// The merge order restricted to a specific column keeps the original row order.//// This may offer some optimization that we have not explored yet.--#[cfg(test)]-mod tests {- use crate::columnar::column_index::merge::detect_cardinality;- use crate::columnar::column_index::multivalued_index::MultiValueIndex;- use crate::columnar::column_index::{- merge_column_index, OptionalIndex, SerializableColumnIndex,- };- use crate::columnar::{- Cardinality, ColumnIndex, MergeRowOrder, RowAddr, RowId, ShuffleMergeOrder, StackMergeOrder,- };-- #[test]- fn test_detect_cardinality() {- assert_eq!(- detect_cardinality(&[], &StackMergeOrder::stack_for_test(&[]).into()),- Cardinality::Full- );- let optional_index: ColumnIndex = OptionalIndex::for_test(1, &[]).into();- let multivalued_index: ColumnIndex = MultiValueIndex::for_test(&[0, 1]).into();- assert_eq!(- detect_cardinality(- &[optional_index.clone(), ColumnIndex::Empty { num_docs: 0 }],- &StackMergeOrder::stack_for_test(&[1, 0]).into()- ),- Cardinality::Optional- );- assert_eq!(- detect_cardinality(- &[optional_index.clone(), ColumnIndex::Full],- &StackMergeOrder::stack_for_test(&[1, 1]).into()- ),- Cardinality::Optional- );- assert_eq!(- detect_cardinality(- &[- multivalued_index.clone(),- ColumnIndex::Empty { num_docs: 0 }- ],- &StackMergeOrder::stack_for_test(&[1, 0]).into()- ),- Cardinality::Multivalued- );- assert_eq!(- detect_cardinality(- &[multivalued_index.clone(), optional_index.clone()],- &StackMergeOrder::stack_for_test(&[1, 1]).into()- ),- Cardinality::Multivalued- );- assert_eq!(- detect_cardinality(- &[optional_index, multivalued_index],- &StackMergeOrder::stack_for_test(&[1, 1]).into()- ),- Cardinality::Multivalued- );- }-- #[test]- fn test_merge_index_multivalued_sorted() {- let column_indexes: Vec<ColumnIndex> = vec![MultiValueIndex::for_test(&[0, 2, 5]).into()];- let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(- &[2],- vec![- RowAddr {- segment_ord: 0u32,- row_id: 1u32,- },- RowAddr {- segment_ord: 0u32,- row_id: 0u32,- },- ],- )- .into();- let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);- let SerializableColumnIndex::Multivalued(start_index_iterable) = merged_column_index else {- panic!("Excpected a multivalued index")- };- let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();- assert_eq!(&start_indexes, &[0, 3, 5]);- }-- #[test]- fn test_merge_index_multivalued_sorted_several_segment() {- let column_indexes: Vec<ColumnIndex> = vec![- MultiValueIndex::for_test(&[0, 2, 5]).into(),- ColumnIndex::Empty { num_docs: 0 },- MultiValueIndex::for_test(&[0, 1, 4]).into(),- ];- let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(- &[2, 0, 2],- vec![- RowAddr {- segment_ord: 2u32,- row_id: 1u32,- },- RowAddr {- segment_ord: 0u32,- row_id: 0u32,- },- RowAddr {- segment_ord: 2u32,- row_id: 0u32,- },- ],- )- .into();- let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);- let SerializableColumnIndex::Multivalued(start_index_iterable) = merged_column_index else {- panic!("Excpected a multivalued index")- };- let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();- assert_eq!(&start_indexes, &[0, 3, 5, 6]);- }-}diff --git a/crates/tantivy/src/columnar/column_index/merge/shuffled.rs b/crates/tantivy/src/columnar/column_index/merge/shuffled.rsindex 4d9c96c0..f84f0a9d 100644--- a/crates/tantivy/src/columnar/column_index/merge/shuffled.rs+++ b/crates/tantivy/src/columnar/column_index/merge/shuffled.rs@@ -1,171 +1,10 @@-use std::iter;+use crate::columnar::column_index::SerializableColumnIndex;+use crate::columnar::Cardinality;-use crate::columnar::column_index::{SerializableColumnIndex, Set};-use crate::columnar::iterable::Iterable;-use crate::columnar::{Cardinality, ColumnIndex, RowId, ShuffleMergeOrder};--pub fn merge_column_index_shuffled<'a>(- column_indexes: &'a [ColumnIndex],+pub fn merge_column_index_shuffled(cardinality_after_merge: Cardinality,- shuffle_merge_order: &'a ShuffleMergeOrder,-) -> SerializableColumnIndex<'a> {+) -> SerializableColumnIndex {match cardinality_after_merge {Cardinality::Full => SerializableColumnIndex::Full,- Cardinality::Optional => {- let non_null_row_ids =- merge_column_index_shuffled_optional(column_indexes, shuffle_merge_order);- SerializableColumnIndex::Optional {- non_null_row_ids,- num_rows: shuffle_merge_order.num_rows(),- }- }- Cardinality::Multivalued => {- let multivalue_start_index =- merge_column_index_shuffled_multivalued(column_indexes, shuffle_merge_order);- SerializableColumnIndex::Multivalued(multivalue_start_index)- }- }-}--/// Merge several column indexes into one, ordering rows according to the merge_order passed as-/// argument. While it is true that the `merge_order` may imply deletes and hence could in theory a-/// multivalued index into an optional one, this is not supported today for simplification.-///-/// In other words the column_indexes passed as argument may NOT be multivalued.-fn merge_column_index_shuffled_optional<'a>(- column_indexes: &'a [ColumnIndex],- merge_order: &'a ShuffleMergeOrder,-) -> Box<dyn Iterable<RowId> + 'a> {- Box::new(ShuffledIndex {- column_indexes,- merge_order,- })-}--struct ShuffledIndex<'a> {- column_indexes: &'a [ColumnIndex],- merge_order: &'a ShuffleMergeOrder,-}--impl<'a> Iterable<u32> for ShuffledIndex<'a> {- fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {- Box::new(- self.merge_order- .iter_new_to_old_row_addrs()- .enumerate()- .filter_map(|(new_row_id, old_row_addr)| {- let column_index = &self.column_indexes[old_row_addr.segment_ord as usize];- let row_id = new_row_id as u32;- if column_index.has_value(old_row_addr.row_id) {- Some(row_id)- } else {- None- }- }),- )- }-}--fn merge_column_index_shuffled_multivalued<'a>(- column_indexes: &'a [ColumnIndex],- merge_order: &'a ShuffleMergeOrder,-) -> Box<dyn Iterable<RowId> + 'a> {- Box::new(ShuffledMultivaluedIndex {- column_indexes,- merge_order,- })-}--struct ShuffledMultivaluedIndex<'a> {- column_indexes: &'a [ColumnIndex],- merge_order: &'a ShuffleMergeOrder,-}--fn iter_num_values<'a>(- column_indexes: &'a [ColumnIndex],- merge_order: &'a ShuffleMergeOrder,-) -> impl Iterator<Item = u32> + 'a {- merge_order.iter_new_to_old_row_addrs().map(|row_addr| {- let column_index = &column_indexes[row_addr.segment_ord as usize];- match column_index {- ColumnIndex::Empty { .. } => 0u32,- ColumnIndex::Full => 1,- ColumnIndex::Optional(optional_index) => {- u32::from(optional_index.contains(row_addr.row_id))- }- ColumnIndex::Multivalued(multivalued_index) => {- multivalued_index.range(row_addr.row_id).len() as u32- }- }- })-}--/// Transforms an iterator containing the number of vals per row (with `num_rows` elements)-/// into a `start_offset` iterator starting at 0 and (with `num_rows + 1` element)-fn integrate_num_vals(num_vals: impl Iterator<Item = u32>) -> impl Iterator<Item = RowId> {- iter::once(0u32).chain(num_vals.scan(0, |state, num_vals| {- *state += num_vals;- Some(*state)- }))-}--impl<'a> Iterable<u32> for ShuffledMultivaluedIndex<'a> {- fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {- let num_vals_per_row = iter_num_values(self.column_indexes, self.merge_order);- Box::new(integrate_num_vals(num_vals_per_row))- }-}--#[cfg(test)]-mod tests {- use super::*;- use crate::columnar::column_index::OptionalIndex;- use crate::columnar::RowAddr;-- #[test]- fn test_integrate_num_vals_empty() {- assert!(integrate_num_vals(iter::empty()).eq(iter::once(0)));- }-- #[test]- fn test_integrate_num_vals_one_el() {- assert!(integrate_num_vals(iter::once(10)).eq([0, 10].into_iter()));- }-- #[test]- fn test_integrate_num_vals_several() {- assert!(integrate_num_vals([3, 0, 10, 20].into_iter()).eq([0, 3, 3, 13, 33].into_iter()));- }-- #[test]- fn test_merge_column_index_optional_shuffle() {- let optional_index: ColumnIndex = OptionalIndex::for_test(2, &[0]).into();- let column_indexes = [optional_index, ColumnIndex::Full];- let row_addrs = vec![- RowAddr {- segment_ord: 0u32,- row_id: 1u32,- },- RowAddr {- segment_ord: 1u32,- row_id: 0u32,- },- ];- let shuffle_merge_order = ShuffleMergeOrder::for_test(&[2, 1], row_addrs);- let serializable_index = merge_column_index_shuffled(- &column_indexes[..],- Cardinality::Optional,- &shuffle_merge_order,- );- let SerializableColumnIndex::Optional {- non_null_row_ids,- num_rows,- } = serializable_index- else {- panic!()- };- assert_eq!(num_rows, 2);- let non_null_rows: Vec<RowId> = non_null_row_ids.boxed_iter().collect();- assert_eq!(&non_null_rows, &[1]);}}diff --git a/crates/tantivy/src/columnar/column_index/merge/stacked.rs b/crates/tantivy/src/columnar/column_index/merge/stacked.rsindex c727d066..cd1c3131 100644--- a/crates/tantivy/src/columnar/column_index/merge/stacked.rs+++ b/crates/tantivy/src/columnar/column_index/merge/stacked.rs@@ -1,148 +1,12 @@-use std::iter;--use crate::columnar::column_index::{SerializableColumnIndex, Set};-use crate::columnar::iterable::Iterable;-use crate::columnar::{Cardinality, ColumnIndex, RowId, StackMergeOrder};+use crate::columnar::column_index::SerializableColumnIndex;+use crate::columnar::Cardinality;/// Simple case:/// The new mapping just consists in stacking the different column indexes.////// There are no sort nor deletes involved.-pub fn merge_column_index_stacked<'a>(- columns: &'a [ColumnIndex],- cardinality_after_merge: Cardinality,- stack_merge_order: &'a StackMergeOrder,-) -> SerializableColumnIndex<'a> {+pub fn merge_column_index_stacked(cardinality_after_merge: Cardinality) -> SerializableColumnIndex {match cardinality_after_merge {Cardinality::Full => SerializableColumnIndex::Full,- Cardinality::Optional => SerializableColumnIndex::Optional {- non_null_row_ids: Box::new(StackedOptionalIndex {- columns,- stack_merge_order,- }),- num_rows: stack_merge_order.num_rows(),- },- Cardinality::Multivalued => {- let stacked_multivalued_index = StackedMultivaluedIndex {- columns,- stack_merge_order,- };- SerializableColumnIndex::Multivalued(Box::new(stacked_multivalued_index))- }- }-}--struct StackedOptionalIndex<'a> {- columns: &'a [ColumnIndex],- stack_merge_order: &'a StackMergeOrder,-}--impl<'a> Iterable<RowId> for StackedOptionalIndex<'a> {- fn boxed_iter(&self) -> Box<dyn Iterator<Item = RowId> + 'a> {- Box::new(- self.columns- .iter()- .enumerate()- .flat_map(|(columnar_id, column_index_opt)| {- let columnar_row_range = self.stack_merge_order.columnar_range(columnar_id);- let rows_it: Box<dyn Iterator<Item = RowId>> = match column_index_opt {- ColumnIndex::Full => Box::new(columnar_row_range),- ColumnIndex::Optional(optional_index) => Box::new(- optional_index- .iter_rows()- .map(move |row_id: RowId| columnar_row_range.start + row_id),- ),- ColumnIndex::Multivalued(_) => {- panic!("No multivalued index is allowed when stacking column index");- }- ColumnIndex::Empty { .. } => Box::new(std::iter::empty()),- };- rows_it- }),- )- }-}--#[derive(Clone, Copy)]-struct StackedMultivaluedIndex<'a> {- columns: &'a [ColumnIndex],- stack_merge_order: &'a StackMergeOrder,-}--fn convert_column_opt_to_multivalued_index<'a>(- column_index_opt: &'a ColumnIndex,- num_rows: RowId,-) -> Box<dyn Iterator<Item = RowId> + 'a> {- match column_index_opt {- ColumnIndex::Empty { .. } => Box::new(iter::repeat(0u32).take(num_rows as usize + 1)),- ColumnIndex::Full => Box::new(0..num_rows + 1),- ColumnIndex::Optional(optional_index) => {- Box::new(- (0..num_rows)- // TODO optimize- .map(|row_id| optional_index.rank(row_id))- .chain(std::iter::once(optional_index.num_non_nulls())),- )- }- ColumnIndex::Multivalued(multivalued_index) => multivalued_index.start_index_column.iter(),- }-}--impl<'a> Iterable<RowId> for StackedMultivaluedIndex<'a> {- fn boxed_iter(&self) -> Box<dyn Iterator<Item = RowId> + '_> {- let multivalued_indexes =- self.columns- .iter()- .enumerate()- .map(|(columnar_id, column_opt)| {- let num_rows =- self.stack_merge_order.columnar_range(columnar_id).len() as RowId;- convert_column_opt_to_multivalued_index(column_opt, num_rows)- });- stack_multivalued_indexes(multivalued_indexes)- }-}--// Refactor me-fn stack_multivalued_indexes<'a>(- mut multivalued_indexes: impl Iterator<Item = Box<dyn Iterator<Item = RowId> + 'a>> + 'a,-) -> Box<dyn Iterator<Item = RowId> + 'a> {- let mut offset = 0;- let mut last_row_id = 0;- let mut current_it = multivalued_indexes.next();- Box::new(std::iter::from_fn(move || loop {- if let Some(row_id) = current_it.as_mut()?.next() {- last_row_id = offset + row_id;- return Some(last_row_id);- }- offset = last_row_id;- loop {- current_it = multivalued_indexes.next();- if current_it.as_mut()?.next().is_some() {- break;- }- }- }))-}--#[cfg(test)]-mod tests {- use super::RowId;-- fn it<'a>(row_ids: &'a [RowId]) -> Box<dyn Iterator<Item = RowId> + 'a> {- Box::new(row_ids.iter().copied())- }-- #[test]- fn test_stack() {- let columns = [- it(&[0u32, 0u32]),- it(&[0u32, 1u32, 1u32, 4u32]),- it(&[0u32, 3u32, 5u32]),- it(&[0u32, 4u32]),- ]- .into_iter();- let start_offsets: Vec<RowId> = super::stack_multivalued_indexes(columns).collect();- assert_eq!(start_offsets, &[0, 0, 1, 1, 4, 7, 9, 13]);}}diff --git a/crates/tantivy/src/columnar/column_index/mod.rs b/crates/tantivy/src/columnar/column_index/mod.rsindex 24883bcd..7a279735 100644--- a/crates/tantivy/src/columnar/column_index/mod.rs+++ b/crates/tantivy/src/columnar/column_index/mod.rs@@ -14,72 +14,17 @@ pub use merge::merge_column_index;pub use optional_index::{OptionalIndex, Set};pub use serialize::{open_column_index, serialize_column_index, SerializableColumnIndex};-use super::column_index::multivalued_index::MultiValueIndex;use super::{Cardinality, DocId, RowId};#[derive(Clone, Debug)]pub enum ColumnIndex {- Empty {- num_docs: u32,- },Full,- Optional(OptionalIndex),- /// In addition, at index num_rows, an extra value is added- /// containing the overal number of values.- Multivalued(MultiValueIndex),-}--impl From<OptionalIndex> for ColumnIndex {- fn from(optional_index: OptionalIndex) -> ColumnIndex {- ColumnIndex::Optional(optional_index)- }-}--impl From<MultiValueIndex> for ColumnIndex {- fn from(multi_value_index: MultiValueIndex) -> ColumnIndex {- ColumnIndex::Multivalued(multi_value_index)- }}impl ColumnIndex {- /// Returns the cardinality of the column index.- ///- /// By convention, if the column contains no docs, we consider that it is- /// full.- #[inline]- pub fn get_cardinality(&self) -> Cardinality {- match self {- ColumnIndex::Empty { num_docs: 0 } | ColumnIndex::Full => Cardinality::Full,- ColumnIndex::Empty { .. } => Cardinality::Optional,- ColumnIndex::Optional(_) => Cardinality::Optional,- ColumnIndex::Multivalued(_) => Cardinality::Multivalued,- }- }-- /// Returns true if and only if there are at least one value associated to the row.- pub fn has_value(&self, doc_id: DocId) -> bool {- match self {- ColumnIndex::Empty { .. } => false,- ColumnIndex::Full => true,- ColumnIndex::Optional(optional_index) => optional_index.contains(doc_id),- ColumnIndex::Multivalued(multivalued_index) => {- !multivalued_index.range(doc_id).is_empty()- }- }- }-pub fn value_row_ids(&self, doc_id: DocId) -> Range<RowId> {match self {- ColumnIndex::Empty { .. } => 0..0,ColumnIndex::Full => doc_id..doc_id + 1,- ColumnIndex::Optional(optional_index) => {- if let Some(val) = optional_index.rank_if_exists(doc_id) {- val..val + 1- } else {- 0..0- }- }- ColumnIndex::Multivalued(multivalued_index) => multivalued_index.range(doc_id),}}@@ -98,84 +43,16 @@ impl ColumnIndex {row_ids: &mut Vec<RowId>,) {match self {- ColumnIndex::Empty { .. } => {}ColumnIndex::Full => {doc_ids_out.extend_from_slice(doc_ids);row_ids.extend_from_slice(doc_ids);}- ColumnIndex::Optional(optional_index) => {- for doc_id in doc_ids {- if let Some(row_id) = optional_index.rank_if_exists(*doc_id) {- doc_ids_out.push(*doc_id);- row_ids.push(row_id);- }- }- }- ColumnIndex::Multivalued(multivalued_index) => {- for doc_id in doc_ids {- for row_id in multivalued_index.range(*doc_id) {- doc_ids_out.push(*doc_id);- row_ids.push(row_id);- }- }- }}}pub fn docid_range_to_rowids(&self, doc_id_range: Range<DocId>) -> Range<RowId> {match self {- ColumnIndex::Empty { .. } => 0..0,ColumnIndex::Full => doc_id_range,- ColumnIndex::Optional(optional_index) => {- let row_start = optional_index.rank(doc_id_range.start);- let row_end = optional_index.rank(doc_id_range.end);- row_start..row_end- }- ColumnIndex::Multivalued(multivalued_index) => {- let end_docid = doc_id_range.end.min(multivalued_index.num_docs() - 1) + 1;- let start_docid = doc_id_range.start.min(end_docid);-- let row_start = multivalued_index.start_index_column.get_val(start_docid);- let row_end = multivalued_index.start_index_column.get_val(end_docid);-- row_start..row_end- }- }- }-- pub fn select_batch_in_place(&self, doc_id_start: DocId, rank_ids: &mut Vec<RowId>) {- match self {- ColumnIndex::Empty { .. } => {- rank_ids.clear();- }- ColumnIndex::Full => {- // No need to do anything:- // value_idx and row_idx are the same.- }- ColumnIndex::Optional(optional_index) => {- optional_index.select_batch(&mut rank_ids[..]);- }- ColumnIndex::Multivalued(multivalued_index) => {- multivalued_index.select_batch_in_place(doc_id_start, rank_ids)- }}}}--#[cfg(test)]-mod tests {- use super::{Cardinality, ColumnIndex};-- #[test]- fn test_column_index_get_cardinality() {- assert_eq!(- ColumnIndex::Empty { num_docs: 0 }.get_cardinality(),- Cardinality::Full- );- assert_eq!(ColumnIndex::Full.get_cardinality(), Cardinality::Full);- assert_eq!(- ColumnIndex::Empty { num_docs: 1 }.get_cardinality(),- Cardinality::Optional- );- }-}diff --git a/crates/tantivy/src/columnar/column_index/multivalued_index.rs b/crates/tantivy/src/columnar/column_index/multivalued_index.rsindex e346d3e1..e8cd8cf5 100644--- a/crates/tantivy/src/columnar/column_index/multivalued_index.rs+++ b/crates/tantivy/src/columnar/column_index/multivalued_index.rs@@ -1,11 +1,10 @@use std::io;use std::io::Write;-use std::ops::Range;use std::sync::Arc;use crate::common::OwnedBytes;-use super::{DocId, RowId};+use super::RowId;use crate::columnar::column_values::{load_u64_based_column_values, serialize_u64_based_column_values, CodecType, ColumnValues,};@@ -57,88 +56,9 @@ impl MultiValueIndex {open_multivalued_index(bytes).unwrap()}- /// Returns `[start, end)`, such that the values associated with- /// the given document are `start..end`.- #[inline]- pub(crate) fn range(&self, doc_id: DocId) -> Range<RowId> {- let start = self.start_index_column.get_val(doc_id);- let end = self.start_index_column.get_val(doc_id + 1);- start..end- }-/// Returns the number of documents in the index.#[inline]pub fn num_docs(&self) -> u32 {self.start_index_column.num_vals() - 1}-- /// Converts a list of ranks (row ids of values) in a 1:n index to the corresponding list of- /// docids. Positions are converted inplace to docids.- ///- /// Since there is no index for value pos -> docid, but docid -> value pos range, we scan the- /// index.- ///- /// Correctness: positions needs to be sorted. idx_reader needs to contain monotonically- /// increasing positions.- ///- /// TODO: Instead of a linear scan we can employ a exponential search into binary search to- /// match a docid to its value position.- #[allow(clippy::bool_to_int_with_if)]- pub(crate) fn select_batch_in_place(&self, docid_start: DocId, ranks: &mut Vec<u32>) {- if ranks.is_empty() {- return;- }- let mut cur_doc = docid_start;- let mut last_doc = None;-- assert!(self.start_index_column.get_val(docid_start) <= ranks[0]);-- let mut write_doc_pos = 0;- for i in 0..ranks.len() {- let pos = ranks[i];- loop {- let end = self.start_index_column.get_val(cur_doc + 1);- if end > pos {- ranks[write_doc_pos] = cur_doc;- write_doc_pos += if last_doc == Some(cur_doc) { 0 } else { 1 };- last_doc = Some(cur_doc);- break;- }- cur_doc += 1;- }- }- ranks.truncate(write_doc_pos);- }-}--#[cfg(test)]-mod tests {- use std::ops::Range;-- use super::MultiValueIndex;-- fn index_to_pos_helper(- index: &MultiValueIndex,- doc_id_range: Range<u32>,- positions: &[u32],- ) -> Vec<u32> {- let mut positions = positions.to_vec();- index.select_batch_in_place(doc_id_range.start, &mut positions);- positions- }-- #[test]- fn test_positions_to_docid() {- let index = MultiValueIndex::for_test(&[0, 10, 12, 15, 22, 23]);- assert_eq!(index.num_docs(), 5);- let positions = &[10u32, 11, 15, 20, 21, 22];- assert_eq!(index_to_pos_helper(&index, 0..5, positions), vec![1, 3, 4]);- assert_eq!(index_to_pos_helper(&index, 1..5, positions), vec![1, 3, 4]);- assert_eq!(index_to_pos_helper(&index, 0..5, &[9]), vec![0]);- assert_eq!(index_to_pos_helper(&index, 1..5, &[10]), vec![1]);- assert_eq!(index_to_pos_helper(&index, 1..5, &[11]), vec![1]);- assert_eq!(index_to_pos_helper(&index, 2..5, &[12]), vec![2]);- assert_eq!(index_to_pos_helper(&index, 2..5, &[12, 14]), vec![2]);- assert_eq!(index_to_pos_helper(&index, 2..5, &[12, 14, 15]), vec![2, 3]);- }}diff --git a/crates/tantivy/src/columnar/column_index/serialize.rs b/crates/tantivy/src/columnar/column_index/serialize.rsindex 88e2d6a8..9a4501e7 100644--- a/crates/tantivy/src/columnar/column_index/serialize.rs+++ b/crates/tantivy/src/columnar/column_index/serialize.rs@@ -3,29 +3,17 @@ use std::io::Write;use crate::common::{CountingWriter, OwnedBytes};-use super::{Cardinality, RowId};-use crate::columnar::column_index::multivalued_index::serialize_multivalued_index;-use crate::columnar::column_index::optional_index::serialize_optional_index;+use super::Cardinality;use crate::columnar::column_index::ColumnIndex;-use crate::columnar::iterable::Iterable;-pub enum SerializableColumnIndex<'a> {+pub enum SerializableColumnIndex {Full,- Optional {- non_null_row_ids: Box<dyn Iterable<RowId> + 'a>,- num_rows: RowId,- },- // TODO remove the Arc<dyn> apart from serialization this is not- // dynamic at all.- Multivalued(Box<dyn Iterable<RowId> + 'a>),}-impl<'a> SerializableColumnIndex<'a> {+impl SerializableColumnIndex {pub fn get_cardinality(&self) -> Cardinality {match self {SerializableColumnIndex::Full => Cardinality::Full,- SerializableColumnIndex::Optional { .. } => Cardinality::Optional,- SerializableColumnIndex::Multivalued(_) => Cardinality::Multivalued,}}}@@ -40,13 +28,6 @@ pub fn serialize_column_index(output.write_all(&[cardinality])?;match column_index {SerializableColumnIndex::Full => {}- SerializableColumnIndex::Optional {- non_null_row_ids,- num_rows,- } => serialize_optional_index(non_null_row_ids.as_ref(), num_rows, &mut output)?,- SerializableColumnIndex::Multivalued(multivalued_index) => {- serialize_multivalued_index(&*multivalued_index, &mut output)?- }}let column_index_num_bytes = output.written_bytes() as u32;Ok(column_index_num_bytes)@@ -65,14 +46,6 @@ pub fn open_column_index(mut bytes: OwnedBytes) -> io::Result<ColumnIndex> {bytes.advance(1);match cardinality {Cardinality::Full => Ok(ColumnIndex::Full),- Cardinality::Optional => {- let optional_index = super::optional_index::open_optional_index(bytes)?;- Ok(ColumnIndex::Optional(optional_index))- }- Cardinality::Multivalued => {- let multivalue_index = super::multivalued_index::open_multivalued_index(bytes)?;- Ok(ColumnIndex::Multivalued(multivalue_index))- }}}diff --git a/crates/tantivy/src/columnar/column_values/mod.rs b/crates/tantivy/src/columnar/column_values/mod.rsindex ea214924..39261688 100644--- a/crates/tantivy/src/columnar/column_values/mod.rs+++ b/crates/tantivy/src/columnar/column_values/mod.rs@@ -18,7 +18,6 @@ mod merge;pub(crate) mod monotonic_mapping;pub(crate) mod monotonic_mapping_u128;mod stats;-mod u128_based;mod u64_based;mod vec_column;@@ -26,10 +25,6 @@ mod monotonic_column;pub(crate) use merge::MergedColumnValues;pub use stats::ColumnStats;-pub use u128_based::{- open_u128_as_compact_u64, open_u128_mapped, serialize_column_values_u128,- CompactSpaceU64Accessor,-};pub use u64_based::{load_u64_based_column_values, serialize_and_load_u64_based_column_values,serialize_u64_based_column_values, CodecType, ALL_U64_CODEC_TYPES,diff --git a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/blank_range.rs b/crates/tantivy/src/columnar/column_values/u128_based/compact_space/blank_range.rsdeleted file mode 100644index 4d1ceefd..00000000--- a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/blank_range.rs+++ /dev/null@@ -1,43 +0,0 @@-use std::ops::RangeInclusive;--/// The range of a blank in value space.-///-/// A blank is an unoccupied space in the data.-/// Use try_into() to construct.-/// A range has to have at least length of 3. Invalid ranges will be rejected.-///-/// Ordered by range length.-#[derive(Debug, Eq, PartialEq, Clone)]-pub(crate) struct BlankRange {- blank_range: RangeInclusive<u128>,-}-impl TryFrom<RangeInclusive<u128>> for BlankRange {- type Error = &'static str;- fn try_from(range: RangeInclusive<u128>) -> Result<Self, Self::Error> {- let blank_size = range.end().saturating_sub(*range.start());- if blank_size < 2 {- Err("invalid range")- } else {- Ok(BlankRange { blank_range: range })- }- }-}-impl BlankRange {- pub(crate) fn blank_size(&self) -> u128 {- self.blank_range.end() - self.blank_range.start() + 1- }- pub(crate) fn blank_range(&self) -> RangeInclusive<u128> {- self.blank_range.clone()- }-}--impl Ord for BlankRange {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.blank_size().cmp(&other.blank_size())- }-}-impl PartialOrd for BlankRange {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}diff --git a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/build_compact_space.rs b/crates/tantivy/src/columnar/column_values/u128_based/compact_space/build_compact_space.rsdeleted file mode 100644index a2a13645..00000000--- a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/build_compact_space.rs+++ /dev/null@@ -1,252 +0,0 @@-use std::collections::{BTreeSet, BinaryHeap};-use std::iter;-use std::ops::RangeInclusive;--use itertools::Itertools;--use super::blank_range::BlankRange;-use super::{CompactSpace, RangeMapping};--/// Put the blanks for the sorted values into a binary heap-fn get_blanks(values_sorted: &BTreeSet<u128>) -> BinaryHeap<BlankRange> {- let mut blanks: BinaryHeap<BlankRange> = BinaryHeap::new();- for (first, second) in values_sorted.iter().copied().tuple_windows() {- // Correctness Overflow: the values are deduped and sorted (BTreeSet property), that means- // there's always space between two values.- let blank_range = first + 1..=second - 1;- let blank_range: Result<BlankRange, _> = blank_range.try_into();- if let Ok(blank_range) = blank_range {- blanks.push(blank_range);- }- }-- blanks-}--struct BlankCollector {- blanks: Vec<BlankRange>,- staged_blanks_sum: u128,-}-impl BlankCollector {- fn new() -> Self {- Self {- blanks: vec![],- staged_blanks_sum: 0,- }- }- fn stage_blank(&mut self, blank: BlankRange) {- self.staged_blanks_sum += blank.blank_size();- self.blanks.push(blank);- }- fn drain(&mut self) -> impl Iterator<Item = BlankRange> + '_ {- self.staged_blanks_sum = 0;- self.blanks.drain(..)- }- fn staged_blanks_sum(&self) -> u128 {- self.staged_blanks_sum- }- fn num_staged_blanks(&self) -> usize {- self.blanks.len()- }-}-fn num_bits(val: u128) -> u8 {- (128u32 - val.leading_zeros()) as u8-}--/// Will collect blanks and add them to compact space if more bits are saved than cost from-/// metadata.-pub fn get_compact_space(- values_deduped_sorted: &BTreeSet<u128>,- total_num_values: u32,- cost_per_blank: usize,-) -> CompactSpace {- let mut compact_space_builder = CompactSpaceBuilder::new();- if values_deduped_sorted.is_empty() {- return compact_space_builder.finish();- }-- // We start by space that's limited to min_value..=max_value- // Replace after stabilization of https://github.com/rust-lang/rust/issues/62924- let min_value = values_deduped_sorted.iter().next().copied().unwrap_or(0);- let max_value = values_deduped_sorted.iter().last().copied().unwrap_or(0);-- let mut blanks: BinaryHeap<BlankRange> = get_blanks(values_deduped_sorted);-- // +1 for null, in case min and max covers the whole space, we are off by one.- let mut amplitude_compact_space = (max_value - min_value).saturating_add(1);- if min_value != 0 {- compact_space_builder.add_blanks(iter::once(0..=min_value - 1));- }- if max_value != u128::MAX {- compact_space_builder.add_blanks(iter::once(max_value + 1..=u128::MAX));- }-- let mut amplitude_bits: u8 = num_bits(amplitude_compact_space);-- let mut blank_collector = BlankCollector::new();-- // We will stage blanks until they reduce the compact space by at least 1 bit and then flush- // them if the metadata cost is lower than the total number of saved bits.- // Binary heap to process the gaps by their size- while let Some(blank_range) = blanks.pop() {- blank_collector.stage_blank(blank_range);-- let staged_spaces_sum: u128 = blank_collector.staged_blanks_sum();- let amplitude_new_compact_space = amplitude_compact_space - staged_spaces_sum;- let amplitude_new_bits = num_bits(amplitude_new_compact_space);-- if amplitude_bits == amplitude_new_bits {- continue;- }- let saved_bits = (amplitude_bits - amplitude_new_bits) as usize * total_num_values as usize;- // TODO: Maybe calculate exact cost of blanks and run this more expensive computation only,- // when amplitude_new_bits changes- let cost = blank_collector.num_staged_blanks() * cost_per_blank;-- // We want to end up with a compact space that fits into 32 bits.- // In order to deal with pathological cases, we force the algorithm to keep- // refining the compact space the amplitude bits is lower than 32.- //- // The worst case scenario happens for a large number of u128s regularly- // spread over the full u128 space.- //- // This change will force the algorithm to degenerate into dictionary encoding.- if amplitude_bits <= 32 && cost >= saved_bits {- // Continue here, since although we walk over the blanks by size,- // we can potentially save a lot at the last bits, which are smaller blanks- //- // E.g. if the first range reduces the compact space by 1000 from 2000 to 1000, which- // saves 11-10=1 bit and the next range reduces the compact space by 950 to- // 50, which saves 10-6=4 bit- continue;- }-- amplitude_compact_space = amplitude_new_compact_space;- amplitude_bits = amplitude_new_bits;- compact_space_builder.add_blanks(blank_collector.drain().map(|blank| blank.blank_range()));- }-- assert!(amplitude_bits <= 32);-- // special case, when we don't collected any blanks because:- // * the data is empty (early exit)- // * the algorithm did decide it's not worth the cost, which can be the case for single values- //- // We drain one collected blank unconditionally, so the empty case is reserved for empty- // data, and therefore empty compact_space means the data is empty and no data is covered- // (conversely to all data) and we can assign null to it.- if compact_space_builder.is_empty() {- compact_space_builder.add_blanks(- blank_collector- .drain()- .map(|blank| blank.blank_range())- .take(1),- );- }-- let compact_space = compact_space_builder.finish();- if max_value - min_value != u128::MAX {- debug_assert_eq!(- compact_space.amplitude_compact_space(),- amplitude_compact_space- );- }- compact_space-}--#[derive(Debug, Clone, Eq, PartialEq)]-struct CompactSpaceBuilder {- blanks: Vec<RangeInclusive<u128>>,-}--impl CompactSpaceBuilder {- /// Creates a new compact space builder which will initially cover the whole space.- fn new() -> Self {- Self { blanks: Vec::new() }- }-- /// Assumes that repeated add_blank calls don't overlap and are not adjacent,- /// e.g. [3..=5, 5..=10] is not allowed- ///- /// Both of those assumptions are true when blanks are produced from sorted values.- fn add_blanks(&mut self, blank: impl Iterator<Item = RangeInclusive<u128>>) {- self.blanks.extend(blank);- }-- fn is_empty(&self) -> bool {- self.blanks.is_empty()- }-- /// Convert blanks to covered space and assign null value- fn finish(mut self) -> CompactSpace {- // sort by start. ranges are not allowed to overlap- self.blanks.sort_unstable_by_key(|blank| *blank.start());-- let mut covered_space = Vec::with_capacity(self.blanks.len());-- // begining of the blanks- if let Some(first_blank_start) = self.blanks.first().map(RangeInclusive::start) {- if *first_blank_start != 0 {- covered_space.push(0..=first_blank_start - 1);- }- }-- // Between the blanks- let between_blanks = self.blanks.iter().tuple_windows().map(|(left, right)| {- assert!(- left.end() < right.start(),- "overlapping or adjacent ranges detected"- );- *left.end() + 1..=*right.start() - 1- });- covered_space.extend(between_blanks);-- // end of the blanks- if let Some(last_blank_end) = self.blanks.last().map(RangeInclusive::end) {- if *last_blank_end != u128::MAX {- covered_space.push(last_blank_end + 1..=u128::MAX);- }- }-- if covered_space.is_empty() {- covered_space.push(0..=0); // empty data case- };-- let mut compact_start: u32 = 1; // 0 is reserved for `null`- let mut ranges_mapping: Vec<RangeMapping> = Vec::with_capacity(covered_space.len());- for cov in covered_space {- let range_mapping = super::RangeMapping {- value_range: cov,- compact_start,- };- let covered_range_len = range_mapping.range_length();- ranges_mapping.push(range_mapping);- compact_start += covered_range_len;- }- // println!("num ranges {}", ranges_mapping.len());- CompactSpace { ranges_mapping }- }-}--#[cfg(test)]-mod tests {- use super::*;- use crate::columnar::column_values::u128_based::compact_space::COST_PER_BLANK_IN_BITS;-- #[test]- fn test_binary_heap_pop_order() {- let mut blanks: BinaryHeap<BlankRange> = BinaryHeap::new();- blanks.push((0..=10).try_into().unwrap());- blanks.push((100..=200).try_into().unwrap());- blanks.push((100..=110).try_into().unwrap());- assert_eq!(blanks.pop().unwrap().blank_size(), 101);- assert_eq!(blanks.pop().unwrap().blank_size(), 11);- }-- #[test]- fn test_worst_case_scenario() {- let vals: BTreeSet<u128> = (0..8).map(|i| i * ((1u128 << 34) / 8)).collect();- let compact_space = get_compact_space(&vals, vals.len() as u32, COST_PER_BLANK_IN_BITS);- assert!(compact_space.amplitude_compact_space() < u32::MAX as u128);- }-}diff --git a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/mod.rs b/crates/tantivy/src/columnar/column_values/u128_based/compact_space/mod.rsdeleted file mode 100644index bf7f0138..00000000--- a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/mod.rs+++ /dev/null@@ -1,844 +0,0 @@-/// This codec takes a large number space (u128) and reduces it to a compact number space.-///-/// It will find spaces in the number range. For example:-///-/// 100, 101, 102, 103, 104, 50000, 50001-/// could be mapped to-/// 100..104 -> 0..4-/// 50000..50001 -> 5..6-///-/// Compact space 0..=6 requires much less bits than 100..=50001-///-/// The codec is created to compress ip addresses, but may be employed in other use cases.-use std::{- cmp::Ordering,- collections::BTreeSet,- io::{self, Write},- ops::{Range, RangeInclusive},-};--mod blank_range;-mod build_compact_space;--use crate::bitpacker::{BitPacker, BitUnpacker};-use crate::common::{BinarySerializable, CountingWriter, OwnedBytes, VInt, VIntU128};-use build_compact_space::get_compact_space;--use crate::columnar::column_values::ColumnValues;-use crate::columnar::RowId;--/// The cost per blank is quite hard actually, since blanks are delta encoded, the actual cost of-/// blanks depends on the number of blanks.-///-/// The number is taken by looking at a real dataset. It is optimized for larger datasets.-const COST_PER_BLANK_IN_BITS: usize = 36;--#[derive(Debug, Clone, Eq, PartialEq)]-pub struct CompactSpace {- ranges_mapping: Vec<RangeMapping>,-}--/// Maps the range from the original space to compact_start + range.len()-#[derive(Debug, Clone, Eq, PartialEq)]-struct RangeMapping {- value_range: RangeInclusive<u128>,- compact_start: u32,-}-impl RangeMapping {- fn range_length(&self) -> u32 {- (self.value_range.end() - self.value_range.start()) as u32 + 1- }-- // The last value of the compact space in this range- fn compact_end(&self) -> u32 {- self.compact_start + self.range_length() - 1- }-}--impl BinarySerializable for CompactSpace {- fn serialize<W: io::Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {- VInt(self.ranges_mapping.len() as u64).serialize(writer)?;-- let mut prev_value = 0;- for value_range in self- .ranges_mapping- .iter()- .map(|range_mapping| &range_mapping.value_range)- {- let blank_delta_start = value_range.start() - prev_value;- VIntU128(blank_delta_start).serialize(writer)?;- prev_value = *value_range.start();-- let blank_delta_end = value_range.end() - prev_value;- VIntU128(blank_delta_end).serialize(writer)?;- prev_value = *value_range.end();- }-- Ok(())- }-- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {- let num_ranges = VInt::deserialize(reader)?.0;- let mut ranges_mapping: Vec<RangeMapping> = vec![];- let mut value = 0u128;- let mut compact_start = 1u32; // 0 is reserved for `null`- for _ in 0..num_ranges {- let blank_delta_start = VIntU128::deserialize(reader)?.0;- value += blank_delta_start;- let blank_start = value;-- let blank_delta_end = VIntU128::deserialize(reader)?.0;- value += blank_delta_end;- let blank_end = value;-- let range_mapping = RangeMapping {- value_range: blank_start..=blank_end,- compact_start,- };- let range_length = range_mapping.range_length();- ranges_mapping.push(range_mapping);- compact_start += range_length;- }-- Ok(Self { ranges_mapping })- }-}--impl CompactSpace {- /// Amplitude is the value range of the compact space including the sentinel value used to- /// identify null values. The compact space is 0..=amplitude .- ///- /// It's only used to verify we don't exceed u64 number space, which would indicate a bug.- fn amplitude_compact_space(&self) -> u128 {- self.ranges_mapping- .last()- .map(|last_range| last_range.compact_end() as u128)- .unwrap_or(1) // compact space starts at 1, 0 == null- }-- fn get_range_mapping(&self, pos: usize) -> &RangeMapping {- &self.ranges_mapping[pos]- }-- /// Returns either Ok(the value in the compact space) or if it is outside the compact space the- /// Err(position where it would be inserted)- fn u128_to_compact(&self, value: u128) -> Result<u32, usize> {- self.ranges_mapping- .binary_search_by(|probe| {- let value_range: &RangeInclusive<u128> = &probe.value_range;- if value < *value_range.start() {- Ordering::Greater- } else if value > *value_range.end() {- Ordering::Less- } else {- Ordering::Equal- }- })- .map(|pos| {- let range_mapping = &self.ranges_mapping[pos];- let pos_in_range: u32 = (value - range_mapping.value_range.start()) as u32;- range_mapping.compact_start + pos_in_range- })- }-- /// Unpacks a value from compact space u32 to u128 space- fn compact_to_u128(&self, compact: u32) -> u128 {- let pos = self- .ranges_mapping- .binary_search_by_key(&compact, |range_mapping| range_mapping.compact_start)- // Correctness: Overflow. The first range starts at compact space 0, the error from- // binary search can never be 0- .unwrap_or_else(|e| e - 1);-- let range_mapping = &self.ranges_mapping[pos];- let diff = compact - range_mapping.compact_start;- range_mapping.value_range.start() + diff as u128- }-}--pub struct CompactSpaceCompressor {- params: IPCodecParams,-}--#[derive(Debug, Clone)]-pub struct IPCodecParams {- compact_space: CompactSpace,- bit_unpacker: BitUnpacker,- min_value: u128,- max_value: u128,- num_vals: RowId,- num_bits: u8,-}--impl CompactSpaceCompressor {- pub fn num_vals(&self) -> RowId {- self.params.num_vals- }-- /// Taking the vals as Vec may cost a lot of memory. It is used to sort the vals.- pub fn train_from(iter: impl Iterator<Item = u128>) -> Self {- let mut values_sorted = BTreeSet::new();- // Total number of values, with their redundancy.- let mut total_num_values = 0u32;- for val in iter {- total_num_values += 1u32;- values_sorted.insert(val);- }- let min_value = *values_sorted.iter().next().unwrap_or(&0);- let max_value = *values_sorted.iter().last().unwrap_or(&0);-- let compact_space =- get_compact_space(&values_sorted, total_num_values, COST_PER_BLANK_IN_BITS);- let amplitude_compact_space = compact_space.amplitude_compact_space();-- assert!(- amplitude_compact_space <= u64::MAX as u128,- "case unsupported."- );-- let num_bits = crate::bitpacker::compute_num_bits(amplitude_compact_space as u64);-- assert_eq!(- compact_space- .u128_to_compact(max_value)- .expect("could not convert max value to compact space"),- amplitude_compact_space as u32- );- CompactSpaceCompressor {- params: IPCodecParams {- compact_space,- bit_unpacker: BitUnpacker::new(num_bits),- min_value,- max_value,- num_vals: total_num_values,- num_bits,- },- }- }-- fn write_footer(self, writer: &mut impl Write) -> io::Result<()> {- let writer = &mut CountingWriter::wrap(writer);- self.params.serialize(writer)?;-- let footer_len = writer.written_bytes() as u32;- footer_len.serialize(writer)?;-- Ok(())- }-- pub fn compress_into(- self,- vals: impl Iterator<Item = u128>,- write: &mut impl Write,- ) -> io::Result<()> {- let mut bitpacker = BitPacker::default();- for val in vals {- let compact = self- .params- .compact_space- .u128_to_compact(val)- .map_err(|_| {- io::Error::new(- io::ErrorKind::InvalidData,- "Could not convert value to compact_space. This is a bug.",- )- })?;- bitpacker.write(compact as u64, self.params.num_bits, write)?;- }- bitpacker.close(write)?;- self.write_footer(write)?;- Ok(())- }-}--#[derive(Debug, Clone)]-pub struct CompactSpaceDecompressor {- data: OwnedBytes,- params: IPCodecParams,-}--impl BinarySerializable for IPCodecParams {- fn serialize<W: io::Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {- // header flags for future optional dictionary encoding- let footer_flags = 0u64;- footer_flags.serialize(writer)?;-- VIntU128(self.min_value).serialize(writer)?;- VIntU128(self.max_value).serialize(writer)?;- VIntU128(self.num_vals as u128).serialize(writer)?;- self.num_bits.serialize(writer)?;-- self.compact_space.serialize(writer)?;-- Ok(())- }-- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {- let _header_flags = u64::deserialize(reader)?;- let min_value = VIntU128::deserialize(reader)?.0;- let max_value = VIntU128::deserialize(reader)?.0;- let num_vals = VIntU128::deserialize(reader)?.0 as u32;- let num_bits = u8::deserialize(reader)?;- let compact_space = CompactSpace::deserialize(reader)?;-- Ok(Self {- compact_space,- bit_unpacker: BitUnpacker::new(num_bits),- min_value,- max_value,- num_vals,- num_bits,- })- }-}--/// Exposes the compact space compressed values as u64.-///-/// This allows faster access to the values, as u64 is faster to work with than u128.-/// It also allows to handle u128 values like u64, via the `open_u64_lenient` as a uniform-/// access interface.-///-/// When converting from the internal u64 to u128 `compact_to_u128` can be used.-pub struct CompactSpaceU64Accessor(CompactSpaceDecompressor);-impl CompactSpaceU64Accessor {- pub(crate) fn open(data: OwnedBytes) -> io::Result<CompactSpaceU64Accessor> {- let decompressor = CompactSpaceU64Accessor(CompactSpaceDecompressor::open(data)?);- Ok(decompressor)- }- /// Convert a compact space value to u128- pub fn compact_to_u128(&self, compact: u32) -> u128 {- self.0.compact_to_u128(compact)- }-}--impl ColumnValues<u64> for CompactSpaceU64Accessor {- #[inline]- fn get_val(&self, doc: u32) -> u64 {- let compact = self.0.get_compact(doc);- compact as u64- }-- fn min_value(&self) -> u64 {- self.0.u128_to_compact(self.0.min_value()).unwrap() as u64- }-- fn max_value(&self) -> u64 {- self.0.u128_to_compact(self.0.max_value()).unwrap() as u64- }-- fn num_vals(&self) -> u32 {- self.0.params.num_vals- }-- #[inline]- fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {- Box::new(self.0.iter_compact().map(|el| el as u64))- }-- #[inline]- fn get_row_ids_for_value_range(- &self,- value_range: RangeInclusive<u64>,- position_range: Range<u32>,- positions: &mut Vec<u32>,- ) {- let value_range = self.0.compact_to_u128(*value_range.start() as u32)- ..=self.0.compact_to_u128(*value_range.end() as u32);- self.0- .get_row_ids_for_value_range(value_range, position_range, positions)- }-}--impl ColumnValues<u128> for CompactSpaceDecompressor {- #[inline]- fn get_val(&self, doc: u32) -> u128 {- self.get(doc)- }-- fn min_value(&self) -> u128 {- self.min_value()- }-- fn max_value(&self) -> u128 {- self.max_value()- }-- fn num_vals(&self) -> u32 {- self.params.num_vals- }-- #[inline]- fn iter(&self) -> Box<dyn Iterator<Item = u128> + '_> {- Box::new(self.iter())- }-- #[inline]- fn get_row_ids_for_value_range(- &self,- value_range: RangeInclusive<u128>,- position_range: Range<u32>,- positions: &mut Vec<u32>,- ) {- if value_range.start() > value_range.end() {- return;- }- let position_range = position_range.start..position_range.end.min(self.num_vals());- let from_value = *value_range.start();- let to_value = *value_range.end();- assert!(to_value >= from_value);- let compact_from = self.u128_to_compact(from_value);- let compact_to = self.u128_to_compact(to_value);-- // Quick return, if both ranges fall into the same non-mapped space, the range can't cover- // any values, so we can early exit- match (compact_to, compact_from) {- (Err(pos1), Err(pos2)) if pos1 == pos2 => return,- _ => {}- }-- let compact_from = compact_from.unwrap_or_else(|pos| {- // Correctness: Out of bounds, if this value is Err(last_index + 1), we early exit,- // since the to_value also mapps into the same non-mapped space- let range_mapping = self.params.compact_space.get_range_mapping(pos);- range_mapping.compact_start- });- // If there is no compact space, we go to the closest upperbound compact space- let compact_to = compact_to.unwrap_or_else(|pos| {- // Correctness: Overflow, if this value is Err(0), we early exit,- // since the from_value also mapps into the same non-mapped space-- // Get end of previous range- let pos = pos - 1;- let range_mapping = self.params.compact_space.get_range_mapping(pos);- range_mapping.compact_end()- });-- let value_range = compact_from..=compact_to;- self.get_positions_for_compact_value_range(value_range, position_range, positions);- }-}--impl CompactSpaceDecompressor {- pub fn open(data: OwnedBytes) -> io::Result<CompactSpaceDecompressor> {- let (data_slice, footer_len_bytes) = data.split_at(data.len() - 4);- let footer_len = u32::deserialize(&mut &footer_len_bytes[..])?;-- let data_footer = &data_slice[data_slice.len() - footer_len as usize..];- let params = IPCodecParams::deserialize(&mut &data_footer[..])?;- let decompressor = CompactSpaceDecompressor { data, params };-- Ok(decompressor)- }-- /// Converting to compact space for the decompressor is more complex, since we may get values- /// which are outside the compact space. e.g. if we map- /// 1000 => 5- /// 2000 => 6- ///- /// and we want a mapping for 1005, there is no equivalent compact space. We instead return an- /// error with the index of the next range.- fn u128_to_compact(&self, value: u128) -> Result<u32, usize> {- self.params.compact_space.u128_to_compact(value)- }-- fn compact_to_u128(&self, compact: u32) -> u128 {- self.params.compact_space.compact_to_u128(compact)- }-- #[inline]- fn iter_compact(&self) -> impl Iterator<Item = u32> + '_ {- (0..self.params.num_vals)- .map(move |idx| self.params.bit_unpacker.get(idx, &self.data) as u32)- }-- #[inline]- fn iter(&self) -> impl Iterator<Item = u128> + '_ {- // TODO: Performance. It would be better to iterate on the ranges and check existence via- // the bit_unpacker.- self.iter_compact()- .map(|compact| self.compact_to_u128(compact))- }-- #[inline]- pub fn get_compact(&self, idx: u32) -> u32 {- self.params.bit_unpacker.get(idx, &self.data) as u32- }-- #[inline]- pub fn get(&self, idx: u32) -> u128 {- let compact = self.get_compact(idx);- self.compact_to_u128(compact)- }-- pub fn min_value(&self) -> u128 {- self.params.min_value- }-- pub fn max_value(&self) -> u128 {- self.params.max_value- }-- fn get_positions_for_compact_value_range(- &self,- value_range: RangeInclusive<u32>,- position_range: Range<u32>,- positions: &mut Vec<u32>,- ) {- self.params.bit_unpacker.get_ids_for_value_range(- *value_range.start() as u64..=*value_range.end() as u64,- position_range,- &self.data,- positions,- );- }-}--#[cfg(test)]-mod tests {-- use itertools::Itertools;-- use super::*;- use crate::columnar::column_values::u128_based::U128Header;- use crate::columnar::column_values::{open_u128_mapped, serialize_column_values_u128};-- #[test]- fn compact_space_test() {- let ips: BTreeSet<u128> = [- 2u128, 4u128, 1000, 1001, 1002, 1003, 1004, 1005, 1008, 1010, 1012, 1260,- ]- .into_iter()- .collect();- let compact_space = get_compact_space(&ips, ips.len() as u32, 11);- let amplitude = compact_space.amplitude_compact_space();- assert_eq!(amplitude, 17);- assert_eq!(1, compact_space.u128_to_compact(2).unwrap());- assert_eq!(2, compact_space.u128_to_compact(3).unwrap());- assert_eq!(compact_space.u128_to_compact(100).unwrap_err(), 1);-- for (num1, num2) in (0..3).tuple_windows() {- assert_eq!(- compact_space.get_range_mapping(num1).compact_end() + 1,- compact_space.get_range_mapping(num2).compact_start- );- }-- let mut output: Vec<u8> = Vec::new();- compact_space.serialize(&mut output).unwrap();-- assert_eq!(- compact_space,- CompactSpace::deserialize(&mut &output[..]).unwrap()- );-- for ip in ips {- let compact = compact_space.u128_to_compact(ip).unwrap();- assert_eq!(compact_space.compact_to_u128(compact), ip);- }- }-- #[test]- fn compact_space_amplitude_test() {- let ips = &[100000u128, 1000000].into_iter().collect();- let compact_space = get_compact_space(ips, ips.len() as u32, 1);- let amplitude = compact_space.amplitude_compact_space();- assert_eq!(amplitude, 2);- }-- fn test_all(mut data: OwnedBytes, expected: &[u128]) {- let _header = U128Header::deserialize(&mut data);- let decompressor = CompactSpaceDecompressor::open(data).unwrap();- for (idx, expected_val) in expected.iter().cloned().enumerate() {- let val = decompressor.get(idx as u32);- assert_eq!(val, expected_val);-- let test_range = |range: RangeInclusive<u128>| {- let expected_positions = expected- .iter()- .positions(|val| range.contains(val))- .map(|pos| pos as u32)- .collect::<Vec<_>>();- let mut positions = Vec::new();- decompressor.get_row_ids_for_value_range(- range,- 0..decompressor.num_vals(),- &mut positions,- );- assert_eq!(positions, expected_positions);- };-- test_range(expected_val.saturating_sub(1)..=expected_val);- test_range(expected_val..=expected_val);- test_range(expected_val..=expected_val.saturating_add(1));- test_range(expected_val.saturating_sub(1)..=expected_val.saturating_add(1));- }- }-- fn test_aux_vals(u128_vals: &[u128]) -> OwnedBytes {- let mut out = Vec::new();- serialize_column_values_u128(&u128_vals, &mut out).unwrap();- let data = OwnedBytes::new(out);- test_all(data.clone(), u128_vals);- data- }-- #[test]- fn test_range_1() {- let vals = &[- 1u128,- 100u128,- 3u128,- 99999u128,- 100000u128,- 100001u128,- 4_000_211_221u128,- 4_000_211_222u128,- 333u128,- ];- let mut data = test_aux_vals(vals);-- let _header = U128Header::deserialize(&mut data);- let decomp = CompactSpaceDecompressor::open(data).unwrap();- let complete_range = 0..vals.len() as u32;- for (pos, val) in vals.iter().enumerate() {- let val = *val;- let pos = pos as u32;- let mut positions = Vec::new();- decomp.get_row_ids_for_value_range(val..=val, pos..pos + 1, &mut positions);- assert_eq!(positions, vec![pos]);- }-- // handle docid range out of bounds- let positions: Vec<u32> = get_positions_for_value_range_helper(&decomp, 0..=1, 1..u32::MAX);- assert!(positions.is_empty());-- let positions =- get_positions_for_value_range_helper(&decomp, 0..=1, complete_range.clone());- assert_eq!(positions, vec![0]);- let positions =- get_positions_for_value_range_helper(&decomp, 0..=2, complete_range.clone());- assert_eq!(positions, vec![0]);- let positions =- get_positions_for_value_range_helper(&decomp, 0..=3, complete_range.clone());- assert_eq!(positions, vec![0, 2]);- assert_eq!(- get_positions_for_value_range_helper(- &decomp,- 99999u128..=99999u128,- complete_range.clone()- ),- vec![3]- );- assert_eq!(- get_positions_for_value_range_helper(- &decomp,- 99999u128..=100000u128,- complete_range.clone()- ),- vec![3, 4]- );- assert_eq!(- get_positions_for_value_range_helper(- &decomp,- 99998u128..=100000u128,- complete_range.clone()- ),- vec![3, 4]- );- assert_eq!(- &get_positions_for_value_range_helper(- &decomp,- 99998u128..=99999u128,- complete_range.clone()- ),- &[3]- );- assert!(get_positions_for_value_range_helper(- &decomp,- 99998u128..=99998u128,- complete_range.clone()- )- .is_empty());- assert_eq!(- &get_positions_for_value_range_helper(- &decomp,- 333u128..=333u128,- complete_range.clone()- ),- &[8]- );- assert_eq!(- &get_positions_for_value_range_helper(- &decomp,- 332u128..=333u128,- complete_range.clone()- ),- &[8]- );- assert_eq!(- &get_positions_for_value_range_helper(- &decomp,- 332u128..=334u128,- complete_range.clone()- ),- &[8]- );- assert_eq!(- &get_positions_for_value_range_helper(- &decomp,- 333u128..=334u128,- complete_range.clone()- ),- &[8]- );-- assert_eq!(- &get_positions_for_value_range_helper(- &decomp,- 4_000_211_221u128..=5_000_000_000u128,- complete_range- ),- &[6, 7]- );- }-- #[test]- fn test_empty() {- let vals = &[];- let data = test_aux_vals(vals);- let _decomp = CompactSpaceDecompressor::open(data).unwrap();- }-- #[test]- fn test_range_2() {- let vals = &[- 100u128,- 99999u128,- 100000u128,- 100001u128,- 4_000_211_221u128,- 4_000_211_222u128,- 333u128,- ];- let mut data = test_aux_vals(vals);- let _header = U128Header::deserialize(&mut data);- let decomp = CompactSpaceDecompressor::open(data).unwrap();- let complete_range = 0..vals.len() as u32;- assert!(- &get_positions_for_value_range_helper(&decomp, 0..=5, complete_range.clone())- .is_empty(),- );- assert_eq!(- &get_positions_for_value_range_helper(&decomp, 0..=100, complete_range.clone()),- &[0]- );- assert_eq!(- &get_positions_for_value_range_helper(&decomp, 0..=105, complete_range),- &[0]- );- }-- fn get_positions_for_value_range_helper<C: ColumnValues<T> + ?Sized, T: PartialOrd>(- column: &C,- value_range: RangeInclusive<T>,- doc_id_range: Range<u32>,- ) -> Vec<u32> {- let mut positions = Vec::new();- column.get_row_ids_for_value_range(value_range, doc_id_range, &mut positions);- positions- }-- #[test]- fn test_range_3() {- let vals = &[- 200u128,- 201,- 202,- 203,- 204,- 204,- 206,- 207,- 208,- 209,- 210,- 1_000_000,- 5_000_000_000,- ];- let mut out = Vec::new();- serialize_column_values_u128(&&vals[..], &mut out).unwrap();- let decomp = open_u128_mapped(OwnedBytes::new(out)).unwrap();- let complete_range = 0..vals.len() as u32;-- assert_eq!(- get_positions_for_value_range_helper(&*decomp, 199..=200, complete_range.clone()),- vec![0]- );-- assert_eq!(- get_positions_for_value_range_helper(&*decomp, 199..=201, complete_range.clone()),- vec![0, 1]- );-- assert_eq!(- get_positions_for_value_range_helper(&*decomp, 200..=200, complete_range.clone()),- vec![0]- );-- assert_eq!(- get_positions_for_value_range_helper(&*decomp, 1_000_000..=1_000_000, complete_range),- vec![11]- );- }-- #[test]- fn test_bug1() {- let vals = &[9223372036854775806];- let _data = test_aux_vals(vals);- }-- #[test]- fn test_bug2() {- let vals = &[340282366920938463463374607431768211455u128];- let _data = test_aux_vals(vals);- }-- #[test]- fn test_bug3() {- let vals = &[340282366920938463463374607431768211454];- let _data = test_aux_vals(vals);- }-- #[test]- fn test_bug4() {- let vals = &[340282366920938463463374607431768211455, 0];- let _data = test_aux_vals(vals);- }-- #[test]- fn test_first_large_gaps() {- let vals = &[1_000_000_000u128; 100];- let _data = test_aux_vals(vals);- }-- use proptest::prelude::*;-- fn num_strategy() -> impl Strategy<Value = u128> {- prop_oneof![- 1 => prop::num::u128::ANY.prop_map(|num| u128::MAX - (num % 10) ),- 1 => prop::num::u128::ANY.prop_map(|num| i64::MAX as u128 + 5 - (num % 10) ),- 1 => prop::num::u128::ANY.prop_map(|num| i128::MAX as u128 + 5 - (num % 10) ),- 1 => prop::num::u128::ANY.prop_map(|num| num % 10 ),- 20 => prop::num::u128::ANY,- ]- }-- proptest! {- #![proptest_config(ProptestConfig::with_cases(10))]-- #[test]- fn compress_decompress_random(vals in proptest::collection::vec(num_strategy() , 1..1000)) {- let _data = test_aux_vals(&vals);- }- }-}diff --git a/crates/tantivy/src/columnar/column_values/u128_based/mod.rs b/crates/tantivy/src/columnar/column_values/u128_based/mod.rsdeleted file mode 100644index 9e678753..00000000--- a/crates/tantivy/src/columnar/column_values/u128_based/mod.rs+++ /dev/null@@ -1,197 +0,0 @@-use std::fmt::Debug;-use std::io;-use std::io::Write;-use std::sync::Arc;--mod compact_space;--use crate::common::{BinarySerializable, OwnedBytes, VInt};-pub use compact_space::{- CompactSpaceCompressor, CompactSpaceDecompressor, CompactSpaceU64Accessor,-};--use super::{ColumnValues, MonotonicallyMappableToU128};-use crate::columnar::column_values::monotonic_map_column;-use crate::columnar::column_values::monotonic_mapping::{- StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal,-};-use crate::columnar::iterable::Iterable;--#[derive(Debug, Copy, Clone, PartialEq, Eq)]-pub(crate) struct U128Header {- pub num_vals: u32,- pub codec_type: U128FastFieldCodecType,-}--impl BinarySerializable for U128Header {- fn serialize<W: io::Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {- VInt(self.num_vals as u64).serialize(writer)?;- self.codec_type.serialize(writer)?;- Ok(())- }-- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {- let num_vals = VInt::deserialize(reader)?.0 as u32;- let codec_type = U128FastFieldCodecType::deserialize(reader)?;- Ok(U128Header {- num_vals,- codec_type,- })- }-}--/// Serializes u128 values with the compact space codec.-pub fn serialize_column_values_u128<T: MonotonicallyMappableToU128>(- iterable: &dyn Iterable<T>,- output: &mut impl io::Write,-) -> io::Result<()> {- let compressor = CompactSpaceCompressor::train_from(- iterable- .boxed_iter()- .map(MonotonicallyMappableToU128::to_u128),- );- let header = U128Header {- num_vals: compressor.num_vals(),- codec_type: U128FastFieldCodecType::CompactSpace,- };- header.serialize(output)?;- compressor.compress_into(- iterable- .boxed_iter()- .map(MonotonicallyMappableToU128::to_u128),- output,- )?;- Ok(())-}--#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]-#[repr(u8)]-/// Available codecs to use to encode the u128 (via [`MonotonicallyMappableToU128`]) converted data.-pub(crate) enum U128FastFieldCodecType {- /// This codec takes a large number space (u128) and reduces it to a compact number space, by- /// removing the holes.- CompactSpace = 1,-}--impl BinarySerializable for U128FastFieldCodecType {- fn serialize<W: Write + ?Sized>(&self, wrt: &mut W) -> io::Result<()> {- self.to_code().serialize(wrt)- }-- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {- let code = u8::deserialize(reader)?;- let codec_type: Self = Self::from_code(code)- .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Unknown code `{code}.`"))?;- Ok(codec_type)- }-}--impl U128FastFieldCodecType {- pub(crate) fn to_code(self) -> u8 {- self as u8- }-- pub(crate) fn from_code(code: u8) -> Option<Self> {- match code {- 1 => Some(Self::CompactSpace),- _ => None,- }- }-}--/// Returns the correct codec reader wrapped in the `Arc` for the data.-pub fn open_u128_mapped<T: MonotonicallyMappableToU128 + Debug>(- mut bytes: OwnedBytes,-) -> io::Result<Arc<dyn ColumnValues<T>>> {- let header = U128Header::deserialize(&mut bytes)?;- assert_eq!(header.codec_type, U128FastFieldCodecType::CompactSpace);- let reader = CompactSpaceDecompressor::open(bytes)?;- let inverted: StrictlyMonotonicMappingInverter<StrictlyMonotonicMappingToInternal<T>> =- StrictlyMonotonicMappingToInternal::<T>::new().into();- Ok(Arc::new(monotonic_map_column(reader, inverted)))-}--/// Returns the u64 representation of the u128 data.-/// The internal representation of the data as u64 is useful for faster processing.-///-/// In order to convert to u128 back cast to `CompactSpaceU64Accessor` and call-/// `compact_to_u128`.-///-/// # Notice-/// In case there are new codecs added, check for usages of `CompactSpaceDecompressorU64` and-/// also handle the new codecs.-pub fn open_u128_as_compact_u64(mut bytes: OwnedBytes) -> io::Result<Arc<dyn ColumnValues<u64>>> {- let header = U128Header::deserialize(&mut bytes)?;- assert_eq!(header.codec_type, U128FastFieldCodecType::CompactSpace);- let reader = CompactSpaceU64Accessor::open(bytes)?;- Ok(Arc::new(reader))-}--#[cfg(test)]-pub mod tests {- use super::*;- use crate::columnar::column_values::u64_based::{- serialize_and_load_u64_based_column_values, serialize_u64_based_column_values,- ALL_U64_CODEC_TYPES,- };- use crate::columnar::column_values::CodecType;-- #[test]- fn test_serialize_deserialize_u128_header() {- let original = U128Header {- num_vals: 11,- codec_type: U128FastFieldCodecType::CompactSpace,- };- let mut out = Vec::new();- original.serialize(&mut out).unwrap();- let restored = U128Header::deserialize(&mut &out[..]).unwrap();- assert_eq!(restored, original);- }-- #[test]- fn test_serialize_deserialize() {- let original = [1u64, 5u64, 10u64];- let restored: Vec<u64> =- serialize_and_load_u64_based_column_values(&&original[..], &ALL_U64_CODEC_TYPES)- .iter()- .collect();- assert_eq!(&restored, &original[..]);- }-- #[test]- fn test_fastfield_bool_size_bitwidth_1() {- let mut buffer = Vec::new();- serialize_u64_based_column_values::<bool>(- &&[false, true][..],- &ALL_U64_CODEC_TYPES,- &mut buffer,- )- .unwrap();- // TODO put the header as a footer so that it serves as a padding.- // 5 bytes of header, 1 byte of value, 7 bytes of padding.- assert_eq!(buffer.len(), 5 + 1);- }-- #[test]- fn test_fastfield_bool_bit_size_bitwidth_0() {- let mut buffer = Vec::new();- serialize_u64_based_column_values::<bool>(- &&[false, true][..],- &ALL_U64_CODEC_TYPES,- &mut buffer,- )- .unwrap();- // 6 bytes of header, 0 bytes of value, 7 bytes of padding.- assert_eq!(buffer.len(), 6);- }-- #[test]- fn test_fastfield_gcd() {- let mut buffer = Vec::new();- let vals: Vec<u64> = (0..80).map(|val| (val % 7) * 1_000u64).collect();- serialize_u64_based_column_values(&&vals[..], &[CodecType::Bitpacked], &mut buffer)- .unwrap();- // Values are stored over 3 bits.- assert_eq!(buffer.len(), 6 + (3 * 80 / 8));- }-}diff --git a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rsindex 7d89941d..bd070533 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs@@ -79,7 +79,7 @@ impl ColumnValues for BitpackedReader {fn get_row_ids_for_value_range(&self,range: RangeInclusive<u64>,- doc_id_range: Range<u32>,+ mut doc_id_range: Range<u32>,positions: &mut Vec<u32>,) {let Some(transformed_range) =@@ -88,6 +88,8 @@ impl ColumnValues for BitpackedReader {positions.clear();return;};+ doc_id_range.end = doc_id_range.end.min(self.num_vals());+self.bit_unpacker.get_ids_for_value_range(transformed_range,doc_id_range,diff --git a/crates/tantivy/src/columnar/columnar/column_type.rs b/crates/tantivy/src/columnar/columnar/column_type.rsindex 24dc9e66..2016c8f8 100644--- a/crates/tantivy/src/columnar/columnar/column_type.rs+++ b/crates/tantivy/src/columnar/columnar/column_type.rs@@ -1,6 +1,5 @@use std::fmt;use std::fmt::Debug;-use std::net::Ipv6Addr;use serde::{Deserialize, Serialize};@@ -16,10 +15,8 @@ pub enum ColumnType {U64 = 1u8,F64 = 2u8,Bytes = 3u8,- Str = 4u8,- Bool = 5u8,- IpAddr = 6u8,- DateTime = 7u8,+ Bool = 4u8,+ DateTime = 5u8,}impl fmt::Display for ColumnType {@@ -29,9 +26,7 @@ impl fmt::Display for ColumnType {ColumnType::U64 => "u64",ColumnType::F64 => "f64",ColumnType::Bytes => "bytes",- ColumnType::Str => "str",ColumnType::Bool => "bool",- ColumnType::IpAddr => "ip",ColumnType::DateTime => "datetime",};write!(f, "{short_str}")@@ -39,14 +34,12 @@ impl fmt::Display for ColumnType {}// The order needs to match _exactly_ the order in the enum-const COLUMN_TYPES: [ColumnType; 8] = [+const COLUMN_TYPES: [ColumnType; 6] = [ColumnType::I64,ColumnType::U64,ColumnType::F64,ColumnType::Bytes,- ColumnType::Str,ColumnType::Bool,- ColumnType::IpAddr,ColumnType::DateTime,];@@ -79,11 +72,7 @@ impl ColumnType {ColumnType::I64 => Some(NumericalType::I64),ColumnType::U64 => Some(NumericalType::U64),ColumnType::F64 => Some(NumericalType::F64),- ColumnType::Bytes- | ColumnType::Str- | ColumnType::Bool- | ColumnType::IpAddr- | ColumnType::DateTime => None,+ ColumnType::Bytes | ColumnType::Bool | ColumnType::DateTime => None,}}}@@ -142,16 +131,6 @@ impl HasAssociatedColumnType for crate::common::DateTime {}}-impl HasAssociatedColumnType for Ipv6Addr {- fn column_type() -> ColumnType {- ColumnType::IpAddr- }-- fn default_value() -> Self {- Ipv6Addr::from([0u8; 16])- }-}-#[cfg(test)]mod tests {use super::*;@@ -178,6 +157,6 @@ mod tests {num_cardinality += 1;}}- assert_eq!(num_cardinality, 3);+ assert_eq!(num_cardinality, 1);}}diff --git a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rsindex 27a7c307..a30f13a6 100644--- a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs+++ b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs@@ -12,7 +12,7 @@ use crate::columnar::{BytesColumn, MergeRowOrder, ShuffleMergeOrder};// Serialize [Dictionary, Column, dictionary num bytes U64::LE]// Column: [Column Index, Column Values, column index num bytes U32::LE]pub fn merge_bytes_or_str_column(- column_index: SerializableColumnIndex<'_>,+ column_index: SerializableColumnIndex,bytes_columns: &[Option<BytesColumn>],merge_row_order: &MergeRowOrder,output: &mut impl Write,@@ -79,8 +79,15 @@ impl<'a> RemappedTermOrdinalsValues<'a> {Box::new(shuffle_merge_order.iter_new_to_old_row_addrs()- .flat_map(move |old_addr| {+ .filter_map(move |old_addr| {let segment_ord = self.term_ord_mapping.get_segment(old_addr.segment_ord);+ if !segment_ord.is_empty() {+ Some((old_addr, segment_ord))+ } else {+ None+ }+ })+ .flat_map(|(old_addr, segment_ord)| {self.bytes_columns[old_addr.segment_ord as usize].as_ref().into_iter()@@ -98,7 +105,7 @@ fn compute_term_bitset(column: &BytesColumn, row_bitset: &ReadOnlyBitSet) -> Bitlet num_terms = column.dictionary().num_terms();let mut term_bitset = BitSet::with_max_value(num_terms as u32);for row_id in row_bitset.iter() {- for term_ord in column.term_ord_column.values_for_doc(row_id) {+ if let Some(term_ord) = column.term_ord_column.first(row_id) {term_bitset.insert(term_ord as u32);}}diff --git a/crates/tantivy/src/columnar/columnar/merge/mod.rs b/crates/tantivy/src/columnar/columnar/merge/mod.rsindex 6acc8f14..1ee7e4a3 100644--- a/crates/tantivy/src/columnar/columnar/merge/mod.rs+++ b/crates/tantivy/src/columnar/columnar/merge/mod.rs@@ -4,16 +4,12 @@ mod term_merger;use std::collections::{BTreeMap, HashSet};use std::io;-use std::net::Ipv6Addr;use std::sync::Arc;-use itertools::Itertools;pub use merge_mapping::{MergeRowOrder, ShuffleMergeOrder, StackMergeOrder};use super::writer::ColumnarSerializer;-use crate::columnar::column::{- serialize_column_mappable_to_u128, serialize_column_mappable_to_u64,-};+use crate::columnar::column::serialize_column_mappable_to_u64;use crate::columnar::column_values::MergedColumnValues;use crate::columnar::columnar::merge::merge_dict_column::merge_bytes_or_str_column;use crate::columnar::columnar::writer::CompatibleNumericalTypes;@@ -37,9 +33,7 @@ use crate::columnar::{pub(crate) enum ColumnTypeCategory {Numerical,Bytes,- Str,Bool,- IpAddr,DateTime,}@@ -50,9 +44,7 @@ impl From<ColumnType> for ColumnTypeCategory {ColumnType::U64 => ColumnTypeCategory::Numerical,ColumnType::F64 => ColumnTypeCategory::Numerical,ColumnType::Bytes => ColumnTypeCategory::Bytes,- ColumnType::Str => ColumnTypeCategory::Str,ColumnType::Bool => ColumnTypeCategory::Bool,- ColumnType::IpAddr => ColumnTypeCategory::IpAddr,ColumnType::DateTime => ColumnTypeCategory::DateTime,}}@@ -84,10 +76,6 @@ pub fn merge_columnar(output: &mut impl io::Write,) -> io::Result<()> {let mut serializer = ColumnarSerializer::new(output);- let num_rows_per_columnar = columnar_readers- .iter()- .map(|reader| reader.num_rows())- .collect::<Vec<u32>>();let columns_to_merge =group_columns_for_merge(columnar_readers, required_columns, &merge_row_order)?;@@ -106,7 +94,6 @@ pub fn merge_columnar(serializer.start_serialize_column(column_name.as_bytes(), column_type);merge_column(column_type,- &num_rows_per_columnar,columns,&merge_row_order,&mut column_serializer,@@ -125,13 +112,12 @@ fn dynamic_column_to_u64_monotonic(dynamic_column: DynamicColumn) -> Option<ColuDynamicColumn::U64(column) => Some(column.to_u64_monotonic()),DynamicColumn::F64(column) => Some(column.to_u64_monotonic()),DynamicColumn::DateTime(column) => Some(column.to_u64_monotonic()),- DynamicColumn::IpAddr(_) | DynamicColumn::Bytes(_) | DynamicColumn::Str(_) => None,+ DynamicColumn::Bytes(_) => None,}}fn merge_column(column_type: ColumnType,- num_docs_per_column: &[u32],columns: Vec<Option<DynamicColumn>>,merge_row_order: &MergeRowOrder,wrt: &mut impl io::Write,@@ -145,23 +131,16 @@ fn merge_column(let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());let mut column_values: Vec<Option<Arc<dyn ColumnValues>>> =Vec::with_capacity(columns.len());- for (i, dynamic_column_opt) in columns.into_iter().enumerate() {+ for dynamic_column_opt in columns {if let Some(Column { index: idx, values }) =dynamic_column_opt.and_then(dynamic_column_to_u64_monotonic){column_indexes.push(idx);column_values.push(Some(values));- } else {- column_indexes.push(ColumnIndex::Empty {- num_docs: num_docs_per_column[i],- });- column_values.push(None);}}- let merged_column_index = crate::columnar::column_index::merge_column_index(- &column_indexes[..],- merge_row_order,- );+ let merged_column_index =+ crate::columnar::column_index::merge_column_index(merge_row_order);let merge_column_values = MergedColumnValues {column_indexes: &column_indexes[..],column_values: &column_values[..],@@ -169,61 +148,17 @@ fn merge_column(};serialize_column_mappable_to_u64(merged_column_index, &merge_column_values, wrt)?;}- ColumnType::IpAddr => {- let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());- let mut column_values: Vec<Option<Arc<dyn ColumnValues<Ipv6Addr>>>> =- Vec::with_capacity(columns.len());- for (i, dynamic_column_opt) in columns.into_iter().enumerate() {- if let Some(DynamicColumn::IpAddr(Column { index: idx, values })) =- dynamic_column_opt- {- column_indexes.push(idx);- column_values.push(Some(values));- } else {- column_indexes.push(ColumnIndex::Empty {- num_docs: num_docs_per_column[i],- });- column_values.push(None);- }- }-- let merged_column_index = crate::columnar::column_index::merge_column_index(- &column_indexes[..],- merge_row_order,- );- let merge_column_values = MergedColumnValues {- column_indexes: &column_indexes[..],- column_values: &column_values,- merge_row_order,- };-- serialize_column_mappable_to_u128(merged_column_index, &merge_column_values, wrt)?;- }- ColumnType::Bytes | ColumnType::Str => {+ ColumnType::Bytes => {let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());let mut bytes_columns: Vec<Option<BytesColumn>> = Vec::with_capacity(columns.len());- for (i, dynamic_column_opt) in columns.into_iter().enumerate() {- match dynamic_column_opt {- Some(DynamicColumn::Str(str_column)) => {- column_indexes.push(str_column.term_ord_column.index.clone());- bytes_columns.push(Some(str_column.into()));- }- Some(DynamicColumn::Bytes(bytes_column)) => {- column_indexes.push(bytes_column.term_ord_column.index.clone());- bytes_columns.push(Some(bytes_column));- }- _ => {- column_indexes.push(ColumnIndex::Empty {- num_docs: num_docs_per_column[i],- });- bytes_columns.push(None);- }+ for dynamic_column in columns.into_iter().flatten() {+ if let DynamicColumn::Bytes(bytes_column) = dynamic_column {+ column_indexes.push(bytes_column.term_ord_column.index.clone());+ bytes_columns.push(Some(bytes_column));}}- let merged_column_index = crate::columnar::column_index::merge_column_index(- &column_indexes[..],- merge_row_order,- );+ let merged_column_index =+ crate::columnar::column_index::merge_column_index(merge_row_order);merge_bytes_or_str_column(merged_column_index, &bytes_columns, merge_row_order, wrt)?;}}@@ -286,10 +221,10 @@ impl GroupedColumnsHandle {for (columnar_id, column) in self.columns.iter().enumerate() {if let Some(column) = column {let column = column.open()?;+// We skip columns that end up with 0 documents.// That way, we make sure they don't end up influencing the merge type or// creating empty columns.-if is_empty_after_merge(merge_row_order, &column, columnar_id) {columns.push(None);} else {@@ -368,36 +303,7 @@ fn is_empty_after_merge(if let Some(alive_bitset) = &shuffled.alive_bitsets[columnar_ord] {let column_index = column.column_index();match column_index {- ColumnIndex::Empty { .. } => true,ColumnIndex::Full => alive_bitset.len() == 0,- ColumnIndex::Optional(optional_index) => {- for doc in optional_index.iter_rows() {- if alive_bitset.contains(doc) {- return false;- }- }- true- }- ColumnIndex::Multivalued(multivalued_index) => {- for (doc_id, (start_index, end_index)) in multivalued_index- .start_index_column- .iter()- .tuple_windows()- .enumerate()- {- let doc_id = doc_id as u32;- if start_index == end_index {- // There are no values in this document- continue;- }- // The document contains values and is present in the alive bitset.- // The column is therefore not empty.- if alive_bitset.contains(doc_id) {- return false;- }- }- true- }}} else {// No document is being deleted.@@ -482,11 +388,7 @@ fn min_max_if_numerical(column: &DynamicColumn) -> Option<(NumericalValue, NumerDynamicColumn::I64(column) => Some((column.min_value().into(), column.max_value().into())),DynamicColumn::U64(column) => Some((column.min_value().into(), column.max_value().into())),DynamicColumn::F64(column) => Some((column.min_value().into(), column.max_value().into())),- DynamicColumn::Bool(_)- | DynamicColumn::IpAddr(_)- | DynamicColumn::DateTime(_)- | DynamicColumn::Bytes(_)- | DynamicColumn::Str(_) => None,+ DynamicColumn::Bool(_) | DynamicColumn::DateTime(_) | DynamicColumn::Bytes(_) => None,}}diff --git a/crates/tantivy/src/columnar/columnar/merge/tests.rs b/crates/tantivy/src/columnar/columnar/merge/tests.rsindex 0f77129d..da1bdde4 100644--- a/crates/tantivy/src/columnar/columnar/merge/tests.rs+++ b/crates/tantivy/src/columnar/columnar/merge/tests.rs@@ -1,5 +1,5 @@use super::*;-use crate::columnar::{Cardinality, ColumnarWriter, HasAssociatedColumnType, RowId};+use crate::columnar::{ColumnarWriter, HasAssociatedColumnType, RowId};fn make_columnar<T: Into<NumericalValue> + HasAssociatedColumnType + Copy>(column_name: &str,@@ -73,28 +73,6 @@ fn test_group_columns_with_required_column() {assert!(column_map.contains_key(&("numbers".to_string(), ColumnTypeCategory::Numerical)));}-#[test]-fn test_group_columns_required_column_with_no_existing_columns() {- let columnar1 = make_columnar("numbers", &[2u64]);- let columnar2 = make_columnar("numbers", &[2u64]);- let columnars = &[&columnar1, &columnar2];- let merge_order = StackMergeOrder::stack(columnars).into();- let column_map: BTreeMap<_, _> = group_columns_for_merge(- columnars,- &[("required_col".to_string(), ColumnType::Str)],- &merge_order,- )- .unwrap();- assert_eq!(column_map.len(), 2);- let columns = &column_map- .get(&("required_col".to_string(), ColumnTypeCategory::Str))- .unwrap()- .columns;- assert_eq!(columns.len(), 2);- assert!(columns[0].is_none());- assert!(columns[1].is_none());-}-#[test]fn test_group_columns_required_column_is_above_all_columns_have_the_same_type_rule() {let columnar1 = make_columnar("numbers", &[2i64]);@@ -139,354 +117,3 @@ fn test_missing_column() {assert!(columns[1].is_some());}}--fn make_numerical_columnar_multiple_columns(- columns: &[(&str, &[&[NumericalValue]])],-) -> ColumnarReader {- let mut dataframe_writer = ColumnarWriter::default();- for (column_name, column_values) in columns {- for (row_id, vals) in column_values.iter().enumerate() {- for val in vals.iter() {- dataframe_writer.record_numerical(row_id as u32, column_name, *val);- }- }- }- let num_rows = columns- .iter()- .map(|(_, val_rows)| val_rows.len() as RowId)- .max()- .unwrap_or(0u32);- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer- .serialize(num_rows, None, &mut buffer)- .unwrap();- ColumnarReader::open(buffer).unwrap()-}--#[track_caller]-fn make_byte_columnar_multiple_columns(- columns: &[(&str, &[&[&[u8]]])],- num_rows: u32,-) -> ColumnarReader {- let mut dataframe_writer = ColumnarWriter::default();- for (column_name, column_values) in columns {- assert_eq!(- column_values.len(),- num_rows as usize,- "All columns must have `{num_rows}` rows"- );- for (row_id, vals) in column_values.iter().enumerate() {- for val in vals.iter() {- dataframe_writer.record_bytes(row_id as u32, column_name, val);- }- }- }- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer- .serialize(num_rows, None, &mut buffer)- .unwrap();- ColumnarReader::open(buffer).unwrap()-}--fn make_text_columnar_multiple_columns(columns: &[(&str, &[&[&str]])]) -> ColumnarReader {- let mut dataframe_writer = ColumnarWriter::default();- for (column_name, column_values) in columns {- for (row_id, vals) in column_values.iter().enumerate() {- for val in vals.iter() {- dataframe_writer.record_str(row_id as u32, column_name, val);- }- }- }- let num_rows = columns- .iter()- .map(|(_, val_rows)| val_rows.len() as RowId)- .max()- .unwrap_or(0u32);- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer- .serialize(num_rows, None, &mut buffer)- .unwrap();- ColumnarReader::open(buffer).unwrap()-}--#[test]-fn test_merge_columnar_numbers() {- let columnar1 =- make_numerical_columnar_multiple_columns(&[("numbers", &[&[NumericalValue::from(-1f64)]])]);- let columnar2 = make_numerical_columnar_multiple_columns(&[(- "numbers",- &[&[], &[NumericalValue::from(-3f64)]],- )]);- let mut buffer = Vec::new();- let columnars = &[&columnar1, &columnar2];- let stack_merge_order = StackMergeOrder::stack(columnars);- crate::columnar::columnar::merge_columnar(- columnars,- &[],- MergeRowOrder::Stack(stack_merge_order),- &mut buffer,- )- .unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_rows(), 3);- assert_eq!(columnar_reader.num_columns(), 1);- let cols = columnar_reader.read_columns("numbers").unwrap();- let dynamic_column = cols[0].open().unwrap();- let DynamicColumn::F64(vals) = dynamic_column else {- panic!()- };- assert_eq!(vals.get_cardinality(), Cardinality::Optional);- assert_eq!(vals.first(0u32), Some(-1f64));- assert_eq!(vals.first(1u32), None);- assert_eq!(vals.first(2u32), Some(-3f64));-}--#[test]-fn test_merge_columnar_texts() {- let columnar1 = make_text_columnar_multiple_columns(&[("texts", &[&["a"]])]);- let columnar2 = make_text_columnar_multiple_columns(&[("texts", &[&[], &["b"]])]);- let mut buffer = Vec::new();- let columnars = &[&columnar1, &columnar2];- let stack_merge_order = StackMergeOrder::stack(columnars);- crate::columnar::columnar::merge_columnar(- columnars,- &[],- MergeRowOrder::Stack(stack_merge_order),- &mut buffer,- )- .unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_rows(), 3);- assert_eq!(columnar_reader.num_columns(), 1);- let cols = columnar_reader.read_columns("texts").unwrap();- let dynamic_column = cols[0].open().unwrap();- let DynamicColumn::Str(vals) = dynamic_column else {- panic!()- };- assert_eq!(vals.ords().get_cardinality(), Cardinality::Optional);-- let get_str_for_ord = |ord| {- let mut out = String::new();- vals.ord_to_str(ord, &mut out).unwrap();- out- };-- assert_eq!(vals.dictionary.num_terms(), 2);- assert_eq!(get_str_for_ord(0), "a");- assert_eq!(get_str_for_ord(1), "b");-- let get_str_for_row = |row_id| {- let term_ords: Vec<u64> = vals.term_ords(row_id).collect();- assert!(term_ords.len() <= 1);- let mut out = String::new();- if term_ords.len() == 1 {- vals.ord_to_str(term_ords[0], &mut out).unwrap();- }- out- };-- assert_eq!(get_str_for_row(0), "a");- assert_eq!(get_str_for_row(1), "");- assert_eq!(get_str_for_row(2), "b");-}--#[test]-fn test_merge_columnar_byte() {- let columnar1 = make_byte_columnar_multiple_columns(&[("bytes", &[&[b"bbbb"], &[b"baaa"]])], 2);- let columnar2 = make_byte_columnar_multiple_columns(&[("bytes", &[&[], &[b"a"]])], 2);- let mut buffer = Vec::new();- let columnars = &[&columnar1, &columnar2];- let stack_merge_order = StackMergeOrder::stack(columnars);- crate::columnar::columnar::merge_columnar(- columnars,- &[],- MergeRowOrder::Stack(stack_merge_order),- &mut buffer,- )- .unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_rows(), 4);- assert_eq!(columnar_reader.num_columns(), 1);- let cols = columnar_reader.read_columns("bytes").unwrap();- let dynamic_column = cols[0].open().unwrap();- let DynamicColumn::Bytes(vals) = dynamic_column else {- panic!()- };- let get_bytes_for_ord = |ord| {- let mut out = Vec::new();- vals.ord_to_bytes(ord, &mut out).unwrap();- out- };-- assert_eq!(vals.dictionary.num_terms(), 3);- assert_eq!(get_bytes_for_ord(0), b"a");- assert_eq!(get_bytes_for_ord(1), b"baaa");- assert_eq!(get_bytes_for_ord(2), b"bbbb");-- let get_bytes_for_row = |row_id| {- let term_ords: Vec<u64> = vals.term_ords(row_id).collect();- assert!(term_ords.len() <= 1);- let mut out = Vec::new();- if term_ords.len() == 1 {- vals.ord_to_bytes(term_ords[0], &mut out).unwrap();- }- out- };-- assert_eq!(get_bytes_for_row(0), b"bbbb");- assert_eq!(get_bytes_for_row(1), b"baaa");- assert_eq!(get_bytes_for_row(2), b"");- assert_eq!(get_bytes_for_row(3), b"a");-}--#[test]-fn test_merge_columnar_byte_with_missing() {- let columnar1 = make_byte_columnar_multiple_columns(&[], 3);- let columnar2 = make_byte_columnar_multiple_columns(&[("col", &[&[b"b"], &[]])], 2);- let columnar3 = make_byte_columnar_multiple_columns(- &[- ("col", &[&[], &[b"b"], &[b"a", b"b"]]),- ("col2", &[&[b"hello"], &[], &[b"a", b"b"]]),- ],- 3,- );- let mut buffer = Vec::new();- let columnars = &[&columnar1, &columnar2, &columnar3];- let stack_merge_order = StackMergeOrder::stack(columnars);- crate::columnar::columnar::merge_columnar(- columnars,- &[],- MergeRowOrder::Stack(stack_merge_order),- &mut buffer,- )- .unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_rows(), 3 + 2 + 3);- assert_eq!(columnar_reader.num_columns(), 2);- let cols = columnar_reader.read_columns("col").unwrap();- let dynamic_column = cols[0].open().unwrap();- let DynamicColumn::Bytes(vals) = dynamic_column else {- panic!()- };- let get_bytes_for_ord = |ord| {- let mut out = Vec::new();- vals.ord_to_bytes(ord, &mut out).unwrap();- out- };- assert_eq!(vals.dictionary.num_terms(), 2);- assert_eq!(get_bytes_for_ord(0), b"a");- assert_eq!(get_bytes_for_ord(1), b"b");- let get_bytes_for_row = |row_id| {- let terms: Vec<Vec<u8>> = vals- .term_ords(row_id)- .map(|term_ord| {- let mut out = Vec::new();- vals.ord_to_bytes(term_ord, &mut out).unwrap();- out- })- .collect();- terms- };- assert!(get_bytes_for_row(0).is_empty());- assert!(get_bytes_for_row(1).is_empty());- assert!(get_bytes_for_row(2).is_empty());- assert_eq!(get_bytes_for_row(3), vec![b"b".to_vec()]);- assert!(get_bytes_for_row(4).is_empty());- assert!(get_bytes_for_row(5).is_empty());- assert_eq!(get_bytes_for_row(6), vec![b"b".to_vec()]);- assert_eq!(get_bytes_for_row(7), vec![b"a".to_vec(), b"b".to_vec()]);-}--#[test]-fn test_merge_columnar_different_types() {- let columnar1 = make_text_columnar_multiple_columns(&[("mixed", &[&["a"]])]);- let columnar2 = make_text_columnar_multiple_columns(&[("mixed", &[&[], &["b"]])]);- let columnar3 = make_columnar("mixed", &[1i64]);- let mut buffer = Vec::new();- let columnars = &[&columnar1, &columnar2, &columnar3];- let stack_merge_order = StackMergeOrder::stack(columnars);- crate::columnar::columnar::merge_columnar(- columnars,- &[],- MergeRowOrder::Stack(stack_merge_order),- &mut buffer,- )- .unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_rows(), 4);- assert_eq!(columnar_reader.num_columns(), 2);- let cols = columnar_reader.read_columns("mixed").unwrap();-- // numeric column- let dynamic_column = cols[0].open().unwrap();- let DynamicColumn::I64(vals) = dynamic_column else {- panic!()- };- assert_eq!(vals.get_cardinality(), Cardinality::Optional);- assert_eq!(vals.values_for_doc(0).collect_vec(), Vec::<i64>::new());- assert_eq!(vals.values_for_doc(1).collect_vec(), Vec::<i64>::new());- assert_eq!(vals.values_for_doc(2).collect_vec(), Vec::<i64>::new());- assert_eq!(vals.values_for_doc(3).collect_vec(), vec![1]);- assert_eq!(vals.values_for_doc(4).collect_vec(), Vec::<i64>::new());-- // text column- let dynamic_column = cols[1].open().unwrap();- let DynamicColumn::Str(vals) = dynamic_column else {- panic!()- };- assert_eq!(vals.ords().get_cardinality(), Cardinality::Optional);- let get_str_for_ord = |ord| {- let mut out = String::new();- vals.ord_to_str(ord, &mut out).unwrap();- out- };-- assert_eq!(vals.dictionary.num_terms(), 2);- assert_eq!(get_str_for_ord(0), "a");- assert_eq!(get_str_for_ord(1), "b");-- let get_str_for_row = |row_id| {- let term_ords: Vec<String> = vals- .term_ords(row_id)- .map(|el| {- let mut out = String::new();- vals.ord_to_str(el, &mut out).unwrap();- out- })- .collect();- term_ords- };-- assert_eq!(get_str_for_row(0), vec!["a".to_string()]);- assert_eq!(get_str_for_row(1), Vec::<String>::new());- assert_eq!(get_str_for_row(2), vec!["b".to_string()]);- assert_eq!(get_str_for_row(3), Vec::<String>::new());-}--#[test]-fn test_merge_columnar_different_empty_cardinality() {- let columnar1 = make_text_columnar_multiple_columns(&[("mixed", &[&["a"]])]);- let columnar2 = make_columnar("mixed", &[1i64]);- let mut buffer = Vec::new();- let columnars = &[&columnar1, &columnar2];- let stack_merge_order = StackMergeOrder::stack(columnars);- crate::columnar::columnar::merge_columnar(- columnars,- &[],- MergeRowOrder::Stack(stack_merge_order),- &mut buffer,- )- .unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_rows(), 2);- assert_eq!(columnar_reader.num_columns(), 2);- let cols = columnar_reader.read_columns("mixed").unwrap();-- // numeric column- let dynamic_column = cols[0].open().unwrap();- assert_eq!(dynamic_column.get_cardinality(), Cardinality::Optional);-- // text column- let dynamic_column = cols[1].open().unwrap();- assert_eq!(dynamic_column.get_cardinality(), Cardinality::Optional);-}diff --git a/crates/tantivy/src/columnar/columnar/mod.rs b/crates/tantivy/src/columnar/columnar/mod.rsindex bb9e485b..a33cce8f 100644--- a/crates/tantivy/src/columnar/columnar/mod.rs+++ b/crates/tantivy/src/columnar/columnar/mod.rs@@ -6,8 +6,6 @@ mod writer;pub use column_type::{ColumnType, HasAssociatedColumnType};pub use format_version::{Version, CURRENT_VERSION};-#[cfg(test)]-pub(crate) use merge::ColumnTypeCategory;pub use merge::{merge_columnar, MergeRowOrder, ShuffleMergeOrder, StackMergeOrder};pub use reader::ColumnarReader;pub use writer::ColumnarWriter;diff --git a/crates/tantivy/src/columnar/columnar/reader/mod.rs b/crates/tantivy/src/columnar/columnar/reader/mod.rsindex 662c2f32..95fe48a9 100644--- a/crates/tantivy/src/columnar/columnar/reader/mod.rs+++ b/crates/tantivy/src/columnar/columnar/reader/mod.rs@@ -197,7 +197,7 @@ mod tests {#[test]fn test_list_columns() {let mut columnar_writer = ColumnarWriter::default();- columnar_writer.record_column_type("col1", ColumnType::Str, false);+ columnar_writer.record_column_type("col1", ColumnType::U64, false);columnar_writer.record_column_type("col2", ColumnType::U64, false);let mut buffer = Vec::new();columnar_writer.serialize(1, None, &mut buffer).unwrap();@@ -205,7 +205,7 @@ mod tests {let columns = columnar.list_columns().unwrap();assert_eq!(columns.len(), 2);assert_eq!(&columns[0].0, "col1");- assert_eq!(columns[0].1.column_type(), ColumnType::Str);+ assert_eq!(columns[0].1.column_type(), ColumnType::U64);assert_eq!(&columns[1].0, "col2");assert_eq!(columns[1].1.column_type(), ColumnType::U64);}diff --git a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs b/crates/tantivy/src/columnar/columnar/writer/column_writers.rsindex d66e3501..f9e35fb5 100644--- a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs+++ b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs@@ -71,22 +71,15 @@ impl ColumnWriter {}/// Records a change of the document being recorded.- ///- /// This function will also update the cardinality of the column- /// if necessary.pub(super) fn record<S: SymbolValue>(&mut self, doc: RowId, value: S, arena: &mut MemoryArena) {// Difference between `doc` and the last doc.match delta_with_last_doc(self.last_doc_opt, doc) {- DocumentStep::Same => {- // This is the last encounterred document.- self.cardinality = Cardinality::Multivalued;- }+ DocumentStep::Same => {}DocumentStep::Next => {self.last_doc_opt = Some(doc);self.write_symbol::<S>(ColumnOperation::NewDoc(doc), arena);}DocumentStep::Skipped => {- self.cardinality = self.cardinality.max(Cardinality::Optional);self.last_doc_opt = Some(doc);self.write_symbol::<S>(ColumnOperation::NewDoc(doc), arena);}@@ -94,15 +87,8 @@ impl ColumnWriter {self.write_symbol(ColumnOperation::Value(value), arena);}- // Get the cardinality.- // The overall number of docs in the column is necessary to- // deal with the case where the all docs contain 1 value, except some documents- // at the end of the column.- pub(crate) fn get_cardinality(&self, num_docs: RowId) -> Cardinality {- match delta_with_last_doc(self.last_doc_opt, num_docs) {- DocumentStep::Same | DocumentStep::Next => self.cardinality,- DocumentStep::Skipped => self.cardinality.max(Cardinality::Optional),- }+ pub(crate) fn get_cardinality(&self) -> Cardinality {+ self.cardinality}/// Appends a new symbol to the `ColumnWriter`.@@ -214,8 +200,8 @@ impl NumericalColumnWriter {self.compatible_numerical_types.to_numerical_type()}- pub fn cardinality(&self, num_docs: RowId) -> Cardinality {- self.column_writer.get_cardinality(num_docs)+ pub fn cardinality(&self) -> Cardinality {+ self.column_writer.get_cardinality()}pub fn record_numerical_value(diff --git a/crates/tantivy/src/columnar/columnar/writer/mod.rs b/crates/tantivy/src/columnar/columnar/writer/mod.rsindex 4556047d..65da1134 100644--- a/crates/tantivy/src/columnar/columnar/writer/mod.rs+++ b/crates/tantivy/src/columnar/columnar/writer/mod.rs@@ -4,7 +4,6 @@ mod serializer;mod value_index;use std::io;-use std::net::Ipv6Addr;use crate::common::CountingWriter;use crate::stacker::{Addr, ArenaHashMap, MemoryArena};@@ -13,7 +12,7 @@ pub(crate) use column_writers::CompatibleNumericalTypes;pub(crate) use serializer::ColumnarSerializer;use crate::columnar::column_index::SerializableColumnIndex;-use crate::columnar::column_values::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};+use crate::columnar::column_values::MonotonicallyMappableToU64;use crate::columnar::columnar::column_type::ColumnType;use crate::columnar::columnar::writer::column_writers::{ColumnWriter, NumericalColumnWriter, StrOrBytesColumnWriter,@@ -29,7 +28,6 @@ use crate::columnar::{Cardinality, RowId};struct SpareBuffers {value_index_builders: PreallocatedIndexBuilders,u64_values: Vec<u64>,- ip_addr_values: Vec<Ipv6Addr>,}/// Makes it possible to create a new columnar.@@ -38,9 +36,7 @@ struct SpareBuffers {/// use tantivy::columnar::ColumnarWriter;////// let mut columnar_writer = ColumnarWriter::default();-/// columnar_writer.record_str(0u32 /* doc id */, "product_name", "Red backpack");/// columnar_writer.record_numerical(0u32 /* doc id */, "price", 10u64);-/// columnar_writer.record_str(1u32 /* doc id */, "product_name", "Apple");/// columnar_writer.record_numerical(0u32 /* doc id */, "price", 10.5f64); //< uh oh we ended up mixing integer and floats./// let mut wrt: Vec<u8> = Vec::new();/// columnar_writer.serialize(2u32, None, &mut wrt).unwrap();@@ -145,20 +141,14 @@ impl ColumnarWriter {) {if sort_values_within_row {assert!(- column_type == ColumnType::Bytes || column_type == ColumnType::Str,- "sort_values_within_row is only allowed for Bytes and Str columns",+ column_type == ColumnType::Bytes,+ "sort_values_within_row is only allowed for Bytes columns",);}match column_type {- ColumnType::Str | ColumnType::Bytes => {- let (hash_map, dictionaries) = (- if column_type == ColumnType::Str {- &mut self.str_field_hash_map- } else {- &mut self.bytes_field_hash_map- },- &mut self.dictionaries,- );+ ColumnType::Bytes => {+ let (hash_map, dictionaries) =+ (&mut self.bytes_field_hash_map, &mut self.dictionaries);hash_map.mutate_or_create(column_name.as_bytes(),|column_opt: Option<StrOrBytesColumnWriter>| {@@ -197,10 +187,6 @@ impl ColumnarWriter {},);}- ColumnType::IpAddr => self.ip_addr_field_hash_map.mutate_or_create(- column_name.as_bytes(),- |column_opt: Option<ColumnWriter>| column_opt.unwrap_or_default(),- ),}}@@ -221,18 +207,6 @@ impl ColumnarWriter {);}- pub fn record_ip_addr(&mut self, doc: RowId, column_name: &str, ip_addr: Ipv6Addr) {- let (hash_map, arena) = (&mut self.ip_addr_field_hash_map, &mut self.arena);- hash_map.mutate_or_create(- column_name.as_bytes(),- |column_opt: Option<ColumnWriter>| {- let mut column: ColumnWriter = column_opt.unwrap_or_default();- column.record(doc, ip_addr, arena);- column- },- );- }-pub fn record_bool(&mut self, doc: RowId, column_name: &str, val: bool) {let (hash_map, arena) = (&mut self.bool_field_hash_map, &mut self.arena);hash_map.mutate_or_create(@@ -266,27 +240,6 @@ impl ColumnarWriter {);}- pub fn record_str(&mut self, doc: RowId, column_name: &str, value: &str) {- let (hash_map, arena, dictionaries) = (- &mut self.str_field_hash_map,- &mut self.arena,- &mut self.dictionaries,- );- hash_map.mutate_or_create(- column_name.as_bytes(),- |column_opt: Option<StrOrBytesColumnWriter>| {- let mut column: StrOrBytesColumnWriter = column_opt.unwrap_or_else(|| {- // Each column has its own dictionary- let dictionary_id = dictionaries.len() as u32;- dictionaries.push(DictionaryBuilder::default());- StrOrBytesColumnWriter::with_dictionary_id(dictionary_id)- });- column.record_bytes(doc, value.as_bytes(), dictionaries, arena);- column- },- );- }-pub fn record_bytes(&mut self, doc: RowId, column_name: &str, value: &[u8]) {let (hash_map, arena, dictionaries) = (&mut self.bytes_field_hash_map,@@ -329,21 +282,11 @@ impl ColumnarWriter {.iter().map(|(term, addr)| (term, ColumnType::Bytes, addr)),);- columns.extend(- self.str_field_hash_map- .iter()- .map(|(column_name, addr)| (column_name, ColumnType::Str, addr)),- );columns.extend(self.bool_field_hash_map.iter().map(|(column_name, addr)| (column_name, ColumnType::Bool, addr)),);- columns.extend(- self.ip_addr_field_hash_map- .iter()- .map(|(column_name, addr)| (column_name, ColumnType::IpAddr, addr)),- );columns.extend(self.datetime_field_hash_map.iter()@@ -357,12 +300,11 @@ impl ColumnarWriter {match column_type {ColumnType::Bool => {let column_writer: ColumnWriter = self.bool_field_hash_map.read(addr);- let cardinality = column_writer.get_cardinality(num_docs);+ let cardinality = column_writer.get_cardinality();let mut column_serializer =serializer.start_serialize_column(column_name, column_type);serialize_bool_column(cardinality,- num_docs,column_writer.operation_iterator(arena,old_to_new_row_ids,@@ -373,42 +315,16 @@ impl ColumnarWriter {)?;column_serializer.finalize()?;}- ColumnType::IpAddr => {- let column_writer: ColumnWriter = self.ip_addr_field_hash_map.read(addr);- let cardinality = column_writer.get_cardinality(num_docs);- let mut column_serializer =- serializer.start_serialize_column(column_name, ColumnType::IpAddr);- serialize_ip_addr_column(- cardinality,- num_docs,- column_writer.operation_iterator(- arena,- old_to_new_row_ids,- &mut symbol_byte_buffer,- ),- buffers,- &mut column_serializer,- )?;- column_serializer.finalize()?;- }- ColumnType::Bytes | ColumnType::Str => {+ ColumnType::Bytes => {let str_or_bytes_column_writer: StrOrBytesColumnWriter =- if column_type == ColumnType::Bytes {- self.bytes_field_hash_map.read(addr)- } else {- self.str_field_hash_map.read(addr)- };+ self.bytes_field_hash_map.read(addr);let dictionary_builder =&dictionaries[str_or_bytes_column_writer.dictionary_id as usize];- let cardinality = str_or_bytes_column_writer- .column_writer- .get_cardinality(num_docs);+ let cardinality = str_or_bytes_column_writer.column_writer.get_cardinality();let mut column_serializer =serializer.start_serialize_column(column_name, column_type);serialize_bytes_or_str_column(cardinality,- num_docs,- str_or_bytes_column_writer.sort_values_within_row,dictionary_builder,str_or_bytes_column_writer.operation_iterator(arena,@@ -424,13 +340,12 @@ impl ColumnarWriter {ColumnType::F64 | ColumnType::I64 | ColumnType::U64 => {let numerical_column_writer: NumericalColumnWriter =self.numerical_field_hash_map.read(addr);- let cardinality = numerical_column_writer.cardinality(num_docs);+ let cardinality = numerical_column_writer.cardinality();let mut column_serializer =serializer.start_serialize_column(column_name, column_type);let numerical_type = column_type.numerical_type().unwrap();serialize_numerical_column(cardinality,- num_docs,numerical_type,numerical_column_writer.operation_iterator(arena,@@ -444,12 +359,11 @@ impl ColumnarWriter {}ColumnType::DateTime => {let column_writer: ColumnWriter = self.datetime_field_hash_map.read(addr);- let cardinality = column_writer.get_cardinality(num_docs);+ let cardinality = column_writer.get_cardinality();let mut column_serializer =serializer.start_serialize_column(column_name, ColumnType::DateTime);serialize_numerical_column(cardinality,- num_docs,NumericalType::I64,column_writer.operation_iterator(arena,@@ -473,8 +387,6 @@ impl ColumnarWriter {#[allow(clippy::too_many_arguments)]fn serialize_bytes_or_str_column(cardinality: Cardinality,- num_docs: RowId,- sort_values_within_row: bool,dictionary_builder: &DictionaryBuilder,operation_it: impl Iterator<Item = ColumnOperation<UnorderedId>>,buffers: &mut SpareBuffers,@@ -504,8 +416,6 @@ fn serialize_bytes_or_str_column(send_to_serialize_column_mappable_to_u64(operation_iterator,cardinality,- num_docs,- sort_values_within_row,value_index_builders,u64_values,&mut wrt,@@ -516,7 +426,6 @@ fn serialize_bytes_or_str_column(fn serialize_numerical_column(cardinality: Cardinality,- num_docs: RowId,numerical_type: NumericalType,op_iterator: impl Iterator<Item = ColumnOperation<NumericalValue>>,buffers: &mut SpareBuffers,@@ -532,8 +441,6 @@ fn serialize_numerical_column(send_to_serialize_column_mappable_to_u64(coerce_numerical_symbol::<i64>(op_iterator),cardinality,- num_docs,- false,value_index_builders,u64_values,wrt,@@ -543,8 +450,6 @@ fn serialize_numerical_column(send_to_serialize_column_mappable_to_u64(coerce_numerical_symbol::<u64>(op_iterator),cardinality,- num_docs,- false,value_index_builders,u64_values,wrt,@@ -554,8 +459,6 @@ fn serialize_numerical_column(send_to_serialize_column_mappable_to_u64(coerce_numerical_symbol::<f64>(op_iterator),cardinality,- num_docs,- false,value_index_builders,u64_values,wrt,@@ -567,7 +470,6 @@ fn serialize_numerical_column(fn serialize_bool_column(cardinality: Cardinality,- num_docs: RowId,column_operations_it: impl Iterator<Item = ColumnOperation<bool>>,buffers: &mut SpareBuffers,wrt: &mut impl io::Write,@@ -583,8 +485,6 @@ fn serialize_bool_column(ColumnOperation::Value(bool_val) => ColumnOperation::Value(bool_val.to_u64()),}),cardinality,- num_docs,- false,value_index_builders,u64_values,wrt,@@ -592,88 +492,9 @@ fn serialize_bool_column(Ok(())}-fn serialize_ip_addr_column(- cardinality: Cardinality,- num_docs: RowId,- column_operations_it: impl Iterator<Item = ColumnOperation<Ipv6Addr>>,- buffers: &mut SpareBuffers,- wrt: &mut impl io::Write,-) -> io::Result<()> {- let SpareBuffers {- value_index_builders,- ip_addr_values,- ..- } = buffers;- send_to_serialize_column_mappable_to_u128(- column_operations_it,- cardinality,- num_docs,- value_index_builders,- ip_addr_values,- wrt,- )?;- Ok(())-}--fn send_to_serialize_column_mappable_to_u128<- T: Copy + Ord + std::fmt::Debug + Send + Sync + MonotonicallyMappableToU128 + PartialOrd,->(- op_iterator: impl Iterator<Item = ColumnOperation<T>>,- cardinality: Cardinality,- num_rows: RowId,- value_index_builders: &mut PreallocatedIndexBuilders,- values: &mut Vec<T>,- mut wrt: impl io::Write,-) -> io::Result<()> {- values.clear();- // TODO: split index and values- let serializable_column_index = match cardinality {- Cardinality::Full => {- consume_operation_iterator(- op_iterator,- value_index_builders.borrow_required_index_builder(),- values,- );- SerializableColumnIndex::Full- }- Cardinality::Optional => {- let optional_index_builder = value_index_builders.borrow_optional_index_builder();- consume_operation_iterator(op_iterator, optional_index_builder, values);- let optional_index = optional_index_builder.finish(num_rows);- SerializableColumnIndex::Optional {- num_rows,- non_null_row_ids: Box::new(optional_index),- }- }- Cardinality::Multivalued => {- let multivalued_index_builder = value_index_builders.borrow_multivalued_index_builder();- consume_operation_iterator(op_iterator, multivalued_index_builder, values);- let multivalued_index = multivalued_index_builder.finish(num_rows);- SerializableColumnIndex::Multivalued(Box::new(multivalued_index))- }- };- crate::columnar::column::serialize_column_mappable_to_u128(- serializable_column_index,- &&values[..],- &mut wrt,- )?;- Ok(())-}--fn sort_values_within_row_in_place(multivalued_index: &[RowId], values: &mut [u64]) {- let mut start_index: usize = 0;- for end_index in multivalued_index.iter().copied() {- let end_index = end_index as usize;- values[start_index..end_index].sort_unstable();- start_index = end_index;- }-}-fn send_to_serialize_column_mappable_to_u64(op_iterator: impl Iterator<Item = ColumnOperation<u64>>,cardinality: Cardinality,- num_rows: RowId,- sort_values_within_row: bool,value_index_builders: &mut PreallocatedIndexBuilders,values: &mut Vec<u64>,mut wrt: impl io::Write,@@ -688,24 +509,6 @@ fn send_to_serialize_column_mappable_to_u64();SerializableColumnIndex::Full}- Cardinality::Optional => {- let optional_index_builder = value_index_builders.borrow_optional_index_builder();- consume_operation_iterator(op_iterator, optional_index_builder, values);- let optional_index = optional_index_builder.finish(num_rows);- SerializableColumnIndex::Optional {- non_null_row_ids: Box::new(optional_index),- num_rows,- }- }- Cardinality::Multivalued => {- let multivalued_index_builder = value_index_builders.borrow_multivalued_index_builder();- consume_operation_iterator(op_iterator, multivalued_index_builder, values);- let multivalued_index = multivalued_index_builder.finish(num_rows);- if sort_values_within_row {- sort_values_within_row_in_place(multivalued_index, values);- }- SerializableColumnIndex::Multivalued(Box::new(multivalued_index))- }};crate::columnar::column::serialize_column_mappable_to_u64(serializable_column_index,@@ -761,7 +564,7 @@ mod tests {column_writer.record(0u32, NumericalValue::from(14i64), &mut arena);column_writer.record(1u32, NumericalValue::from(15i64), &mut arena);column_writer.record(2u32, NumericalValue::from(-16i64), &mut arena);- assert_eq!(column_writer.get_cardinality(3), Cardinality::Full);+ assert_eq!(column_writer.get_cardinality(), Cardinality::Full);let mut buffer = Vec::new();let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer.operation_iterator(&arena, None, &mut buffer)@@ -783,69 +586,4 @@ mod tests {ColumnOperation::Value(NumericalValue::I64(-16i64))));}-- #[test]- fn test_column_writer_optional_cardinality_missing_first() {- let mut arena = MemoryArena::default();- let mut column_writer = super::ColumnWriter::default();- column_writer.record(1u32, NumericalValue::from(15i64), &mut arena);- column_writer.record(2u32, NumericalValue::from(-16i64), &mut arena);- assert_eq!(column_writer.get_cardinality(3), Cardinality::Optional);- let mut buffer = Vec::new();- let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer- .operation_iterator(&arena, None, &mut buffer)- .collect();- assert_eq!(symbols.len(), 4);- assert!(matches!(symbols[0], ColumnOperation::NewDoc(1u32)));- assert!(matches!(- symbols[1],- ColumnOperation::Value(NumericalValue::I64(15i64))- ));- assert!(matches!(symbols[2], ColumnOperation::NewDoc(2u32)));- assert!(matches!(- symbols[3],- ColumnOperation::Value(NumericalValue::I64(-16i64))- ));- }-- #[test]- fn test_column_writer_optional_cardinality_missing_last() {- let mut arena = MemoryArena::default();- let mut column_writer = super::ColumnWriter::default();- column_writer.record(0u32, NumericalValue::from(15i64), &mut arena);- assert_eq!(column_writer.get_cardinality(2), Cardinality::Optional);- let mut buffer = Vec::new();- let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer- .operation_iterator(&arena, None, &mut buffer)- .collect();- assert_eq!(symbols.len(), 2);- assert!(matches!(symbols[0], ColumnOperation::NewDoc(0u32)));- assert!(matches!(- symbols[1],- ColumnOperation::Value(NumericalValue::I64(15i64))- ));- }-- #[test]- fn test_column_writer_multivalued() {- let mut arena = MemoryArena::default();- let mut column_writer = super::ColumnWriter::default();- column_writer.record(0u32, NumericalValue::from(16i64), &mut arena);- column_writer.record(0u32, NumericalValue::from(17i64), &mut arena);- assert_eq!(column_writer.get_cardinality(1), Cardinality::Multivalued);- let mut buffer = Vec::new();- let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer- .operation_iterator(&arena, None, &mut buffer)- .collect();- assert_eq!(symbols.len(), 3);- assert!(matches!(symbols[0], ColumnOperation::NewDoc(0u32)));- assert!(matches!(- symbols[1],- ColumnOperation::Value(NumericalValue::I64(16i64))- ));- assert!(matches!(- symbols[2],- ColumnOperation::Value(NumericalValue::I64(17i64))- ));- }}diff --git a/crates/tantivy/src/columnar/columnar/writer/serializer.rs b/crates/tantivy/src/columnar/columnar/writer/serializer.rsindex 4aea5e6d..627fea20 100644--- a/crates/tantivy/src/columnar/columnar/writer/serializer.rs+++ b/crates/tantivy/src/columnar/columnar/writer/serializer.rs@@ -98,18 +98,3 @@ impl<'a, W: io::Write> io::Write for ColumnSerializer<'a, W> {self.columnar_serializer.wrt.write_all(buf)}}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_prepare_key_bytes() {- let mut buffer: Vec<u8> = b"somegarbage".to_vec();- prepare_key(b"root\0child", ColumnType::Str, &mut buffer);- assert_eq!(buffer.len(), 12);- assert_eq!(&buffer[..10], b"root0child");- assert_eq!(buffer[10], 0u8);- assert_eq!(buffer[11], ColumnType::Str.to_code());- }-}diff --git a/crates/tantivy/src/columnar/dynamic_column.rs b/crates/tantivy/src/columnar/dynamic_column.rsindex 5d7118a6..f0674634 100644--- a/crates/tantivy/src/columnar/dynamic_column.rs+++ b/crates/tantivy/src/columnar/dynamic_column.rs@@ -1,14 +1,13 @@-use std::net::Ipv6Addr;use std::sync::Arc;use std::{fmt, io};use crate::common::file_slice::FileSlice;use crate::common::{ByteCount, DateTime, HasLen, OwnedBytes};-use super::column::{BytesColumn, Column, StrColumn};+use super::column::{BytesColumn, Column};use super::column_values::{monotonic_map_column, StrictlyMonotonicFn};use super::columnar::ColumnType;-use super::{Cardinality, ColumnIndex, ColumnValues, NumericalType};+use super::{ColumnIndex, ColumnValues, NumericalType};#[derive(Clone)]pub enum DynamicColumn {@@ -16,24 +15,20 @@ pub enum DynamicColumn {I64(Column<i64>),U64(Column<u64>),F64(Column<f64>),- IpAddr(Column<Ipv6Addr>),DateTime(Column<DateTime>),Bytes(BytesColumn),- Str(StrColumn),}impl fmt::Debug for DynamicColumn {fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {- write!(f, "[{} {} |", self.get_cardinality(), self.column_type())?;+ write!(f, "[{} |", self.column_type())?;match self {DynamicColumn::Bool(col) => write!(f, " {col:?}")?,DynamicColumn::I64(col) => write!(f, " {col:?}")?,DynamicColumn::U64(col) => write!(f, " {col:?}")?,DynamicColumn::F64(col) => write!(f, "{col:?}")?,- DynamicColumn::IpAddr(col) => write!(f, "{col:?}")?,DynamicColumn::DateTime(col) => write!(f, "{col:?}")?,DynamicColumn::Bytes(col) => write!(f, "{col:?}")?,- DynamicColumn::Str(col) => write!(f, "{col:?}")?,}write!(f, "]")}@@ -46,27 +41,19 @@ impl DynamicColumn {DynamicColumn::I64(c) => &c.index,DynamicColumn::U64(c) => &c.index,DynamicColumn::F64(c) => &c.index,- DynamicColumn::IpAddr(c) => &c.index,DynamicColumn::DateTime(c) => &c.index,DynamicColumn::Bytes(c) => &c.ords().index,- DynamicColumn::Str(c) => &c.ords().index,}}- pub fn get_cardinality(&self) -> Cardinality {- self.column_index().get_cardinality()- }-pub fn num_values(&self) -> u32 {match self {DynamicColumn::Bool(c) => c.values.num_vals(),DynamicColumn::I64(c) => c.values.num_vals(),DynamicColumn::U64(c) => c.values.num_vals(),DynamicColumn::F64(c) => c.values.num_vals(),- DynamicColumn::IpAddr(c) => c.values.num_vals(),DynamicColumn::DateTime(c) => c.values.num_vals(),DynamicColumn::Bytes(c) => c.ords().values.num_vals(),- DynamicColumn::Str(c) => c.ords().values.num_vals(),}}@@ -76,10 +63,8 @@ impl DynamicColumn {DynamicColumn::I64(_) => ColumnType::I64,DynamicColumn::U64(_) => ColumnType::U64,DynamicColumn::F64(_) => ColumnType::F64,- DynamicColumn::IpAddr(_) => ColumnType::IpAddr,DynamicColumn::DateTime(_) => ColumnType::DateTime,DynamicColumn::Bytes(_) => ColumnType::Bytes,- DynamicColumn::Str(_) => ColumnType::Str,}}@@ -224,9 +209,7 @@ static_dynamic_conversions!(Column<u64>, U64);static_dynamic_conversions!(Column<i64>, I64);static_dynamic_conversions!(Column<f64>, F64);static_dynamic_conversions!(Column<DateTime>, DateTime);-static_dynamic_conversions!(StrColumn, Str);static_dynamic_conversions!(BytesColumn, Bytes);-static_dynamic_conversions!(Column<Ipv6Addr>, IpAddr);#[derive(Clone, Debug)]pub struct DynamicColumnHandle {@@ -259,14 +242,10 @@ impl DynamicColumnHandle {pub fn open_u64_lenient(&self) -> io::Result<Option<Column<u64>>> {let column_bytes = self.file_slice.read_bytes()?;match self.column_type {- ColumnType::Str | ColumnType::Bytes => {+ ColumnType::Bytes => {let column: BytesColumn = super::column::open_column_bytes(column_bytes)?;Ok(Some(column.term_ord_column))}- ColumnType::IpAddr => {- let column = super::column::open_column_u128_as_compact_u64(column_bytes)?;- Ok(Some(column))- }ColumnType::Bool| ColumnType::I64| ColumnType::U64@@ -281,12 +260,10 @@ impl DynamicColumnHandle {fn open_internal(&self, column_bytes: OwnedBytes) -> io::Result<DynamicColumn> {let dynamic_column: DynamicColumn = match self.column_type {ColumnType::Bytes => super::column::open_column_bytes(column_bytes)?.into(),- ColumnType::Str => super::column::open_column_str(column_bytes)?.into(),ColumnType::I64 => super::column::open_column_u64::<i64>(column_bytes)?.into(),ColumnType::U64 => super::column::open_column_u64::<u64>(column_bytes)?.into(),ColumnType::F64 => super::column::open_column_u64::<f64>(column_bytes)?.into(),ColumnType::Bool => super::column::open_column_u64::<bool>(column_bytes)?.into(),- ColumnType::IpAddr => super::column::open_column_u128::<Ipv6Addr>(column_bytes)?.into(),ColumnType::DateTime => {super::column::open_column_u64::<DateTime>(column_bytes)?.into()}diff --git a/crates/tantivy/src/columnar/mod.rs b/crates/tantivy/src/columnar/mod.rsindex b8101b35..3438be2a 100644--- a/crates/tantivy/src/columnar/mod.rs+++ b/crates/tantivy/src/columnar/mod.rs@@ -33,7 +33,7 @@ mod value;use crate::sstable::VoidSSTable;pub use block_accessor::ColumnBlockAccessor;-pub use column::{BytesColumn, Column, StrColumn};+pub use column::{BytesColumn, Column};pub use column_index::ColumnIndex;pub use column_values::{ColumnValues, EmptyColumnValues, MonotonicallyMappableToU128, MonotonicallyMappableToU64,@@ -80,30 +80,18 @@ pub enum Cardinality {/// `Full` is the default for auto-detecting the Cardinality, since it is the most strict.#[default]Full = 0,- /// All documents contain at most one value.- Optional = 1,- /// All documents may contain any number of values.- Multivalued = 2,}impl Display for Cardinality {fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {let short_str = match self {Cardinality::Full => "full",- Cardinality::Optional => "opt",- Cardinality::Multivalued => "mult",};write!(f, "{short_str}")}}impl Cardinality {- pub fn is_optional(&self) -> bool {- matches!(self, Cardinality::Optional)- }- pub fn is_multivalue(&self) -> bool {- matches!(self, Cardinality::Multivalued)- }pub fn is_full(&self) -> bool {matches!(self, Cardinality::Full)}@@ -113,8 +101,6 @@ impl Cardinality {pub(crate) fn try_from_code(code: u8) -> Result<Cardinality, InvalidData> {match code {0 => Ok(Cardinality::Full),- 1 => Ok(Cardinality::Optional),- 2 => Ok(Cardinality::Multivalued),_ => Err(InvalidData),}}diff --git a/crates/tantivy/src/columnar/tests.rs b/crates/tantivy/src/columnar/tests.rsindex bade724b..d800a34d 100644--- a/crates/tantivy/src/columnar/tests.rs+++ b/crates/tantivy/src/columnar/tests.rs@@ -1,33 +1,6 @@-use std::collections::HashMap;-use std::fmt::Debug;-use std::net::Ipv6Addr;--use crate::common::DateTime;-use proptest::prelude::*;-use proptest::sample::subsequence;--use super::column_values::MonotonicallyMappableToU128;-use super::columnar::{ColumnType, ColumnTypeCategory};use super::dynamic_column::{DynamicColumn, DynamicColumnHandle};-use super::value::{Coerce, NumericalValue};-use super::{- BytesColumn, Cardinality, Column, ColumnarReader, ColumnarWriter, RowAddr, RowId,- ShuffleMergeOrder, StackMergeOrder,-};--#[test]-fn test_dataframe_writer_str() {- let mut dataframe_writer = ColumnarWriter::default();- dataframe_writer.record_str(1u32, "my_string", "hello");- dataframe_writer.record_str(3u32, "my_string", "helloeee");- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer.serialize(5, None, &mut buffer).unwrap();- let columnar = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar.num_columns(), 1);- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();- assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 77);-}+use super::value::NumericalValue;+use super::{ColumnarReader, ColumnarWriter};#[test]fn test_dataframe_writer_bytes() {@@ -40,117 +13,7 @@ fn test_dataframe_writer_bytes() {assert_eq!(columnar.num_columns(), 1);let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 77);-}--#[test]-fn test_dataframe_writer_bool() {- let mut dataframe_writer = ColumnarWriter::default();- dataframe_writer.record_bool(1u32, "bool.value", false);- dataframe_writer.record_bool(3u32, "bool.value", true);- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer.serialize(5, None, &mut buffer).unwrap();- let columnar = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar.num_columns(), 1);- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("bool.value").unwrap();- assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 22);- assert_eq!(cols[0].column_type(), ColumnType::Bool);- let dyn_bool_col = cols[0].open().unwrap();- let DynamicColumn::Bool(bool_col) = dyn_bool_col else {- panic!();- };- let vals: Vec<Option<bool>> = (0..5).map(|row_id| bool_col.first(row_id)).collect();- assert_eq!(&vals, &[None, Some(false), None, Some(true), None,]);-}--#[test]-fn test_dataframe_writer_u64_multivalued() {- let mut dataframe_writer = ColumnarWriter::default();- dataframe_writer.record_numerical(2u32, "divisor", 2u64);- dataframe_writer.record_numerical(3u32, "divisor", 3u64);- dataframe_writer.record_numerical(4u32, "divisor", 2u64);- dataframe_writer.record_numerical(5u32, "divisor", 5u64);- dataframe_writer.record_numerical(6u32, "divisor", 2u64);- dataframe_writer.record_numerical(6u32, "divisor", 3u64);- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer.serialize(7, None, &mut buffer).unwrap();- let columnar = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar.num_columns(), 1);- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("divisor").unwrap();- assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 29);- let dyn_i64_col = cols[0].open().unwrap();- let DynamicColumn::I64(divisor_col) = dyn_i64_col else {- panic!();- };- assert_eq!(- divisor_col.get_cardinality(),- super::Cardinality::Multivalued- );- assert_eq!(divisor_col.num_docs(), 7);-}--#[test]-fn test_dataframe_writer_ip_addr() {- let mut dataframe_writer = ColumnarWriter::default();- dataframe_writer.record_ip_addr(1, "ip_addr", Ipv6Addr::from_u128(1001));- dataframe_writer.record_ip_addr(3, "ip_addr", Ipv6Addr::from_u128(1050));- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer.serialize(5, None, &mut buffer).unwrap();- let columnar = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar.num_columns(), 1);- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("ip_addr").unwrap();- assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 42);- assert_eq!(cols[0].column_type(), ColumnType::IpAddr);- let dyn_bool_col = cols[0].open().unwrap();- let DynamicColumn::IpAddr(ip_col) = dyn_bool_col else {- panic!();- };- let vals: Vec<Option<Ipv6Addr>> = (0..5).map(|row_id| ip_col.first(row_id)).collect();- assert_eq!(- &vals,- &[- None,- Some(Ipv6Addr::from_u128(1001)),- None,- Some(Ipv6Addr::from_u128(1050)),- None,- ]- );-}--#[test]-fn test_dataframe_writer_numerical() {- let mut dataframe_writer = ColumnarWriter::default();- dataframe_writer.record_numerical(1u32, "srical.value", NumericalValue::U64(12u64));- dataframe_writer.record_numerical(2u32, "srical.value", NumericalValue::U64(13u64));- dataframe_writer.record_numerical(4u32, "srical.value", NumericalValue::U64(15u64));- let mut buffer: Vec<u8> = Vec::new();- dataframe_writer.serialize(6, None, &mut buffer).unwrap();- let columnar = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar.num_columns(), 1);- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("srical.value").unwrap();- assert_eq!(cols.len(), 1);- // Right now this 31 bytes are spent as follows- //- // - header 14 bytes- // - vals 8 //< due to padding? could have been 1byte?.- // - null footer 6 bytes- assert_eq!(cols[0].num_bytes(), 33);- let column = cols[0].open().unwrap();- let DynamicColumn::I64(column_i64) = column else {- panic!();- };- assert_eq!(column_i64.index.get_cardinality(), Cardinality::Optional);- assert_eq!(column_i64.first(0), None);- assert_eq!(column_i64.first(1), Some(12i64));- assert_eq!(column_i64.first(2), Some(13i64));- assert_eq!(column_i64.first(3), None);- assert_eq!(column_i64.first(4), Some(15i64));- assert_eq!(column_i64.first(5), None);- assert_eq!(column_i64.first(6), None); //< we can change the spec for that one.+ assert_eq!(cols[0].num_bytes(), 66);}#[test]@@ -162,71 +25,6 @@ fn test_dataframe_sort_by_full() {assert_eq!(data, vec![0, 1]);}-#[test]-fn test_dataframe_sort_by_opt() {- let mut dataframe_writer = ColumnarWriter::default();- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(3));- dataframe_writer.record_numerical(3u32, "value", NumericalValue::U64(2));- let data = dataframe_writer.sort_order("value", 5, false);- // 0, 2, 4 is 0.0- assert_eq!(data, vec![0, 2, 4, 3, 1]);- let data = dataframe_writer.sort_order("value", 5, true);- assert_eq!(- data,- vec![4, 2, 0, 3, 1].into_iter().rev().collect::<Vec<_>>()- );-}--#[test]-fn test_dataframe_sort_by_multi() {- let mut dataframe_writer = ColumnarWriter::default();- // valid for sort- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(2));- // those are ignored for sort- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(4));- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(4));- // valid for sort- dataframe_writer.record_numerical(3u32, "value", NumericalValue::U64(3));- // ignored, would change sort order- dataframe_writer.record_numerical(3u32, "value", NumericalValue::U64(1));- let data = dataframe_writer.sort_order("value", 4, false);- assert_eq!(data, vec![0, 2, 1, 3]);-}--#[test]-fn test_dictionary_encoded_str() {- let mut buffer = Vec::new();- let mut columnar_writer = ColumnarWriter::default();- columnar_writer.record_str(1, "my.column", "a");- columnar_writer.record_str(3, "my.column", "c");- columnar_writer.record_str(3, "my.column2", "different_column!");- columnar_writer.record_str(4, "my.column", "b");- columnar_writer.serialize(5, None, &mut buffer).unwrap();- let columnar_reader = ColumnarReader::open(buffer).unwrap();- assert_eq!(columnar_reader.num_columns(), 2);- let col_handles = columnar_reader.read_columns("my.column").unwrap();- assert_eq!(col_handles.len(), 1);- let DynamicColumn::Str(str_col) = col_handles[0].open().unwrap() else {- panic!();- };- let index: Vec<Option<u64>> = (0..5).map(|row_id| str_col.ords().first(row_id)).collect();- assert_eq!(index, &[None, Some(0), None, Some(2), Some(1)]);- assert_eq!(str_col.num_rows(), 5);- let mut term_buffer = String::new();- let term_ords = str_col.ords();- assert_eq!(term_ords.first(0), None);- assert_eq!(term_ords.first(1), Some(0));- str_col.ord_to_str(0u64, &mut term_buffer).unwrap();- assert_eq!(term_buffer, "a");- assert_eq!(term_ords.first(2), None);- assert_eq!(term_ords.first(3), Some(2));- str_col.ord_to_str(2u64, &mut term_buffer).unwrap();- assert_eq!(term_buffer, "c");- assert_eq!(term_ords.first(4), Some(1));- str_col.ord_to_str(1u64, &mut term_buffer).unwrap();- assert_eq!(term_buffer, "b");-}-#[test]fn test_dictionary_encoded_bytes() {let mut buffer = Vec::new();@@ -243,703 +41,20 @@ fn test_dictionary_encoded_bytes() {let DynamicColumn::Bytes(bytes_col) = col_handles[0].open().unwrap() else {panic!();};- let index: Vec<Option<u64>> = (0..5)- .map(|row_id| bytes_col.ords().first(row_id))- .collect();- assert_eq!(index, &[None, Some(0), None, Some(2), Some(1)]);- assert_eq!(bytes_col.num_rows(), 5);let mut term_buffer = Vec::new();- let term_ords = bytes_col.ords();- assert_eq!(term_ords.first(0), None);- assert_eq!(term_ords.first(1), Some(0));bytes_col.dictionary.ord_to_term(0u64, &mut term_buffer).unwrap();assert_eq!(term_buffer, b"a");- assert_eq!(term_ords.first(2), None);- assert_eq!(term_ords.first(3), Some(2));bytes_col.dictionary.ord_to_term(2u64, &mut term_buffer).unwrap();assert_eq!(term_buffer, b"c");- assert_eq!(term_ords.first(4), Some(1));bytes_col.dictionary.ord_to_term(1u64, &mut term_buffer).unwrap();assert_eq!(term_buffer, b"b");}--fn num_strategy() -> impl Strategy<Value = NumericalValue> {- prop_oneof![- 3 => Just(NumericalValue::U64(0u64)),- 3 => Just(NumericalValue::U64(u64::MAX)),- 3 => Just(NumericalValue::I64(0i64)),- 3 => Just(NumericalValue::I64(i64::MIN)),- 3 => Just(NumericalValue::I64(i64::MAX)),- 3 => Just(NumericalValue::F64(1.2f64)),- 1 => any::<f64>().prop_map(NumericalValue::from),- 1 => any::<u64>().prop_map(NumericalValue::from),- 1 => any::<i64>().prop_map(NumericalValue::from),- ]-}--#[derive(Debug, Clone, Copy)]-enum ColumnValue {- Str(&'static str),- Bytes(&'static [u8]),- Numerical(NumericalValue),- IpAddr(Ipv6Addr),- Bool(bool),- DateTime(DateTime),-}--impl<T: Into<NumericalValue>> From<T> for ColumnValue {- fn from(val: T) -> ColumnValue {- ColumnValue::Numerical(val.into())- }-}--impl ColumnValue {- pub(crate) fn column_type_category(&self) -> ColumnTypeCategory {- match self {- ColumnValue::Str(_) => ColumnTypeCategory::Str,- ColumnValue::Bytes(_) => ColumnTypeCategory::Bytes,- ColumnValue::Numerical(_) => ColumnTypeCategory::Numerical,- ColumnValue::IpAddr(_) => ColumnTypeCategory::IpAddr,- ColumnValue::Bool(_) => ColumnTypeCategory::Bool,- ColumnValue::DateTime(_) => ColumnTypeCategory::DateTime,- }- }-}--fn column_name_strategy() -> impl Strategy<Value = &'static str> {- prop_oneof![Just("c1"), Just("c2")]-}--fn string_strategy() -> impl Strategy<Value = &'static str> {- prop_oneof![Just("a"), Just("b")]-}--fn bytes_strategy() -> impl Strategy<Value = &'static [u8]> {- prop_oneof![Just(&[0u8][..]), Just(&[1u8][..])]-}--// A random column value-fn column_value_strategy() -> impl Strategy<Value = ColumnValue> {- prop_oneof![- 10 => string_strategy().prop_map(ColumnValue::Str),- 1 => bytes_strategy().prop_map(ColumnValue::Bytes),- 40 => num_strategy().prop_map(ColumnValue::Numerical),- 1 => (1u16..3u16).prop_map(|ip_addr_byte| ColumnValue::IpAddr(Ipv6Addr::new(- 127,- 0,- 0,- 0,- 0,- 0,- 0,- ip_addr_byte- ))),- 1 => any::<bool>().prop_map(ColumnValue::Bool),- 1 => (679_723_993i64..1_679_723_995i64)- .prop_map(|val| { ColumnValue::DateTime(DateTime::from_timestamp_secs(val)) })- ]-}--// A document contains up to 4 values.-fn doc_strategy() -> impl Strategy<Value = Vec<(&'static str, ColumnValue)>> {- proptest::collection::vec((column_name_strategy(), column_value_strategy()), 0..=4)-}--fn num_docs_strategy() -> impl Strategy<Value = usize> {- prop_oneof!(- // We focus heavily on the 0..2 case as we assume it is sufficient to cover all edge cases.- 0usize..=3usize,- // We leave 50% of the effort exploring more defensively.- 3usize..=12usize- )-}--// A columnar contains up to 2 docs.-fn columnar_docs_strategy() -> impl Strategy<Value = Vec<Vec<(&'static str, ColumnValue)>>> {- num_docs_strategy()- .prop_flat_map(|num_docs| proptest::collection::vec(doc_strategy(), num_docs))-}--fn columnar_docs_and_mapping_strategy(-) -> impl Strategy<Value = (Vec<Vec<(&'static str, ColumnValue)>>, Vec<RowId>)> {- columnar_docs_strategy().prop_flat_map(|docs| {- permutation_strategy(docs.len()).prop_map(move |permutation| (docs.clone(), permutation))- })-}--fn permutation_strategy(n: usize) -> impl Strategy<Value = Vec<RowId>> {- Just((0u32..n as RowId).collect()).prop_shuffle()-}--fn permutation_and_subset_strategy(n: usize) -> impl Strategy<Value = Vec<usize>> {- let vals: Vec<usize> = (0..n).collect();- subsequence(vals, 0..=n).prop_shuffle()-}--fn build_columnar_with_mapping(- docs: &[Vec<(&'static str, ColumnValue)>],- old_to_new_row_ids_opt: Option<&[RowId]>,-) -> ColumnarReader {- let num_docs = docs.len() as u32;- let mut buffer = Vec::new();- let mut columnar_writer = ColumnarWriter::default();- for (doc_id, vals) in docs.iter().enumerate() {- for (column_name, col_val) in vals {- match *col_val {- ColumnValue::Str(str_val) => {- columnar_writer.record_str(doc_id as u32, column_name, str_val);- }- ColumnValue::Bytes(bytes) => {- columnar_writer.record_bytes(doc_id as u32, column_name, bytes)- }- ColumnValue::Numerical(num) => {- columnar_writer.record_numerical(doc_id as u32, column_name, num);- }- ColumnValue::IpAddr(ip_addr) => {- columnar_writer.record_ip_addr(doc_id as u32, column_name, ip_addr);- }- ColumnValue::Bool(bool_val) => {- columnar_writer.record_bool(doc_id as u32, column_name, bool_val);- }- ColumnValue::DateTime(date_time) => {- columnar_writer.record_datetime(doc_id as u32, column_name, date_time);- }- }- }- }- columnar_writer- .serialize(num_docs, old_to_new_row_ids_opt, &mut buffer)- .unwrap();-- ColumnarReader::open(buffer).unwrap()-}--fn build_columnar(docs: &[Vec<(&'static str, ColumnValue)>]) -> ColumnarReader {- build_columnar_with_mapping(docs, None)-}--fn assert_columnar_eq_strict(left: &ColumnarReader, right: &ColumnarReader) {- assert_columnar_eq(left, right, false);-}--fn assert_columnar_eq(- left: &ColumnarReader,- right: &ColumnarReader,- lenient_on_numerical_value: bool,-) {- assert_eq!(left.num_rows(), right.num_rows());- let left_columns = left.list_columns().unwrap();- let right_columns = right.list_columns().unwrap();- assert_eq!(left_columns.len(), right_columns.len());- for i in 0..left_columns.len() {- assert_eq!(left_columns[i].0, right_columns[i].0);- let left_column = left_columns[i].1.open().unwrap();- let right_column = right_columns[i].1.open().unwrap();- assert_dyn_column_eq(&left_column, &right_column, lenient_on_numerical_value);- }-}--fn assert_column_eq<T: Copy + PartialOrd + Debug + Send + Sync + 'static>(- left: &Column<T>,- right: &Column<T>,-) {- assert_eq!(left.get_cardinality(), right.get_cardinality());- assert_eq!(left.num_docs(), right.num_docs());- let num_docs = left.num_docs();- for doc in 0..num_docs {- assert_eq!(- left.index.value_row_ids(doc),- right.index.value_row_ids(doc)- );- }- assert_eq!(left.values.num_vals(), right.values.num_vals());- let num_vals = left.values.num_vals();- for i in 0..num_vals {- assert_eq!(left.values.get_val(i), right.values.get_val(i));- }-}--fn assert_bytes_column_eq(left: &BytesColumn, right: &BytesColumn) {- assert_eq!(- left.term_ord_column.get_cardinality(),- right.term_ord_column.get_cardinality()- );- assert_eq!(left.num_rows(), right.num_rows());- assert_column_eq(&left.term_ord_column, &right.term_ord_column);- assert_eq!(left.dictionary.num_terms(), right.dictionary.num_terms());- let num_terms = left.dictionary.num_terms();- let mut left_terms = left.dictionary.stream().unwrap();- let mut right_terms = right.dictionary.stream().unwrap();- for _ in 0..num_terms {- assert!(left_terms.advance());- assert!(right_terms.advance());- assert_eq!(left_terms.key(), right_terms.key());- }- assert!(!left_terms.advance());- assert!(!right_terms.advance());-}--fn assert_dyn_column_eq(- left_dyn_column: &DynamicColumn,- right_dyn_column: &DynamicColumn,- lenient_on_numerical_value: bool,-) {- assert_eq!(- &left_dyn_column.get_cardinality(),- &right_dyn_column.get_cardinality()- );- match &(left_dyn_column, right_dyn_column) {- (DynamicColumn::Bool(left_col), DynamicColumn::Bool(right_col)) => {- assert_column_eq(left_col, right_col);- }- (DynamicColumn::I64(left_col), DynamicColumn::I64(right_col)) => {- assert_column_eq(left_col, right_col);- }- (DynamicColumn::U64(left_col), DynamicColumn::U64(right_col)) => {- assert_column_eq(left_col, right_col);- }- (DynamicColumn::F64(left_col), DynamicColumn::F64(right_col)) => {- assert_column_eq(left_col, right_col);- }- (DynamicColumn::DateTime(left_col), DynamicColumn::DateTime(right_col)) => {- assert_column_eq(left_col, right_col);- }- (DynamicColumn::IpAddr(left_col), DynamicColumn::IpAddr(right_col)) => {- assert_column_eq(left_col, right_col);- }- (DynamicColumn::Bytes(left_col), DynamicColumn::Bytes(right_col)) => {- assert_bytes_column_eq(left_col, right_col);- }- (DynamicColumn::Str(left_col), DynamicColumn::Str(right_col)) => {- assert_bytes_column_eq(left_col, right_col);- }- (left, right) => {- if lenient_on_numerical_value {- assert_eq!(- ColumnTypeCategory::from(left.column_type()),- ColumnTypeCategory::from(right.column_type())- );- } else {- panic!(- "Column type are not the same: {:?} vs {:?}",- left.column_type(),- right.column_type()- );- }- }- }-}--trait AssertEqualToColumnValue {- fn assert_equal_to_column_value(&self, column_value: &ColumnValue);-}--impl AssertEqualToColumnValue for bool {- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {- let ColumnValue::Bool(val) = column_value else {- panic!()- };- assert_eq!(self, val);- }-}--impl AssertEqualToColumnValue for Ipv6Addr {- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {- let ColumnValue::IpAddr(val) = column_value else {- panic!()- };- assert_eq!(self, val);- }-}--impl<T: Coerce + PartialEq + Debug + Into<NumericalValue>> AssertEqualToColumnValue for T {- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {- let ColumnValue::Numerical(num) = column_value else {- panic!()- };- assert_eq!(self, &T::coerce(*num));- }-}--impl AssertEqualToColumnValue for DateTime {- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {- let ColumnValue::DateTime(dt) = column_value else {- panic!()- };- assert_eq!(self, dt);- }-}--fn assert_column_values<- T: AssertEqualToColumnValue + PartialEq + Copy + PartialOrd + Debug + Send + Sync + 'static,->(- col: &Column<T>,- expected: &HashMap<u32, Vec<&ColumnValue>>,-) {- let mut num_non_empty_rows = 0;- for doc in 0..col.num_docs() {- let doc_vals: Vec<T> = col.values_for_doc(doc).collect();- if doc_vals.is_empty() {- continue;- }- num_non_empty_rows += 1;- let expected_vals = expected.get(&doc).unwrap();- assert_eq!(doc_vals.len(), expected_vals.len());- for (val, &expected) in doc_vals.iter().zip(expected_vals.iter()) {- val.assert_equal_to_column_value(expected)- }- }- assert_eq!(num_non_empty_rows, expected.len());-}--fn assert_bytes_column_values(- col: &BytesColumn,- expected: &HashMap<u32, Vec<&ColumnValue>>,- is_str: bool,-) {- let mut num_non_empty_rows = 0;- let mut buffer = Vec::new();- for doc in 0..col.term_ord_column.num_docs() {- let doc_vals: Vec<u64> = col.term_ords(doc).collect();- if doc_vals.is_empty() {- continue;- }- let expected_vals = expected.get(&doc).unwrap();- assert_eq!(doc_vals.len(), expected_vals.len());- for (&expected_col_val, &ord) in expected_vals.iter().zip(&doc_vals) {- col.ord_to_bytes(ord, &mut buffer).unwrap();- match expected_col_val {- ColumnValue::Str(str_val) => {- assert!(is_str);- assert_eq!(str_val.as_bytes(), &buffer);- }- ColumnValue::Bytes(bytes_val) => {- assert!(!is_str);- assert_eq!(bytes_val, &buffer);- }- _ => {- panic!();- }- }- }- num_non_empty_rows += 1;- }- assert_eq!(num_non_empty_rows, expected.len());-}--// This proptest attempts to create a tiny columnar based of up to 3 rows, and checks that the-// resulting columnar matches the row data.-proptest! {- #![proptest_config(ProptestConfig::with_cases(500))]- #[test]- fn test_single_columnar_builder_proptest(docs in columnar_docs_strategy()) {- let columnar = build_columnar(&docs[..]);- assert_eq!(columnar.num_rows() as usize, docs.len());- let mut expected_columns: HashMap<(&str, ColumnTypeCategory), HashMap<u32, Vec<&ColumnValue>> > = Default::default();- for (doc_id, doc_vals) in docs.iter().enumerate() {- for (col_name, col_val) in doc_vals {- expected_columns- .entry((col_name, col_val.column_type_category()))- .or_default()- .entry(doc_id as u32)- .or_default()- .push(col_val);- }- }- let column_list = columnar.list_columns().unwrap();- assert_eq!(expected_columns.len(), column_list.len());- for (column_name, column) in column_list {- let dynamic_column = column.open().unwrap();- let col_category: ColumnTypeCategory = dynamic_column.column_type().into();- let expected_col_values: &HashMap<u32, Vec<&ColumnValue>> = expected_columns.get(&(column_name.as_str(), col_category)).unwrap();- match &dynamic_column {- DynamicColumn::Bool(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::I64(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::U64(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::F64(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::IpAddr(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::DateTime(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::Bytes(col) =>- assert_bytes_column_values(col, expected_col_values, false),- DynamicColumn::Str(col) =>- assert_bytes_column_values(col, expected_col_values, true),- }- }- }-}--// Same as `test_single_columnar_builder_proptest` but with a shuffling mapping.-proptest! {- #![proptest_config(ProptestConfig::with_cases(500))]- #[test]- fn test_single_columnar_builder_with_shuffle_proptest((docs, mapping) in columnar_docs_and_mapping_strategy()) {- let columnar = build_columnar_with_mapping(&docs[..], Some(&mapping));- assert_eq!(columnar.num_rows() as usize, docs.len());- let mut expected_columns: HashMap<(&str, ColumnTypeCategory), HashMap<u32, Vec<&ColumnValue>> > = Default::default();- for (doc_id, doc_vals) in docs.iter().enumerate() {- for (col_name, col_val) in doc_vals {- expected_columns- .entry((col_name, col_val.column_type_category()))- .or_default()- .entry(mapping[doc_id])- .or_default()- .push(col_val);- }- }- let column_list = columnar.list_columns().unwrap();- assert_eq!(expected_columns.len(), column_list.len());- for (column_name, column) in column_list {- let dynamic_column = column.open().unwrap();- let col_category: ColumnTypeCategory = dynamic_column.column_type().into();- let expected_col_values: &HashMap<u32, Vec<&ColumnValue>> = expected_columns.get(&(column_name.as_str(), col_category)).unwrap();- for _doc_id in 0..columnar.num_rows() {- match &dynamic_column {- DynamicColumn::Bool(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::I64(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::U64(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::F64(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::IpAddr(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::DateTime(col) =>- assert_column_values(col, expected_col_values),- DynamicColumn::Bytes(col) =>- assert_bytes_column_values(col, expected_col_values, false),- DynamicColumn::Str(col) =>- assert_bytes_column_values(col, expected_col_values, true),- }- }- }- }-}--// This tests create 2 or 3 random small columnar and attempts to merge them.-// It compares the resulting merged dataframe with what would have been obtained by building the-// dataframe from the concatenated rows to begin with.-proptest! {- #![proptest_config(ProptestConfig::with_cases(1000))]- #[test]- fn test_columnar_merge_proptest(columnar_docs in proptest::collection::vec(columnar_docs_strategy(), 2..=3)) {- let columnar_readers: Vec<ColumnarReader> = columnar_docs.iter()- .map(|docs| build_columnar(&docs[..]))- .collect::<Vec<_>>();- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();- let mut output: Vec<u8> = Vec::new();- let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]).into();- super::merge_columnar(&columnar_readers_arr[..], &[], stack_merge_order, &mut output).unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> = columnar_docs.iter().flatten().cloned().collect();- let expected_merged_columnar = build_columnar(&concat_rows[..]);- assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);- }-}--#[test]-fn test_columnar_merging_empty_columnar() {- let columnar_docs: Vec<Vec<Vec<(&str, ColumnValue)>>> =- vec![vec![], vec![vec![("c1", ColumnValue::Str("a"))]]];- let columnar_readers: Vec<ColumnarReader> = columnar_docs- .iter()- .map(|docs| build_columnar(&docs[..]))- .collect::<Vec<_>>();- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();- let mut output: Vec<u8> = Vec::new();- let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]);- super::merge_columnar(- &columnar_readers_arr[..],- &[],- super::MergeRowOrder::Stack(stack_merge_order),- &mut output,- )- .unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =- columnar_docs.iter().flatten().cloned().collect();- let expected_merged_columnar = build_columnar(&concat_rows[..]);- assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);-}--#[test]-fn test_columnar_merging_number_columns() {- let columnar_docs: Vec<Vec<Vec<(&str, ColumnValue)>>> = vec![- // columnar 1- vec![- // doc 1.1- vec![("c2", ColumnValue::Numerical(0i64.into()))],- ],- // columnar2- vec![- // doc 2.1- vec![("c2", ColumnValue::Numerical(0u64.into()))],- // doc 2.2- vec![("c2", ColumnValue::Numerical(u64::MAX.into()))],- ],- ];- let columnar_readers: Vec<ColumnarReader> = columnar_docs- .iter()- .map(|docs| build_columnar(&docs[..]))- .collect::<Vec<_>>();- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();- let mut output: Vec<u8> = Vec::new();- let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]);- super::merge_columnar(- &columnar_readers_arr[..],- &[],- super::MergeRowOrder::Stack(stack_merge_order),- &mut output,- )- .unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =- columnar_docs.iter().flatten().cloned().collect();- let expected_merged_columnar = build_columnar(&concat_rows[..]);- assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);-}--// TODO add non trivial remap and merge-// TODO test required_columns-// TODO document edge case: required_columns incompatible with values.--fn columnar_docs_and_remap(-) -> impl Strategy<Value = (Vec<Vec<Vec<(&'static str, ColumnValue)>>>, Vec<RowAddr>)> {- proptest::collection::vec(columnar_docs_strategy(), 2..=3).prop_flat_map(- |columnars_docs: Vec<Vec<Vec<(&str, ColumnValue)>>>| {- let row_addrs: Vec<RowAddr> = columnars_docs- .iter()- .enumerate()- .flat_map(|(segment_ord, columnar_docs)| {- (0u32..columnar_docs.len() as u32).map(move |row_id| RowAddr {- segment_ord: segment_ord as u32,- row_id,- })- })- .collect();- permutation_and_subset_strategy(row_addrs.len()).prop_map(move |shuffled_subset| {- let shuffled_row_addr_subset: Vec<RowAddr> =- shuffled_subset.iter().map(|ord| row_addrs[*ord]).collect();- (columnars_docs.clone(), shuffled_row_addr_subset)- })- },- )-}--proptest! {- #![proptest_config(ProptestConfig::with_cases(1000))]- #[test]- fn test_columnar_merge_and_remap_proptest((columnar_docs, shuffle_merge_order) in columnar_docs_and_remap()) {- let shuffled_rows: Vec<Vec<(&'static str, ColumnValue)>> = shuffle_merge_order.iter()- .map(|row_addr| columnar_docs[row_addr.segment_ord as usize][row_addr.row_id as usize].clone())- .collect();- let expected_merged_columnar = build_columnar(&shuffled_rows[..]);- let columnar_readers: Vec<ColumnarReader> = columnar_docs.iter()- .map(|docs| build_columnar(&docs[..]))- .collect::<Vec<_>>();- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();- let mut output: Vec<u8> = Vec::new();- let segment_num_rows: Vec<RowId> = columnar_docs.iter().map(|docs| docs.len() as RowId).collect();- let shuffle_merge_order = ShuffleMergeOrder::for_test(&segment_num_rows, shuffle_merge_order);- super::merge_columnar(&columnar_readers_arr[..], &[], shuffle_merge_order.into(), &mut output).unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- assert_columnar_eq(&merged_columnar, &expected_merged_columnar, true);- }-}--#[test]-fn test_columnar_merge_empty() {- let columnar_reader_1 = build_columnar(&[]);- let rows: &[Vec<_>] = &[vec![("c1", ColumnValue::Str("a"))]][..];- let columnar_reader_2 = build_columnar(rows);- let mut output: Vec<u8> = Vec::new();- let segment_num_rows: Vec<RowId> = vec![0, 0];- let shuffle_merge_order = ShuffleMergeOrder::for_test(&segment_num_rows, vec![]);- super::merge_columnar(- &[&columnar_reader_1, &columnar_reader_2],- &[],- shuffle_merge_order.into(),- &mut output,- )- .unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- assert_eq!(merged_columnar.num_rows(), 0);- assert_eq!(merged_columnar.num_columns(), 0);-}--#[test]-fn test_columnar_merge_single_str_column() {- let columnar_reader_1 = build_columnar(&[]);- let rows: &[Vec<_>] = &[vec![("c1", ColumnValue::Str("a"))]][..];- let columnar_reader_2 = build_columnar(rows);- let mut output: Vec<u8> = Vec::new();- let segment_num_rows: Vec<RowId> = vec![0, 1];- let shuffle_merge_order = ShuffleMergeOrder::for_test(- &segment_num_rows,- vec![RowAddr {- segment_ord: 1u32,- row_id: 0u32,- }],- );- super::merge_columnar(- &[&columnar_reader_1, &columnar_reader_2],- &[],- shuffle_merge_order.into(),- &mut output,- )- .unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- assert_eq!(merged_columnar.num_rows(), 1);- assert_eq!(merged_columnar.num_columns(), 1);-}--#[test]-fn test_delete_decrease_cardinality() {- let columnar_reader_1 = build_columnar(&[]);- let rows: &[Vec<_>] = &[- vec![- ("c", ColumnValue::from(0i64)),- ("c", ColumnValue::from(0i64)),- ],- vec![("c", ColumnValue::from(0i64))],- ][..];- // c is multivalued here- let columnar_reader_2 = build_columnar(rows);- let mut output: Vec<u8> = Vec::new();- let shuffle_merge_order = ShuffleMergeOrder::for_test(- &[0, 2],- vec![RowAddr {- segment_ord: 1u32,- row_id: 1u32,- }],- );- super::merge_columnar(- &[&columnar_reader_1, &columnar_reader_2],- &[],- shuffle_merge_order.into(),- &mut output,- )- .unwrap();- let merged_columnar = ColumnarReader::open(output).unwrap();- assert_eq!(merged_columnar.num_rows(), 1);- assert_eq!(merged_columnar.num_columns(), 1);- let cols = merged_columnar.read_columns("c").unwrap();- assert_eq!(cols.len(), 1);- assert_eq!(cols[0].column_type(), ColumnType::I64);- assert_eq!(cols[0].open().unwrap().get_cardinality(), Cardinality::Full);-}diff --git a/crates/tantivy/src/core/json_utils.rs b/crates/tantivy/src/core/json_utils.rsindex ee480b03..72d8c9e8 100644--- a/crates/tantivy/src/core/json_utils.rs+++ b/crates/tantivy/src/core/json_utils.rs@@ -197,9 +197,6 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(ReferenceValueLeaf::Bytes(_) => {unimplemented!("Bytes support in dynamic fields is not yet implemented")}- ReferenceValueLeaf::Facet(_) => {- unimplemented!("Facet support in dynamic fields is not yet implemented")- }ReferenceValueLeaf::IpAddr(_) => {unimplemented!("IP address support in dynamic fields is not yet implemented")}diff --git a/crates/tantivy/src/fastfield/facet_reader.rs b/crates/tantivy/src/fastfield/facet_reader.rsdeleted file mode 100644index 0fde3717..00000000--- a/crates/tantivy/src/fastfield/facet_reader.rs+++ /dev/null@@ -1,195 +0,0 @@-use crate::columnar::StrColumn;--use crate::schema::Facet;-use crate::termdict::TermOrdinal;-use crate::DocId;--/// The facet reader makes it possible to access the list of-/// facets associated with a given document in a specific-/// segment.-///-/// Rather than manipulating `Facet` object directly, the API-/// exposes those in the form of list of `Facet` ordinal.-///-/// A segment ordinal can then be translated into a facet via-/// `.facet_from_ord(...)`.-///-/// Facet ordinals are defined as their position in the sorted-/// list of facets. This ordinal is segment local and-/// only makes sense for a given segment.-pub struct FacetReader {- facet_column: StrColumn,-}--impl FacetReader {- /// Creates a new `FacetReader`.- ///- /// A facet reader just wraps :- /// - a `MultiValuedFastFieldReader` that makes it possible to- /// access the list of facet ords for a given document.- /// - a `TermDictionary` that helps associating a facet to- /// an ordinal and vice versa.- pub fn new(facet_column: StrColumn) -> FacetReader {- FacetReader { facet_column }- }-- /// Returns the size of the sets of facets in the segment.- /// This does not take in account the documents that may be marked- /// as deleted.- ///- /// `Facet` ordinals range from `0` to `num_facets() - 1`.- pub fn num_facets(&self) -> usize {- self.facet_column.num_terms()- }-- /// Given a term ordinal returns the term associated with it.- pub fn facet_from_ord(&self, facet_ord: TermOrdinal, output: &mut Facet) -> crate::Result<()> {- let found_term = self.facet_column.ord_to_str(facet_ord, &mut output.0)?;- assert!(found_term, "Term ordinal {facet_ord} no found.");- Ok(())- }-- /// Return the list of facet ordinals associated with a document.- pub fn facet_ords(&self, doc: DocId) -> impl Iterator<Item = u64> + '_ {- self.facet_column.ords().values_for_doc(doc)- }-- /// Accessor to the facet dictionary.- pub fn facet_dict(&self) -> &crate::columnar::Dictionary {- self.facet_column.dictionary()- }-}--#[cfg(test)]-mod tests {- use crate::schema::{Facet, FacetOptions, SchemaBuilder, Value, STORED};- use crate::{DocAddress, Index, IndexWriter, TantivyDocument};-- #[test]- fn test_facet_only_indexed() {- let mut schema_builder = SchemaBuilder::default();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))- .unwrap();- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();- let mut facet_ords = Vec::new();- facet_ords.extend(facet_reader.facet_ords(0u32));- assert_eq!(&facet_ords, &[0u64]);- assert_eq!(facet_reader.num_facets(), 1);- let mut facet = Facet::default();- facet_reader.facet_from_ord(0, &mut facet).unwrap();- assert_eq!(facet.to_path_string(), "/a/b");- let doc = searcher- .doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))- .unwrap();- let value = doc- .get_first(facet_field)- .and_then(|v| v.as_value().as_facet());- assert_eq!(value, None);- }-- #[test]- fn test_facet_several_facets_sorted() {- let mut schema_builder = SchemaBuilder::default();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(facet_field=>Facet::from_text("/parent/child1").unwrap()))- .unwrap();- index_writer- .add_document(doc!(- facet_field=>Facet::from_text("/parent/child2").unwrap(),- facet_field=>Facet::from_text("/parent/child1/blop").unwrap(),- ))- .unwrap();- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();- let mut facet_ords = Vec::new();-- facet_ords.extend(facet_reader.facet_ords(0u32));- assert_eq!(&facet_ords, &[0u64]);-- facet_ords.clear();- facet_ords.extend(facet_reader.facet_ords(1u32));- assert_eq!(&facet_ords, &[1u64, 2u64]);-- assert_eq!(facet_reader.num_facets(), 3);- let mut facet = Facet::default();- facet_reader.facet_from_ord(0, &mut facet).unwrap();- assert_eq!(facet.to_path_string(), "/parent/child1");- facet_reader.facet_from_ord(1, &mut facet).unwrap();- assert_eq!(facet.to_path_string(), "/parent/child1/blop");- facet_reader.facet_from_ord(2, &mut facet).unwrap();- assert_eq!(facet.to_path_string(), "/parent/child2");- }-- #[test]- fn test_facet_stored_and_indexed() -> crate::Result<()> {- let mut schema_builder = SchemaBuilder::default();- let facet_field = schema_builder.add_facet_field("facet", STORED);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer = index.writer_for_tests()?;- index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;- index_writer.commit()?;- let searcher = index.reader()?.searcher();- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();- let mut facet_ords = Vec::new();- facet_ords.extend(facet_reader.facet_ords(0u32));- assert_eq!(&facet_ords, &[0u64]);- let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))?;- let value: Option<Facet> = doc- .get_first(facet_field)- .and_then(|v| v.as_facet())- .map(|facet| Facet::from_encoded_string(facet.to_string()));- assert_eq!(value, Facet::from_text("/a/b").ok());- Ok(())- }-- #[test]- fn test_facet_not_populated_for_all_docs() -> crate::Result<()> {- let mut schema_builder = SchemaBuilder::default();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer = index.writer_for_tests()?;- index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;- index_writer.add_document(TantivyDocument::default())?;- index_writer.commit()?;- let searcher = index.reader()?.searcher();- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();- let mut facet_ords = Vec::new();- facet_ords.extend(facet_reader.facet_ords(0u32));- assert_eq!(&facet_ords, &[0u64]);- facet_ords.clear();- facet_ords.extend(facet_reader.facet_ords(1u32));- assert!(facet_ords.is_empty());- Ok(())- }-- #[test]- fn test_facet_not_populated_for_any_docs() -> crate::Result<()> {- let mut schema_builder = SchemaBuilder::default();- schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer = index.writer_for_tests()?;- index_writer.add_document(TantivyDocument::default())?;- index_writer.add_document(TantivyDocument::default())?;- index_writer.commit()?;- let searcher = index.reader()?.searcher();- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();- assert!(facet_reader.facet_ords(0u32).next().is_none());- assert!(facet_reader.facet_ords(1u32).next().is_none());- Ok(())- }-}diff --git a/crates/tantivy/src/fastfield/mod.rs b/crates/tantivy/src/fastfield/mod.rsindex 6b1074b5..ca7a981a 100644--- a/crates/tantivy/src/fastfield/mod.rs+++ b/crates/tantivy/src/fastfield/mod.rs@@ -8,7 +8,7 @@//! fields given a document id.//!//! Fast fields are useful when a field is required for all or most of-//! the `DocSet`: for instance for scoring, grouping, aggregation, filtering, or faceting.+//! the `DocSet`: for instance for scoring, grouping, aggregation or filtering//!//!//! Fields have to be declared as `FAST` in the schema.@@ -24,7 +24,6 @@ use crate::columnar::MonotonicallyMappableToU64;pub use self::alive_bitset::{intersect_alive_bitsets, write_alive_bitset, AliveBitSet};pub use self::error::{FastFieldNotAvailableError, Result};-pub use self::facet_reader::FacetReader;pub use self::readers::FastFieldReaders;pub use self::writer::FastFieldsWriter;use crate::schema::Type;@@ -32,7 +31,6 @@ use crate::DateTime;mod alive_bitset;mod error;-mod facet_reader;mod readers;mod writer;@@ -75,11 +73,9 @@ impl FastValue for DateTime {#[cfg(test)]mod tests {- use std::net::Ipv6Addr;- use std::ops::{Range, RangeInclusive};+ use std::ops::RangeInclusive;use std::path::Path;- use crate::columnar::StrColumn;use crate::common::{ByteCount, DateTimePrecision, HasLen, TerminatingWrite};use once_cell::sync::Lazy;use rand::prelude::SliceRandom;@@ -91,11 +87,10 @@ mod tests {use crate::index::SegmentId;use crate::merge_policy::NoMergePolicy;use crate::schema::{- DateOptions, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder,- TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,+ DateOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument, FAST,+ INDEXED,};use crate::time::OffsetDateTime;- use crate::tokenizer::{LowerCaser, RawTokenizer, TextAnalyzer, TokenizerManager};use crate::{Index, IndexWriter, SegmentReader};pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {@@ -134,10 +129,7 @@ mod tests {assert_eq!(file.len(), 80);let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let column = fast_field_readers- .u64("field")- .unwrap()- .first_or_default_col(0);+ let column = fast_field_readers.u64("field").unwrap().values;assert_eq!(column.get_val(0), 13u64);assert_eq!(column.get_val(1), 14u64);assert_eq!(column.get_val(2), 2u64);@@ -184,10 +176,7 @@ mod tests {let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 108);let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let col = fast_field_readers- .u64("field")- .unwrap()- .first_or_default_col(0);+ let col = fast_field_readers.u64("field").unwrap().values;assert_eq!(col.get_val(0), 4u64);assert_eq!(col.get_val(1), 14_082_001u64);assert_eq!(col.get_val(2), 3_052u64);@@ -217,10 +206,7 @@ mod tests {let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 81);let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let fast_field_reader = fast_field_readers- .u64("field")- .unwrap()- .first_or_default_col(0);+ let fast_field_reader = fast_field_readers.u64("field").unwrap().values;for doc in 0..10_000 {assert_eq!(fast_field_reader.get_val(doc), 100_000u64);}@@ -250,10 +236,7 @@ mod tests {assert_eq!(file.len(), 4476);{let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let col = fast_field_readers- .u64("field")- .unwrap()- .first_or_default_col(0);+ let col = fast_field_readers.u64("field").unwrap().values;for doc in 1..10_000 {assert_eq!(col.get_val(doc), 5_000_000_000_000_000_000u64 + doc as u64);}@@ -284,10 +267,7 @@ mod tests {{let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();- let col = fast_field_readers- .i64("field")- .unwrap()- .first_or_default_col(0);+ let col = fast_field_readers.i64("field").unwrap().values;assert_eq!(col.min_value(), -100i64);assert_eq!(col.max_value(), 9_999i64);for (doc, i) in (-100i64..10_000i64).enumerate() {@@ -302,65 +282,6 @@ mod tests {Ok(())}- #[test]- fn test_signed_intfastfield_default_val() {- let path = Path::new("test");- let directory: RamDirectory = RamDirectory::create();- let mut schema_builder = Schema::builder();- schema_builder.add_i64_field("field", FAST);- let schema = schema_builder.build();-- {- let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();- let doc = TantivyDocument::default();- fast_field_writers.add_document(&doc).unwrap();- fast_field_writers.serialize(&mut write, None).unwrap();- write.terminate().unwrap();- }-- let file = directory.open_read(path).unwrap();- let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();- let col = fast_field_readers.i64("field").unwrap();- assert_eq!(col.first(0), None);-- let col = fast_field_readers- .i64("field")- .unwrap()- .first_or_default_col(0);- assert_eq!(col.get_val(0), 0);- let col = fast_field_readers- .i64("field")- .unwrap()- .first_or_default_col(-100);- assert_eq!(col.get_val(0), -100);- }-- #[test]- fn test_date_fastfield_default() {- let path = Path::new("test");- let directory: RamDirectory = RamDirectory::create();- let mut schema_builder = Schema::builder();- schema_builder.add_date_field("date", FAST);- let schema = schema_builder.build();- {- let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();- let doc = TantivyDocument::default();- fast_field_writers.add_document(&doc).unwrap();- fast_field_writers.serialize(&mut write, None).unwrap();- write.terminate().unwrap();- }-- let file = directory.open_read(path).unwrap();- let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();- let col = fast_field_readers- .date("date")- .unwrap()- .first_or_default_col(DateTime::default());- assert_eq!(col.get_val(0), DateTime::default());- }-// Warning: this generates the same permutation at each callpub fn generate_permutation() -> Vec<u64> {let mut permutation: Vec<u64> = (0u64..100_000u64).collect();@@ -390,10 +311,7 @@ mod tests {}let file = directory.open_read(path).unwrap();let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let col = fast_field_readers- .u64("field")- .unwrap()- .first_or_default_col(0);+ let col = fast_field_readers.u64("field").unwrap().values;for a in 0..n {assert_eq!(col.get_val(a as u32), permutation[a]);}@@ -438,250 +356,6 @@ mod tests {assert_eq!(reader.searcher().segment_readers().len(), 1);}- fn get_vals_for_docs(column: &Column<u64>, docs: Range<u32>) -> Vec<u64> {- docs.into_iter()- .flat_map(|doc| column.values_for_doc(doc))- .collect()- }-- #[test]- fn test_text_fastfield() {- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("text", TEXT | FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- {- // first segment- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer.set_merge_policy(Box::new(NoMergePolicy));- index_writer- .add_document(doc!(- text_field => "BBBBB", // term ord 1- text_field => "AAAAA", // term ord 0- ))- .unwrap();- index_writer.add_document(doc!()).unwrap();- index_writer- .add_document(doc!(- text_field => "AAAAA", // term_ord 0- ))- .unwrap();- index_writer- .add_document(doc!(- text_field => "AAAAA",- text_field => "BBBBB",- ))- .unwrap();- index_writer- .add_document(doc!(- text_field => "zumberthree", // term_ord 2, after merge term_ord 3- ))- .unwrap();-- index_writer.add_document(doc!()).unwrap();- index_writer.commit().unwrap();-- let reader = index.reader().unwrap();- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 1);- let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();- let str_column = fast_fields.str("text").unwrap().unwrap();- assert!(str_column.ords().values_for_doc(0u32).eq([1, 0]),);- assert!(str_column.ords().values_for_doc(1u32).next().is_none());- assert!(str_column.ords().values_for_doc(2u32).eq([0]),);- assert!(str_column.ords().values_for_doc(3u32).eq([0, 1]),);- assert!(str_column.ords().values_for_doc(4u32).eq([2]),);-- let mut str_term = String::default();- assert!(str_column.ord_to_str(0, &mut str_term).unwrap());- assert_eq!("AAAAA", &str_term);-- let inverted_index = segment_reader.inverted_index(text_field).unwrap();- assert_eq!(inverted_index.terms().num_terms(), 3);- let mut bytes = vec![];- assert!(inverted_index.terms().ord_to_term(0, &mut bytes).unwrap());- assert_eq!(bytes, "aaaaa".as_bytes());- }-- {- // second segment- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();-- index_writer- .add_document(doc!(- text_field => "AAAAA", // term_ord 0- ))- .unwrap();-- index_writer- .add_document(doc!(- text_field => "CCCCC AAAAA", // term_ord 1, after merge 2- ))- .unwrap();-- index_writer.add_document(doc!()).unwrap();- index_writer.commit().unwrap();-- let reader = index.reader().unwrap();- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 2);- let segment_reader = searcher.segment_reader(1);- let fast_fields = segment_reader.fast_fields();- let text_fast_field = fast_fields.str("text").unwrap().unwrap();-- assert_eq!(&get_vals_for_docs(text_fast_field.ords(), 0..2), &[0, 1]);- }-- // TODO uncomment once merging is available- // Merging the segments- {- let segment_ids = index.searchable_segment_ids().unwrap();- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer.merge(&segment_ids).wait().unwrap();- index_writer.wait_merging_threads().unwrap();- }-- let reader = index.reader().unwrap();- let searcher = reader.searcher();- let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();- let text_column = fast_fields.str("text").unwrap().unwrap();-- assert_eq!(- get_vals_for_docs(text_column.ords(), 0..8),- vec![1, 0, 0, 0, 1, 3 /* next segment */, 0, 2]- );- }-- #[test]- fn test_string_fastfield_simple() {- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("text", TEXT | FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer = index.writer_for_tests().unwrap();- writer.add_document(doc!(text_field=>"hello happy tax payer", text_field=>"aaa this string comes lexicographically before the other one.")).unwrap();- writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let segment_reader = searcher.segment_reader(0);- let str_column = segment_reader.fast_fields().str("text").unwrap().unwrap();- // The string values are not sorted here.- let term_ords: Vec<u64> = str_column.term_ords(0u32).collect();- assert_eq!(&term_ords, &[1, 0]);- }-- #[test]- fn test_facet_fastfield_simple() {- let mut schema_builder = Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer = index.writer_for_tests().unwrap();- writer- .add_document(doc!(facet_field=>Facet::from("/a/2"), facet_field=>Facet::from("/a/1")))- .unwrap();- writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let segment_reader = searcher.segment_reader(0);- let facet_reader = segment_reader.facet_reader("facet").unwrap();- // facets, contrary to strings are sorted.- let mut facet_ords = Vec::new();- facet_ords.extend(facet_reader.facet_ords(0u32));- assert_eq!(&facet_ords, &[0, 1]);- }-- #[test]- fn test_string_fastfield() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("text", STRING | FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- {- // first segment- let mut index_writer = index.writer_for_tests()?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- index_writer.add_document(doc!(- text_field => "BBBBB", // term_ord 1- ))?;- index_writer.add_document(doc!())?;- index_writer.add_document(doc!(- text_field => "AAAAA", // term_ord 0- ))?;- index_writer.add_document(doc!(- text_field => "AAAAA", // term_ord 0- ))?;- index_writer.add_document(doc!(- text_field => "zumberthree", // term_ord 2, after merge term_ord 3- ))?;-- index_writer.add_document(doc!())?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 1);- let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();- let text_col = fast_fields.str("text").unwrap().unwrap();-- assert_eq!(get_vals_for_docs(text_col.ords(), 0..6), vec![1, 0, 0, 2]);-- let inverted_index = segment_reader.inverted_index(text_field)?;- assert_eq!(inverted_index.terms().num_terms(), 3);- let mut bytes = vec![];- assert!(inverted_index.terms().ord_to_term(0, &mut bytes)?);- assert_eq!(bytes, "AAAAA".as_bytes());- }-- {- // second segment- let mut index_writer = index.writer_for_tests()?;-- index_writer.add_document(doc!(- text_field => "AAAAA", // term_ord 0- ))?;-- index_writer.add_document(doc!(- text_field => "CCCCC", // term_ord 1, after merge 2- ))?;-- index_writer.add_document(doc!())?;- index_writer.commit()?;-- let reader = index.reader()?;- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 2);- let segment_reader = searcher.segment_reader(1);- let fast_fields = segment_reader.fast_fields();- let text_fast_field = fast_fields.str("text").unwrap().unwrap();-- assert_eq!(&get_vals_for_docs(text_fast_field.ords(), 0..2), &[0, 1]);- }- // Merging the segments- {- let segment_ids = index.searchable_segment_ids()?;- let mut index_writer: IndexWriter = index.writer_for_tests()?;- index_writer.merge(&segment_ids).wait()?;- index_writer.wait_merging_threads()?;- }-- let reader = index.reader()?;- let searcher = reader.searcher();- let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();- let text_fast_field = fast_fields.str("text").unwrap().unwrap();-- assert_eq!(- get_vals_for_docs(text_fast_field.ords(), 0..9),- vec![1, 0, 0, 3 /* next segment */, 0, 2]- );-- Ok(())- }-#[test]fn test_datefastfield() -> crate::Result<()> {let mut schema_builder = Schema::builder();@@ -708,6 +382,7 @@ mod tests {date_field => DateTime::from_u64(4i64.to_u64())))?;index_writer.add_document(doc!(+ date_field => DateTime::from_u64(0i64.to_u64()),multi_date_field => DateTime::from_u64(5i64.to_u64()),multi_date_field => DateTime::from_u64(6i64.to_u64())))?;@@ -721,31 +396,11 @@ mod tests {.column_opt::<DateTime>("date").unwrap().unwrap()- .first_or_default_col(Default::default());- let dates_fast_field = fast_fields- .column_opt::<DateTime>("multi_date")- .unwrap()- .unwrap();- let mut dates = Vec::new();- {- assert_eq!(date_fast_field.get_val(0).into_timestamp_nanos(), 1i64);- dates_fast_field.fill_vals(0u32, &mut dates);- assert_eq!(dates.len(), 2);- assert_eq!(dates[0].into_timestamp_nanos(), 2i64);- assert_eq!(dates[1].into_timestamp_nanos(), 3i64);- }- {- assert_eq!(date_fast_field.get_val(1).into_timestamp_nanos(), 4i64);- dates_fast_field.fill_vals(1u32, &mut dates);- assert!(dates.is_empty());- }- {- assert_eq!(date_fast_field.get_val(2).into_timestamp_nanos(), 0i64);- dates_fast_field.fill_vals(2u32, &mut dates);- assert_eq!(dates.len(), 2);- assert_eq!(dates[0].into_timestamp_nanos(), 5i64);- assert_eq!(dates[1].into_timestamp_nanos(), 6i64);- }+ .values;++ assert_eq!(date_fast_field.get_val(0).into_timestamp_nanos(), 1i64);+ assert_eq!(date_fast_field.get_val(1).into_timestamp_nanos(), 4i64);+ assert_eq!(date_fast_field.get_val(2).into_timestamp_nanos(), 0i64);Ok(())}@@ -815,38 +470,6 @@ mod tests {}}- #[test]- pub fn test_fastfield_bool_default_value() {- let path = Path::new("test_bool");- let directory: RamDirectory = RamDirectory::create();- let mut schema_builder = Schema::builder();- schema_builder.add_bool_field("field_bool", FAST);- let schema = schema_builder.build();- {- let mut write: WritePtr = directory.open_write(path).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();- let doc = TantivyDocument::default();- fast_field_writers.add_document(&doc).unwrap();- fast_field_writers.serialize(&mut write, None).unwrap();- write.terminate().unwrap();- }- let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 86);- let fastfield_readers = FastFieldReaders::open(file, schema).unwrap();- let col = fastfield_readers.bool("field_bool").unwrap();- assert_eq!(col.first(0), None);- let col = fastfield_readers- .bool("field_bool")- .unwrap()- .first_or_default_col(false);- assert_eq!(col.get_val(0), false);- let col = fastfield_readers- .bool("field_bool")- .unwrap()- .first_or_default_col(true);- assert_eq!(col.get_val(0), true);- }-fn get_index(docs: &[crate::TantivyDocument], schema: &Schema) -> crate::Result<RamDirectory> {let directory: RamDirectory = RamDirectory::create();{@@ -934,18 +557,14 @@ mod tests {let reader = index.reader().unwrap();let searcher = reader.searcher();let segment = &searcher.segment_readers()[0];- let field = segment- .fast_fields()- .u64("url_norm_hash")- .unwrap()- .first_or_default_col(0);+ let field = segment.fast_fields().u64("url_norm_hash").unwrap().values;let numbers = [100, 200, 300];let test_range = |range: RangeInclusive<u64>| {- let expexted_count = numbers.iter().filter(|num| range.contains(num)).count();+ let expected_count = numbers.iter().filter(|num| range.contains(num)).count();let mut vec = vec![];field.get_row_ids_for_value_range(range, 0..u32::MAX, &mut vec);- assert_eq!(vec.len(), expexted_count);+ assert_eq!(vec.len(), expected_count);};test_range(50..=50);test_range(150..=150);@@ -957,31 +576,6 @@ mod tests {test_range(100..=299);}- #[test]- fn test_ip_addr_columnar_simple() {- let mut schema_builder = Schema::builder();- let ip_field = schema_builder.add_u64_field("ip", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- let ip_addr = Ipv6Addr::new(1, 2, 3, 4, 5, 1, 2, 3);- index_writer- .add_document(TantivyDocument::default())- .unwrap();- index_writer.add_document(doc!(ip_field=>ip_addr)).unwrap();- index_writer- .add_document(TantivyDocument::default())- .unwrap();- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let fastfields = searcher.segment_reader(0u32).fast_fields();- let column: Column<Ipv6Addr> = fastfields.column_opt("ip").unwrap().unwrap();- assert_eq!(column.num_docs(), 3);- assert_eq!(column.first(0), None);- assert_eq!(column.first(1), Some(ip_addr));- assert_eq!(column.first(2), None);- }-#[test]fn test_mapping_bug_docids_for_value_range() {let mut schema_builder = Schema::builder();@@ -1012,11 +606,7 @@ mod tests {let reader = index.reader().unwrap();let searcher = reader.searcher();let segment = &searcher.segment_readers()[0];- let field = segment- .fast_fields()- .u64("url_norm_hash")- .unwrap()- .first_or_default_col(0);+ let field = segment.fast_fields().u64("url_norm_hash").unwrap().values;let numbers = [1000, 1001, 1003];let test_range = |range: RangeInclusive<u64>| {@@ -1050,40 +640,6 @@ mod tests {test_range_variant(1000, 1002);}- #[test]- fn test_json_object_fast_field() {- let mut schema_builder = Schema::builder();- let without_fast_field = schema_builder.add_json_field("without", STORED);- let with_fast_field = schema_builder.add_json_field("with", STORED | FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer = index.writer_for_tests().unwrap();- writer- .add_document(doc!(without_fast_field=>json!({"hello": "without"})))- .unwrap();- writer- .add_document(doc!(with_fast_field=>json!({"hello": "with"})))- .unwrap();- writer- .add_document(doc!(with_fast_field=>json!({"hello": "with2"})))- .unwrap();- writer- .add_document(doc!(with_fast_field=>json!({"hello": "with1"})))- .unwrap();- writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let segment_reader = searcher.segment_reader(0u32);- let fast_fields = segment_reader.fast_fields();- let column_without_opt = fast_fields.str("without.hello");- assert!(column_without_opt.is_err());- let column_with_opt: Option<StrColumn> = fast_fields.str("with.hello").unwrap();- let column_with: StrColumn = column_with_opt.unwrap();- assert!(column_with.term_ords(0).next().is_none());- assert!(column_with.term_ords(1).eq([0]));- assert!(column_with.term_ords(2).eq([2]));- assert!(column_with.term_ords(3).eq([1]));- }-#[test]fn test_fast_field_in_json_field_expand_dots_disabled() {let mut schema_builder = Schema::builder();@@ -1106,33 +662,7 @@ mod tests {.column_opt::<i64>(r"json.attr\.age").unwrap().unwrap();- let vals: Vec<i64> = column.values_for_doc(0u32).collect();- assert_eq!(&vals, &[32])- }-- #[test]- fn test_fast_field_in_json_field_with_tokenizer() {- let mut schema_builder = Schema::builder();- let json_option = JsonObjectOptions::default().set_fast(Some("default"));- let json = schema_builder.add_json_field("json", json_option);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(json => json!({"age": 32})))- .unwrap();- index_writer- .add_document(doc!(json => json!({"age": "NEW"})))- .unwrap();-- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let fast_fields = searcher.segment_reader(0u32).fast_fields();-- let ff_str = fast_fields.str("json.age").unwrap().unwrap();- let mut output = String::new();- ff_str.ord_to_str(0, &mut output).unwrap();- assert_eq!(output, "new");+ assert_eq!(column.first(0u32), Some(32))}#[test]@@ -1156,8 +686,7 @@ mod tests {.column_opt::<i64>(test_column_name).unwrap().unwrap();- let vals: Vec<i64> = column.values_for_doc(0u32).collect();- assert_eq!(&vals, &[32]);+ assert_eq!(column.first(0u32), Some(32))}}@@ -1178,8 +707,7 @@ mod tests {.column_opt::<i64>("field.with.dot").unwrap().unwrap();- let vals: Vec<i64> = column.values_for_doc(0u32).collect();- assert_eq!(&vals, &[32]);+ assert_eq!(column.first(0u32), Some(32));}#[test]@@ -1200,76 +728,7 @@ mod tests {.column_opt::<i64>("jsonfield.attr.age").unwrap().unwrap();- let vals: Vec<i64> = column.values_for_doc(0u32).collect();- assert_eq!(&vals, &[33]);- }-- #[test]- fn test_fast_field_tokenizer() {- let mut schema_builder = Schema::builder();- let opt = TextOptions::default().set_fast(Some("custom_lowercase"));- let text_field = schema_builder.add_text_field("text", opt);- let schema = schema_builder.build();- let ff_tokenizer_manager = TokenizerManager::default();- ff_tokenizer_manager.register(- "custom_lowercase",- TextAnalyzer::builder(RawTokenizer::default())- .filter(LowerCaser)- .build(),- );-- let mut index = Index::create_in_ram(schema);- index.set_fast_field_tokenizers(ff_tokenizer_manager);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(text_field => "Test1 test2"))- .unwrap();- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();- let column = fast_field_reader.str("text").unwrap().unwrap();- let mut out = String::new();- column.ord_to_str(0u64, &mut out).unwrap();- assert_eq!(&out, "test1 test2");- }-- #[test]- fn test_text_fast_field_tokenizer() {- let mut schema_builder = Schema::builder();-- let text_fieldtype = crate::schema::TextOptions::default()- .set_indexing_options(- crate::schema::TextFieldIndexing::default()- .set_index_option(crate::schema::IndexRecordOption::WithFreqs)- .set_tokenizer("raw"),- )- .set_fast(Some("default"))- .set_stored();-- let log_field = schema_builder.add_text_field("log_level", text_fieldtype);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(log_field => "info"))- .unwrap();- index_writer- .add_document(doc!(log_field => "INFO"))- .unwrap();- index_writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();-- let text_fast_field = fast_field_reader.str("log_level").unwrap().unwrap();- let mut buffer = String::new();- assert!(text_fast_field.ord_to_str(0, &mut buffer).unwrap());- assert_eq!(buffer, "info");- assert!(!text_fast_field.ord_to_str(1, &mut buffer).unwrap());-- assert!(text_fast_field.term_ords(0).eq([0].into_iter()));- assert!(text_fast_field.term_ords(1).eq([0].into_iter()));- assert!(text_fast_field.ords().values_for_doc(0u32).eq([0]));- assert!(text_fast_field.ords().values_for_doc(1u32).eq([0]));+ assert_eq!(column.first(0u32), Some(33))}#[test]@@ -1294,13 +753,11 @@ mod tests {.column_opt::<i64>("jsonfield.attr.age").unwrap().unwrap();- let vals: Vec<i64> = column.values_for_doc(0u32).collect();- assert_eq!(&vals, &[33]);+ assert_eq!(column.first(0u32), Some(33));let column = fast_field_reader.column_opt::<i64>("jsonfield\\.attr.age").unwrap().unwrap();- let vals: Vec<i64> = column.values_for_doc(0u32).collect();- assert_eq!(&vals, &[33]);+ assert_eq!(column.first(0u32), Some(33));}}diff --git a/crates/tantivy/src/fastfield/readers.rs b/crates/tantivy/src/fastfield/readers.rsindex 401de1d6..735f9835 100644--- a/crates/tantivy/src/fastfield/readers.rs+++ b/crates/tantivy/src/fastfield/readers.rs@@ -1,10 +1,9 @@use std::io;-use std::net::Ipv6Addr;use std::sync::Arc;use crate::columnar::{- BytesColumn, Column, ColumnType, ColumnValues, ColumnarReader, DynamicColumn,- DynamicColumnHandle, HasAssociatedColumnType, StrColumn,+ BytesColumn, Column, ColumnType, ColumnarReader, DynamicColumn, DynamicColumnHandle,+ HasAssociatedColumnType,};use crate::common::ByteCount;@@ -139,20 +138,6 @@ impl FastFieldReaders {.sum())}- /// Returns a typed column value object.- ///- /// In that column value:- /// - Rows with no value are associated with the default value.- /// - Rows with several values are associated with the first value.- pub fn column_first_or_default<T>(&self, field: &str) -> crate::Result<Arc<dyn ColumnValues<T>>>- where- T: PartialOrd + Copy + HasAssociatedColumnType + Send + Sync + 'static,- DynamicColumn: Into<Option<Column<T>>>,- {- let col: Column<T> = self.column(field)?;- Ok(col.first_or_default_col(T::default_value()))- }-/// Returns a typed column associated to a given field name.////// Returns an error if no column associated with that field_name exists.@@ -183,24 +168,6 @@ impl FastFieldReaders {self.column(field)}- /// Returns the `ip` fast field reader reader associated to `field`.- ///- /// If `field` is not a u128 fast field, this method returns an Error.- pub fn ip_addr(&self, field: &str) -> crate::Result<Column<Ipv6Addr>> {- self.column(field)- }-- /// Returns a `str` column.- pub fn str(&self, field_name: &str) -> crate::Result<Option<StrColumn>> {- let Some(dynamic_column_handle) =- self.dynamic_column_handle(field_name, ColumnType::Str)?- else {- return Ok(None);- };- let dynamic_column = dynamic_column_handle.open()?;- Ok(dynamic_column.into())- }-/// Returns a `bytes` column.pub fn bytes(&self, field_name: &str) -> crate::Result<Option<BytesColumn>> {let Some(dynamic_column_handle) =@@ -460,16 +427,13 @@ mod tests {assert_eq!(id_columns.first().unwrap().column_type(), ColumnType::U64);let foo_columns = fast_fields.dynamic_column_handles("json.foo").unwrap();- assert_eq!(foo_columns.len(), 3);+ assert_eq!(foo_columns.len(), 2);assert!(foo_columns.iter().any(|column| column.column_type() == ColumnType::I64));assert!(foo_columns.iter().any(|column| column.column_type() == ColumnType::Bool));- assert!(foo_columns- .iter()- .any(|column| column.column_type() == ColumnType::Str));println!("*** {:?}", fast_fields.columnar().list_columns());}diff --git a/crates/tantivy/src/fastfield/writer.rs b/crates/tantivy/src/fastfield/writer.rsindex a8c7d1a5..9eefa602 100644--- a/crates/tantivy/src/fastfield/writer.rs+++ b/crates/tantivy/src/fastfield/writer.rs@@ -2,11 +2,10 @@ use std::io;use crate::columnar::{ColumnarWriter, NumericalValue};use crate::common::{DateTimePrecision, JsonPathWriter};-use crate::tokenizer_api::Token;use crate::indexer::doc_id_mapping::DocIdMapping;use crate::schema::document::{Document, ReferenceValue, ReferenceValueLeaf, Value};-use crate::schema::{value_type_to_column_type, Field, FieldType, Schema, Type};+use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};use crate::tokenizer::{TextAnalyzer, TokenizerManager};use crate::{DocId, TantivyError};@@ -81,7 +80,7 @@ impl FastFieldsWriter {}}- let sort_values_within_row = value_type == Type::Facet;+ let sort_values_within_row = false;if let Some(column_type) = value_type_to_column_type(value_type) {columnar_writer.record_column_type(field_entry.name(),@@ -142,19 +141,7 @@ impl FastFieldsWriter {match value.as_value() {ReferenceValue::Leaf(leaf) => match leaf {ReferenceValueLeaf::Null => {}- ReferenceValueLeaf::Str(val) => {- if let Some(tokenizer) =- &mut self.per_field_tokenizer[field.field_id() as usize]- {- let mut token_stream = tokenizer.token_stream(val);- token_stream.process(&mut |token: &Token| {- self.columnar_writer- .record_str(doc_id, field_name, &token.text);- })- } else {- self.columnar_writer.record_str(doc_id, field_name, val);- }- }+ ReferenceValueLeaf::Str(_) => {}ReferenceValueLeaf::U64(val) => {self.columnar_writer.record_numerical(doc_id,@@ -182,24 +169,14 @@ impl FastFieldsWriter {self.columnar_writer.record_datetime(doc_id, field_name, truncated_datetime);}- ReferenceValueLeaf::Facet(val) => {- self.columnar_writer.record_str(doc_id, field_name, val);- }ReferenceValueLeaf::Bytes(val) => {self.columnar_writer.record_bytes(doc_id, field_name, val);}- ReferenceValueLeaf::IpAddr(val) => {- self.columnar_writer.record_ip_addr(doc_id, field_name, val);- }+ ReferenceValueLeaf::IpAddr(_) => {}ReferenceValueLeaf::Bool(val) => {self.columnar_writer.record_bool(doc_id, field_name, val);}- ReferenceValueLeaf::PreTokStr(val) => {- for token in &val.tokens {- self.columnar_writer- .record_str(doc_id, field_name, &token.text);- }- }+ ReferenceValueLeaf::PreTokStr(_) => {}},ReferenceValue::Array(val) => {// TODO: Check this is the correct behaviour we want.@@ -285,16 +262,7 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(match json_val.as_value() {ReferenceValue::Leaf(leaf) => match leaf {ReferenceValueLeaf::Null => {} // TODO: Handle null- ReferenceValueLeaf::Str(val) => {- if let Some(text_analyzer) = tokenizer.as_mut() {- let mut token_stream = text_analyzer.token_stream(val);- token_stream.process(&mut |token| {- columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);- })- } else {- columnar_writer.record_str(doc, json_path_writer.as_str(), val);- }- }+ ReferenceValueLeaf::Str(_) => {}ReferenceValueLeaf::U64(val) => {columnar_writer.record_numerical(doc,@@ -322,9 +290,6 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(ReferenceValueLeaf::Date(val) => {columnar_writer.record_datetime(doc, json_path_writer.as_str(), val);}- ReferenceValueLeaf::Facet(_) => {- unimplemented!("Facet support in dynamic fields is not yet implemented")- }ReferenceValueLeaf::Bytes(_) => {// TODO: This can be re added once it is added to the JSON Utils section as well.// columnar_writer.record_bytes(doc, json_path_writer.as_str(), val);@@ -363,140 +328,3 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(}}}--#[cfg(test)]-mod tests {- use crate::columnar::{Column, ColumnarReader, ColumnarWriter, StrColumn};- use crate::common::JsonPathWriter;-- use super::record_json_value_to_columnar_writer;- use crate::fastfield::writer::JSON_DEPTH_LIMIT;- use crate::DocId;-- fn test_columnar_from_jsons_aux(- json_docs: &[serde_json::Value],- expand_dots: bool,- ) -> ColumnarReader {- let mut columnar_writer = ColumnarWriter::default();- let mut json_path = JsonPathWriter::default();- json_path.set_expand_dots(expand_dots);- for (doc, json_doc) in json_docs.iter().enumerate() {- record_json_value_to_columnar_writer(- doc as u32,- json_doc,- JSON_DEPTH_LIMIT,- &mut json_path,- &mut columnar_writer,- &mut None,- );- }- let mut buffer = Vec::new();- columnar_writer- .serialize(json_docs.len() as DocId, None, &mut buffer)- .unwrap();- ColumnarReader::open(buffer).unwrap()- }-- #[test]- fn test_json_fastfield_record_simple() {- let json_doc = serde_json::json!({- "float": 1.02,- "text": "hello happy tax payer",- "nested": {"child": 3, "child2": 5},- "arr": ["hello", "happy", "tax", "payer"]- });- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);- let columns = columnar_reader.list_columns().unwrap();- {- assert_eq!(columns[0].0, "arr");- let column_arr_opt: Option<StrColumn> = columns[0].1.open().unwrap().into();- assert!(column_arr_opt- .unwrap()- .term_ords(0)- .eq([1, 0, 3, 2].into_iter()));- }- {- assert_eq!(columns[1].0, "float");- let column_float_opt: Option<Column<f64>> = columns[1].1.open().unwrap().into();- assert!(column_float_opt- .unwrap()- .values_for_doc(0)- .eq([1.02f64].into_iter()));- }- {- assert_eq!(columns[2].0, "nested\u{1}child");- let column_nest_child_opt: Option<Column<i64>> = columns[2].1.open().unwrap().into();- assert!(column_nest_child_opt- .unwrap()- .values_for_doc(0)- .eq([3].into_iter()));- }- {- assert_eq!(columns[3].0, "nested\u{1}child2");- let column_nest_child2_opt: Option<Column<i64>> = columns[3].1.open().unwrap().into();- assert!(column_nest_child2_opt- .unwrap()- .values_for_doc(0)- .eq([5].into_iter()));- }- {- assert_eq!(columns[4].0, "text");- let column_text_opt: Option<StrColumn> = columns[4].1.open().unwrap().into();- assert!(column_text_opt.unwrap().term_ords(0).eq([0].into_iter()));- }- }-- #[test]- fn test_json_fastfield_deep_obj() {- let json_doc = serde_json::json!(- {"a": {"a": {"a": {"a": {"a":- {"a": {"a": {"a": {"a": {"a":- {"a": {"a": {"a": {"a": {"a":- {"a": {"a": {"a": {"depth_accepted": 19, "a": { "depth_truncated": 20}- }}}}}}}}}}}}}}}}}}});- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);- let columns = columnar_reader.list_columns().unwrap();- assert_eq!(columns.len(), 1);- assert!(columns[0].0.ends_with("a\u{1}a\u{1}a\u{1}depth_accepted"));- }-- #[test]- fn test_json_fastfield_deep_arr() {- let json_doc = json!(- {"obj":- [[[[[,- [[[[[,- [[[[[,- [[18, [19, //< within limits- [20]]]]]]]]]]]]]]]]]]]});- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);- let columns = columnar_reader.list_columns().unwrap();- assert_eq!(columns.len(), 1);- assert_eq!(columns[0].0, "obj");- let dynamic_column = columns[0].1.open().unwrap();- let col: Option<Column<i64>> = dynamic_column.into();- let vals: Vec<i64> = col.unwrap().values_for_doc(0).collect();- assert_eq!(&vals, &[18, 19])- }-- #[test]- fn test_json_fast_field_do_not_expand_dots() {- let json_doc = json!({"field.with.dots": {"child.with.dot": "hello"}});- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);- let columns = columnar_reader.list_columns().unwrap();- assert_eq!(columns.len(), 1);- assert_eq!(columns[0].0, "field.with.dots\u{1}child.with.dot");- }-- #[test]- fn test_json_fast_field_expand_dots() {- let json_doc = json!({"field.with.dots": {"child.with.dot": "hello"}});- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], true);- let columns = columnar_reader.list_columns().unwrap();- assert_eq!(columns.len(), 1);- assert_eq!(- columns[0].0,- "field\u{1}with\u{1}dots\u{1}child\u{1}with\u{1}dot"- );- }-}diff --git a/crates/tantivy/src/index/segment_reader.rs b/crates/tantivy/src/index/segment_reader.rsindex c927a7d0..c2f2b49f 100644--- a/crates/tantivy/src/index/segment_reader.rs+++ b/crates/tantivy/src/index/segment_reader.rs@@ -8,7 +8,7 @@ use itertools::Itertools;use crate::directory::{CompositeFile, FileSlice};use crate::error::DataCorruption;-use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};+use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FastFieldReaders};use crate::fieldnorm::{FieldNormReader, FieldNormReaders};use crate::index::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};use crate::json_utils::json_path_sep_to_dot;@@ -92,22 +92,6 @@ impl SegmentReader {&self.fast_fields_readers}- /// Accessor to the `FacetReader` associated with a given `Field`.- pub fn facet_reader(&self, field_name: &str) -> crate::Result<FacetReader> {- let schema = self.schema();- let field = schema.get_field(field_name)?;- let field_entry = schema.get_field_entry(field);- if field_entry.field_type().value_type() != Type::Facet {- return Err(crate::TantivyError::SchemaError(format!(- "`{field_name}` is not a facet field.`"- )));- }- let Some(facet_column) = self.fast_fields().str(field_name)? else {- panic!("Facet Field `{field_name}` is missing. This should not happen");- };- Ok(FacetReader::new(facet_column))- }-/// Accessor to the segment's `Field norms`'s reader.////// Field norms are the length (in tokens) of the fields.diff --git a/crates/tantivy/src/indexer/doc_id_mapping.rs b/crates/tantivy/src/indexer/doc_id_mapping.rsindex 9869d78a..1ebeb42b 100644--- a/crates/tantivy/src/indexer/doc_id_mapping.rs+++ b/crates/tantivy/src/indexer/doc_id_mapping.rs@@ -467,22 +467,11 @@ mod tests_indexsorting {let segment_reader = searcher.segment_reader(0);let fast_fields = segment_reader.fast_fields();- let fast_field = fast_fields- .u64("my_number")- .unwrap()- .first_or_default_col(999);+ let fast_field = fast_fields.u64("my_number").unwrap().values;assert_eq!(fast_field.get_val(0), 10u64);assert_eq!(fast_field.get_val(1), 20u64);assert_eq!(fast_field.get_val(2), 30u64);- let multifield = fast_fields.u64("multi_numbers").unwrap();- let vals: Vec<u64> = multifield.values_for_doc(0u32).collect();- assert_eq!(vals, &[] as &[u64]);- let vals: Vec<_> = multifield.values_for_doc(1u32).collect();- assert_eq!(vals, &[5, 6]);-- let vals: Vec<_> = multifield.values_for_doc(2u32).collect();- assert_eq!(vals, &[3]);Ok(())}@@ -523,10 +512,7 @@ mod tests_indexsorting {let segment_reader = searcher.segment_reader(0);let fast_fields = segment_reader.fast_fields();- let fast_field = fast_fields- .date("date")- .unwrap()- .first_or_default_col(DateTime::from_timestamp_secs(0));+ let fast_field = fast_fields.date("date").unwrap().values;assert_eq!(fast_field.get_val(0), DateTime::from_timestamp_secs(1001));assert_eq!(fast_field.get_val(1), DateTime::from_timestamp_secs(1000));assert_eq!(fast_field.get_val(2), DateTime::from_timestamp_secs(999));diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rsindex a6c3924c..0392789f 100644--- a/crates/tantivy/src/indexer/index_writer.rs+++ b/crates/tantivy/src/indexer/index_writer.rs@@ -770,10 +770,8 @@ impl<D: Document> Drop for IndexWriter<D> {#[cfg(test)]mod tests {use std::collections::{HashMap, HashSet};- use std::net::Ipv6Addr;- use crate::columnar::{Cardinality, Column, MonotonicallyMappableToU128};- use itertools::Itertools;+ use crate::columnar::{Column, MonotonicallyMappableToU128};use proptest::prop_oneof;use super::super::operation::UserOperation;@@ -784,14 +782,15 @@ mod tests {use crate::indexer::NoMergePolicy;use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};use crate::schema::{- self, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions, Schema,- TextFieldIndexing, TextOptions, Value, FAST, INDEXED, STORED, STRING, TEXT,+ self, IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions, Value,+ FAST, INDEXED, STORED, STRING, TEXT,};use crate::store::DOCSTORE_CACHE_CAPACITY;use crate::{DateTime, DocAddress, Index, IndexSettings, IndexSortByField, IndexWriter, Order,ReloadPolicy, TantivyDocument, Term,};+ use proptest::prelude::*;const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do \eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad \@@ -1378,59 +1377,6 @@ mod tests {assert!(commit_again.is_ok());}- #[test]- fn test_delete_and_merge_removes_terms_fast_field_dict() {- let mut schema_builder = schema::Schema::builder();- let text_field = schema_builder.add_text_field("text", STRING | FAST);- let schema = schema_builder.build();-- let index = Index::builder().schema(schema).create_in_ram().unwrap();- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(text_field => "one"))- .unwrap();- index_writer- .add_document(doc!(text_field => "two"))- .unwrap();- index_writer- .add_document(doc!(text_field => "three"))- .unwrap();- index_writer.commit().unwrap();- let index_reader = index.reader().unwrap();- let searcher = index_reader.searcher();- let segment_reader = searcher.segment_reader(0);- let text_fast_field = segment_reader.fast_fields().str("text").unwrap().unwrap();- let mut buffer = String::new();- assert!(text_fast_field.ord_to_str(0, &mut buffer).unwrap());- assert_eq!(buffer, "one");- assert!(text_fast_field.ord_to_str(1, &mut buffer).unwrap());- assert_eq!(buffer, "three");- assert!(text_fast_field.ord_to_str(2, &mut buffer).unwrap());- assert_eq!(buffer, "two");- assert!(!text_fast_field.ord_to_str(3, &mut buffer).unwrap());-- assert_eq!(segment_reader.max_doc(), 3);- index_writer.delete_term(Term::from_field_text(text_field, "three"));- index_writer.commit().unwrap();- index_writer- .merge(&[segment_reader.segment_id()])- .wait()- .unwrap();- index_reader.reload().unwrap();- let searcher = index_reader.searcher();- let segment_reader = searcher.segment_reader(0);- assert_eq!(segment_reader.max_doc(), 2);- let text_fast_field = segment_reader.fast_fields().str("text").unwrap().unwrap();- let mut buffer = String::new();- assert!(text_fast_field.ord_to_str(0, &mut buffer).unwrap());- assert_eq!(buffer, "one");- assert!(text_fast_field.ord_to_str(1, &mut buffer).unwrap());- assert_eq!(buffer, "two");- assert!(!text_fast_field.ord_to_str(2, &mut buffer).unwrap());- assert!(text_fast_field.term_ords(0).eq([0].into_iter()));- assert!(text_fast_field.term_ords(1).eq([1].into_iter()));- }-#[test]fn test_delete_with_sort_by_field() -> crate::Result<()> {let mut schema_builder = schema::Schema::builder();@@ -1475,7 +1421,7 @@ mod tests {let in_order_alive_ids: Vec<u64> = segment_reader.doc_ids_alive()- .flat_map(|doc| fast_field_reader.values_for_doc(doc))+ .flat_map(|doc| fast_field_reader.first(doc)).collect();assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);Ok(())@@ -1535,7 +1481,7 @@ mod tests {let fast_field_reader = segment_reader.fast_fields().u64("id")?;let in_order_alive_ids: Vec<u64> = segment_reader.doc_ids_alive()- .flat_map(|doc| fast_field_reader.values_for_doc(doc))+ .flat_map(|doc| fast_field_reader.first(doc)).collect();assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 2, 0]);Ok(())@@ -1665,28 +1611,6 @@ mod tests {(existing_ids, deleted_ids)}- fn get_id_list(ops: &[IndexingOp]) -> Vec<u64> {- let mut id_list = Vec::new();- for op in ops {- match op {- IndexingOp::AddDoc { id, value: _ } => {- id_list.push(*id);- }- IndexingOp::AddMultipleDoc { id, .. } => {- id_list.push(*id);- }- IndexingOp::DeleteDoc { id } => {- id_list.retain(|el| el != id);- }- IndexingOp::DeleteDocQuery { id } => {- id_list.retain(|el| el != id);- }- _ => {}- }- }- id_list- }-fn test_operation_strategy(ops: &[IndexingOp],sort_index: bool,@@ -1694,12 +1618,8 @@ mod tests {) -> crate::Result<Index> {let mut schema_builder = schema::Schema::builder();let json_field = schema_builder.add_json_field("json", FAST | TEXT | STORED);- let ip_field = schema_builder.add_ip_addr_field("ip", FAST | INDEXED | STORED);- let ips_field = schema_builder- .add_ip_addr_field("ips", IpAddrOptions::default().set_fast().set_indexed());let i64_field = schema_builder.add_i64_field("i64", INDEXED);let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);- let id_opt_field = schema_builder.add_u64_field("id_opt", FAST | INDEXED | STORED);let f64_field = schema_builder.add_f64_field("f64", INDEXED);let date_field = schema_builder.add_date_field("date", INDEXED);let bytes_field = schema_builder.add_bytes_field("bytes", FAST | INDEXED | STORED);@@ -1715,22 +1635,12 @@ mod tests {);let large_text_field = schema_builder.add_text_field("large_text_field", TEXT | STORED);- let multi_text_fields = schema_builder.add_text_field("multi_text_fields", TEXT | STORED);- let multi_numbers = schema_builder.add_u64_field(- "multi_numbers",- NumericOptions::default().set_fast().set_stored(),- );- let multi_bools = schema_builder.add_bool_field(- "multi_bools",- NumericOptions::default().set_fast().set_stored(),- );- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());let schema = schema_builder.build();let settings = if sort_index {IndexSettings {sort_by_field: Some(IndexSortByField {- field: "id_opt".to_string(),+ field: "id".to_string(),order: Order::Asc,}),..Default::default()@@ -1752,49 +1662,27 @@ mod tests {// Every 3rd doc has only id fieldlet id_is_full_doc = |id| id % 3 != 0;- let multi_text_field_text1 = "test1 test2 test3 test1 test2 test3";- // rotate left- let multi_text_field_text2 = "test2 test3 test1 test2 test3 test1";- // rotate right- let multi_text_field_text3 = "test3 test1 test2 test3 test1 test2";-- let ip_from_id = |id| Ipv6Addr::from_u128(id as u128);-let add_docs = |index_writer: &mut IndexWriter,id: u64,value: IndexValue,num: u64|-> crate::Result<()> {- let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));- let ip = ip_from_id(id);let doc = if !id_is_full_doc(id) {// every 3rd doc has no ip fielddoc!(id_field=>id,)} else {- let json = json!({"date1": format!("2022-{id}-01T00:00:01Z"), "date2": format!("{id}-05-01T00:00:01Z"), "id": id, "ip": ip.to_string(), "val": value});+ let json = json!({"date1": format!("2022-{id}-01T00:00:01Z"), "date2": format!("{id}-05-01T00:00:01Z"), "id": id, "val": value});doc!(id_field=>id,json_field=>json,bytes_field => id.to_le_bytes().as_slice(),- id_opt_field => id,- ip_field => ip,- ips_field => ip,- ips_field => ip,- multi_numbers=> id,- multi_numbers => id,bool_field => (id % 2u64) != 0,i64_field => id as i64,f64_field => id as f64,date_field => DateTime::from_timestamp_secs(id as i64),- multi_bools => (id % 2u64) != 0,- multi_bools => (id % 2u64) == 0,text_field => id.to_string(),- facet_field => facet,large_text_field => LOREM,- multi_text_fields => multi_text_field_text1,- multi_text_fields => multi_text_field_text2,- multi_text_fields => multi_text_field_text3,)};for _ in 0..num {@@ -1864,7 +1752,7 @@ mod tests {let ff_reader = segment_reader.fast_fields().u64("id").unwrap();segment_reader.doc_ids_alive()- .flat_map(move |doc| ff_reader.values_for_doc(doc).collect_vec().into_iter())+ .flat_map(move |doc| ff_reader.first(doc).into_iter())}).collect();@@ -1875,29 +1763,12 @@ mod tests {let ff_reader = segment_reader.fast_fields().u64("id").unwrap();segment_reader.doc_ids_alive()- .flat_map(move |doc| ff_reader.values_for_doc(doc).collect_vec().into_iter())+ .flat_map(move |doc| ff_reader.first(doc).into_iter())}).collect();let (expected_ids_and_num_occurrences, deleted_ids) = expected_ids(ops);- let id_list = get_id_list(ops);-- // multivalue fast field content- let mut all_ips = Vec::new();- let mut num_ips = 0;- for segment_reader in searcher.segment_readers().iter() {- let ip_reader: Column<Ipv6Addr> = segment_reader- .fast_fields()- .column_opt("ips")- .unwrap()- .unwrap();- for doc in segment_reader.doc_ids_alive() {- all_ips.extend(ip_reader.values_for_doc(doc));- }- num_ips += ip_reader.values.num_vals();- }-let num_docs_expected = expected_ids_and_num_occurrences.values().map(|id_occurrences| *id_occurrences as usize)@@ -1927,17 +1798,6 @@ mod tests {);if force_end_merge && num_segments_before_merge > 1 && num_segments_after_merge == 1 {- let mut expected_multi_ips: Vec<_> = id_list- .iter()- .filter(|id| id_is_full_doc(**id))- .flat_map(|id| vec![ip_from_id(*id), ip_from_id(*id)])- .collect();- assert_eq!(num_ips, expected_multi_ips.len() as u32);-- expected_multi_ips.sort();- all_ips.sort();- assert_eq!(expected_multi_ips, all_ips);-// Test fastfield num_docslet num_docs: usize = searcher.segment_readers()@@ -1945,7 +1805,7 @@ mod tests {.map(|segment_reader| {let ff_reader = segment_reader.fast_fields()- .column_opt::<Ipv6Addr>("ips")+ .column_opt::<i64>("i64").unwrap().unwrap();ff_reader.num_docs() as usize@@ -1954,85 +1814,6 @@ mod tests {assert_eq!(num_docs, num_docs_expected);}- // Load all ips addr- let mut ips: HashSet<Ipv6Addr> = Default::default();- for reader in searcher.segment_readers() {- if let Some(ff_reader) = reader.fast_fields().column_opt::<Ipv6Addr>("ips").unwrap() {- for doc in reader.doc_ids_alive() {- ips.extend(ff_reader.values_for_doc(doc));- }- }- }-- let expected_ips = expected_ids_and_num_occurrences- .keys()- .flat_map(|id| {- if !id_is_full_doc(*id) {- None- } else {- Some(Ipv6Addr::from_u128(*id as u128))- }- })- .collect::<HashSet<_>>();- assert_eq!(ips, expected_ips);-- let expected_ips = expected_ids_and_num_occurrences- .keys()- .filter_map(|id| {- if !id_is_full_doc(*id) {- None- } else {- Some(Ipv6Addr::from_u128(*id as u128))- }- })- .collect::<HashSet<_>>();-- let mut ips: HashSet<Ipv6Addr> = Default::default();- for reader in searcher.segment_readers() {- if let Some(ff_reader) = reader.fast_fields().column_opt::<Ipv6Addr>("ips").unwrap() {- for doc in reader.doc_ids_alive() {- ips.extend(ff_reader.values_for_doc(doc));- }- }- }- assert_eq!(ips, expected_ips);-- // multivalue fast field tests- for segment_reader in searcher.segment_readers().iter() {- let id_reader = segment_reader.fast_fields().u64("id").unwrap();- let ff_reader = segment_reader- .fast_fields()- .column_opt("multi_numbers")- .unwrap()- .unwrap();- let bool_ff_reader = segment_reader- .fast_fields()- .column_opt::<bool>("multi_bools")- .unwrap()- .unwrap();- for doc in segment_reader.doc_ids_alive() {- let id = id_reader.first(doc).unwrap();-- let vals: Vec<u64> = ff_reader.values_for_doc(doc).collect();- if id_is_full_doc(id) {- assert_eq!(vals.len(), 2);- assert_eq!(vals[0], vals[1]);- assert!(expected_ids_and_num_occurrences.contains_key(&vals[0]));- assert_eq!(id_reader.first(doc), Some(vals[0]));- } else {- assert_eq!(vals.len(), 0);- }-- let bool_vals: Vec<bool> = bool_ff_reader.values_for_doc(doc).collect();- if id_is_full_doc(id) {- assert_eq!(bool_vals.len(), 2);- assert_ne!(bool_vals[0], bool_vals[1]);- } else {- assert_eq!(bool_vals.len(), 0);- }- }- }-// doc store testsfor segment_reader in searcher.segment_readers().iter() {let store_reader = segment_reader@@ -2059,40 +1840,9 @@ mod tests {.as_u64().unwrap();assert!(expected_ids_and_num_occurrences.contains_key(&id));- if id_is_full_doc(id) {- let id2 = store_reader- .get::<TantivyDocument>(doc_id)- .unwrap()- .get_first(multi_numbers)- .unwrap()- .as_u64()- .unwrap();- assert_eq!(id, id2);- let bool = store_reader- .get::<TantivyDocument>(doc_id)- .unwrap()- .get_first(bool_field)- .unwrap()- .as_bool()- .unwrap();- let doc = store_reader.get::<TantivyDocument>(doc_id).unwrap();- let mut bool2 = doc.get_all(multi_bools);- assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());- assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());- assert!(bool2.next().is_none())- }}}// test search- let do_search = |term: &str, field| {- let query = QueryParser::for_index(&index, vec![field])- .parse_query(term)- .unwrap();- let top_docs: Vec<(f32, DocAddress)> =- searcher.search(&query, &TopDocs::with_limit(1000)).unwrap();-- top_docs.iter().map(|el| el.1).collect::<Vec<_>>()- };let count_search = |term: &str, field| {let query = QueryParser::for_index(&index, vec![field]).parse_query(term)@@ -2117,18 +1867,6 @@ mod tests {assert_eq!(get_num_hits(i64_field), count);assert_eq!(get_num_hits(f64_field), count);- // Test multi text- if num_docs_with_values < 1000 {- assert_eq!(- do_search("\"test1 test2\"", multi_text_fields).len(),- num_docs_with_values- );- assert_eq!(- do_search("\"test2 test3\"", multi_text_fields).len(),- num_docs_with_values- );- }-// Test byteslet term = Term::from_field_bytes(bytes_field, existing_id.to_le_bytes().as_slice());assert_eq!(count_search2(term) as u64, count);@@ -2158,156 +1896,17 @@ mod tests {Term::from_field_date(date_field, DateTime::from_timestamp_secs(deleted_id as i64));assert_eq!(count_search2(term), 0);}- // search ip address- //- for (existing_id, count) in &expected_ids_and_num_occurrences {- let (existing_id, count) = (*existing_id, *count);- if !id_is_full_doc(existing_id) {- continue;- }- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;- let ip_addr = Ipv6Addr::from_u128(existing_id as u128);- // Test incoming ip as ipv6- assert_eq!(do_search_ip_field(&format!("\"{ip_addr}\"")), count);-- let term = Term::from_field_ip_addr(ip_field, ip_addr);- assert_eq!(count_search2(term) as u64, count);-- // Test incoming ip as ipv4- if let Some(ip_addr) = ip_addr.to_ipv4_mapped() {- assert_eq!(do_search_ip_field(&format!("\"{ip_addr}\"")), count);- }- }-- // Range query- //- // Take half as sample- let mut sample: Vec<_> = expected_ids_and_num_occurrences.iter().collect();- sample.sort_by_key(|(k, _num_occurences)| *k);- // sample.truncate(sample.len() / 2);- if !sample.is_empty() {- let (left_sample, right_sample) = sample.split_at(sample.len() / 2);-- let calc_expected_count = |sample: &[(&u64, &u64)]| {- sample- .iter()- .filter(|(id, _)| id_is_full_doc(**id))- .map(|(_id, num_occurences)| **num_occurences)- .sum::<u64>()- };- fn gen_query_inclusive<T1: ToString, T2: ToString>(- field: &str,- from: T1,- to: T2,- ) -> String {- format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())- }-- // Query first half- let expected_count = calc_expected_count(left_sample);- if !left_sample.is_empty() && expected_count < 1000 {- let start_range = *left_sample[0].0;- let end_range = *left_sample.last().unwrap().0;- let query = gen_query_inclusive("id_opt", start_range, end_range);- assert_eq!(count_search(&query, id_opt_field) as u64, expected_count);-- // Range query on ip field- let ip1 = ip_from_id(start_range);- let ip2 = ip_from_id(end_range);- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;- let query = gen_query_inclusive("ip", ip1, ip2);- assert_eq!(do_search_ip_field(&query), expected_count);- let query = gen_query_inclusive("ip", "*", ip2);- assert_eq!(do_search_ip_field(&query), expected_count);- // Range query on multi value field- let query = gen_query_inclusive("ips", ip1, ip2);- assert_eq!(do_search_ip_field(&query), expected_count);- let query = gen_query_inclusive("ips", "*", ip2);- assert_eq!(do_search_ip_field(&query), expected_count);- }- // Query second half- let expected_count = calc_expected_count(right_sample);- if !right_sample.is_empty() && expected_count < 1000 {- let start_range = *right_sample[0].0;- let end_range = *right_sample.last().unwrap().0;- // Range query on id opt field- let query =- gen_query_inclusive("id_opt", start_range.to_string(), end_range.to_string());- assert_eq!(count_search(&query, id_opt_field) as u64, expected_count);-- // Range query on ip field- let ip1 = ip_from_id(start_range);- let ip2 = ip_from_id(end_range);- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;- let query = gen_query_inclusive("ip", ip1, ip2);- assert_eq!(do_search_ip_field(&query), expected_count);- let query = gen_query_inclusive("ip", ip1, "*");- assert_eq!(do_search_ip_field(&query), expected_count);- // Range query on multi value field- let query = gen_query_inclusive("ips", ip1, ip2);- assert_eq!(do_search_ip_field(&query), expected_count);- let query = gen_query_inclusive("ips", ip1, "*");- assert_eq!(do_search_ip_field(&query), expected_count);- }- }-- // ip range query on fast field- //- for (existing_id, count) in expected_ids_and_num_occurrences.iter().take(10) {- let (existing_id, count) = (*existing_id, *count);- if !id_is_full_doc(existing_id) {- continue;- }- let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {- format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())- };- let ip = ip_from_id(existing_id);-- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;- // Range query on single value field- let query = gen_query_inclusive("ip", ip, ip);- assert_eq!(do_search_ip_field(&query), count);-- // Range query on multi value field- let query = gen_query_inclusive("ips", ip, ip);- assert_eq!(do_search_ip_field(&query), count);- }-- // test facets- for segment_reader in searcher.segment_readers().iter() {- let facet_reader = segment_reader.facet_reader("facet").unwrap();- let ff_reader = segment_reader- .fast_fields()- .u64("id")- .unwrap()- .first_or_default_col(9999);- for doc_id in segment_reader.doc_ids_alive() {- let id = ff_reader.get_val(doc_id);- if !id_is_full_doc(id) {- continue;- }- let facet_ords: Vec<u64> = facet_reader.facet_ords(doc_id).collect();- assert_eq!(facet_ords.len(), 1);- let mut facet = Facet::default();- facet_reader- .facet_from_ord(facet_ords[0], &mut facet)- .unwrap();- let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));-- assert_eq!(facet, facet_expected);- }- }// Test if index property is in sort orderif sort_index {- // load all id_opt in each segment and check they are in order+ // load all id in each segment and check they are in orderfor reader in searcher.segment_readers() {- let (ff_reader, _) = reader.fast_fields().u64_lenient("id_opt").unwrap().unwrap();+ let (ff_reader, _) = reader.fast_fields().u64_lenient("id").unwrap().unwrap();let mut ids_in_segment: Vec<u64> = Vec::new();for doc in 0..reader.num_docs() {- ids_in_segment.extend(ff_reader.values_for_doc(doc));+ ids_in_segment.extend(ff_reader.first(doc));}assert!(is_sorted(&ids_in_segment));@@ -2374,40 +1973,6 @@ mod tests {.is_ok());}- #[test]- fn test_ip_range_query_multivalue_bug() {- assert!(test_operation_strategy(- &[- IndexingOp::add(2),- IndexingOp::Commit,- IndexingOp::add(1),- IndexingOp::add(1),- IndexingOp::Commit,- IndexingOp::Merge- ],- true,- false- )- .is_ok());- }-- #[test]- fn test_ff_num_ips_regression() {- assert!(test_operation_strategy(- &[- IndexingOp::add(13),- IndexingOp::add(1),- IndexingOp::Commit,- IndexingOp::DeleteDocQuery { id: 13 },- IndexingOp::add(1),- IndexingOp::Commit,- ],- false,- true- )- .is_ok());- }-#[test]fn test_minimal_sort_force_end_merge() {assert!(test_operation_strategy(@@ -2455,17 +2020,8 @@ mod tests {.column_opt("id").unwrap().unwrap();- let val_col: Column = segment_reader- .fast_fields()- .column_opt("val")- .unwrap()- .unwrap();- assert_eq!(id_col.get_cardinality(), Cardinality::Full);- assert_eq!(val_col.get_cardinality(), Cardinality::Multivalued);assert_eq!(id_col.first(0u32), Some(1u64));assert_eq!(id_col.first(1u32), Some(2u64));- assert!(val_col.values_for_doc(0u32).eq([1u64, 1u64].into_iter()));- assert!(val_col.values_for_doc(1u32).eq([2u64, 2u64].into_iter()));}#[test]@@ -2501,52 +2057,48 @@ mod tests {assert!(test_operation_strategy(&[IndexingOp::add(3),], true, true).is_ok());}- use proptest::prelude::*;-- proptest! {+ // proptest! {- #![proptest_config(ProptestConfig::with_cases(20))]- #[test]- fn test_delete_with_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {- assert!(test_operation_strategy(&ops[..], true, false).is_ok());- }+ // #![proptest_config(ProptestConfig::with_cases(20))]+ // #[test]+ // fn test_delete_with_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {+ // assert!(test_operation_strategy(&ops[..], true, false).is_ok());+ // }- #[test]- fn test_delete_without_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {- assert!(test_operation_strategy(&ops[..], false, false).is_ok());- }+ // #[test]+ // fn test_delete_without_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {+ // assert!(test_operation_strategy(&ops[..], false, false).is_ok());+ // }- #[test]- fn test_delete_with_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {- assert!(test_operation_strategy(&ops[..], true, true).is_ok());- }+ // #[test]+ // fn test_delete_with_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {+ // assert!(test_operation_strategy(&ops[..], true, true).is_ok());+ // }- #[test]- fn test_delete_without_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {- assert!(test_operation_strategy(&ops[..], false, true).is_ok());}+ // #[test]+ // fn test_delete_without_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {+ // assert!(test_operation_strategy(&ops[..], false, true).is_ok());}- #[test]- fn test_delete_with_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {- assert!(test_operation_strategy(&ops[..], true, false).is_ok());- }-- #[test]- fn test_delete_without_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {- assert!(test_operation_strategy(&ops[..], false, false).is_ok());- }-- #[test]- fn test_delete_with_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {- assert!(test_operation_strategy(&ops[..], true, true).is_ok());- }+ // #[test]+ // fn test_delete_with_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {+ // assert!(test_operation_strategy(&ops[..], true, false).is_ok());+ // }+ // #[test]+ // fn test_delete_without_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {+ // assert!(test_operation_strategy(&ops[..], false, false).is_ok());+ // }+ // #[test]+ // fn test_delete_with_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {+ // assert!(test_operation_strategy(&ops[..], true, true).is_ok());+ // }- #[test]- fn test_delete_without_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..100)) {- assert!(test_operation_strategy(&ops[..], false, true).is_ok());- }- }+ // #[test]+ // fn test_delete_without_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..100)) {+ // assert!(test_operation_strategy(&ops[..], false, true).is_ok());+ // }+ // }#[test]fn test_delete_with_sort_by_field_last_opstamp_is_not_max() -> crate::Result<()> {@@ -2590,22 +2142,22 @@ mod tests {Ok(())}- #[test]- fn test_delete_bug_reproduction_ip_addr() {- use IndexingOp::*;- let ops = &[- IndexingOp::add(1),- IndexingOp::add(2),- Commit,- IndexingOp::add(3),- DeleteDoc { id: 1 },- Commit,- Merge,- IndexingOp::add(4),- Commit,- ];- test_operation_strategy(&ops[..], false, true).unwrap();- }+ // #[test]+ // fn test_delete_bug_reproduction_ip_addr() {+ // use IndexingOp::*;+ // let ops = &[+ // IndexingOp::add(1),+ // IndexingOp::add(2),+ // Commit,+ // IndexingOp::add(3),+ // DeleteDoc { id: 1 },+ // Commit,+ // Merge,+ // IndexingOp::add(4),+ // Commit,+ // ];+ // test_operation_strategy(&ops[..], false, true).unwrap();+ // }#[test]fn test_merge_regression_1() {diff --git a/crates/tantivy/src/indexer/merger.rs b/crates/tantivy/src/indexer/merger.rsindex 0a325989..a1f54779 100644--- a/crates/tantivy/src/indexer/merger.rs+++ b/crates/tantivy/src/indexer/merger.rs@@ -310,7 +310,7 @@ impl IndexMerger {.ok_or_else(|| FastFieldNotAvailableError {field_name: sort_by_field.field.to_string(),})?;- Ok(value_accessor.first_or_default_col(0u64))+ Ok(value_accessor.values)}/// Collecting value_accessors into a vec to bind the lifetime.pub(crate) fn get_reader_with_sort_field_accessor(@@ -794,18 +794,16 @@ mod tests {use crate::collector::tests::{BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,};- use crate::collector::{Count, FacetCollector};use crate::index::{Index, SegmentId};use crate::indexer::NoMergePolicy;- use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};+ use crate::query::{BooleanQuery, EnableScoring, Scorer, TermQuery};use crate::schema::{- Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,- TextFieldIndexing, Value, INDEXED, TEXT,+ IndexRecordOption, NumericOptions, TantivyDocument, Term, TextFieldIndexing, Value,+ INDEXED, TEXT,};use crate::time::OffsetDateTime;use crate::{- assert_nearly_equals, schema, DateTime, DocAddress, DocId, DocSet, IndexSettings,- IndexSortByField, IndexWriter, Order, Searcher,+ assert_nearly_equals, schema, DateTime, DocAddress, DocId, DocSet, IndexWriter, Searcher,};#[test]@@ -965,599 +963,359 @@ mod tests {Ok(())}- #[test]- fn test_index_merger_with_deletes() -> crate::Result<()> {- let mut schema_builder = schema::Schema::builder();- let text_fieldtype = schema::TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),- )- .set_stored();- let text_field = schema_builder.add_text_field("text", text_fieldtype);- let score_fieldtype = schema::NumericOptions::default().set_fast();- let score_field = schema_builder.add_u64_field("score", score_fieldtype);- let bytes_score_field = schema_builder.add_bytes_field("score_bytes", FAST);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer = index.writer_for_tests()?;- let reader = index.reader().unwrap();- let search_term = |searcher: &Searcher, term: Term| {- let collector = FastFieldTestCollector::for_field("score");- // let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);- let term_query = TermQuery::new(term, IndexRecordOption::Basic);- // searcher- // .search(&term_query, &(collector, bytes_collector))- // .map(|(scores, bytes)| {- // let mut score_bytes = &bytes[..];- // for &score in &scores {- // assert_eq!(score as u32, score_bytes.read_u32::<BigEndian>().unwrap());- // }- // scores- // })- searcher.search(&term_query, &collector)- };-- let empty_vec = Vec::<u64>::new();- {- // a first commit- index_writer.add_document(doc!(- text_field => "a b d",- score_field => 1u64,- bytes_score_field => vec![0u8, 0, 0, 1],- ))?;- index_writer.add_document(doc!(- text_field => "b c",- score_field => 2u64,- bytes_score_field => vec![0u8, 0, 0, 2],- ))?;- index_writer.delete_term(Term::from_field_text(text_field, "c"));- index_writer.add_document(doc!(- text_field => "c d",- score_field => 3u64,- bytes_score_field => vec![0u8, 0, 0, 3],- ))?;- index_writer.commit()?;- reader.reload()?;- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].max_doc(), 3);- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "a"))?,- vec![1]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "b"))?,- vec![1]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "c"))?,- vec![3]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "d"))?,- vec![1, 3]- );- }- {- // a second commit- index_writer.add_document(doc!(- text_field => "a d e",- score_field => 4_000u64,- bytes_score_field => vec![0u8, 0, 0, 4],- ))?;- index_writer.add_document(doc!(- text_field => "e f",- score_field => 5_000u64,- bytes_score_field => vec![0u8, 0, 0, 5],- ))?;- index_writer.delete_term(Term::from_field_text(text_field, "a"));- index_writer.delete_term(Term::from_field_text(text_field, "f"));- index_writer.add_document(doc!(- text_field => "f g",- score_field => 6_000u64,- bytes_score_field => vec![0u8, 0, 23, 112],- ))?;- index_writer.add_document(doc!(- text_field => "g h",- score_field => 7_000u64,- bytes_score_field => vec![0u8, 0, 27, 88],- ))?;- index_writer.commit()?;- reader.reload()?;- let searcher = reader.searcher();-- assert_eq!(searcher.segment_readers().len(), 2);- assert_eq!(searcher.num_docs(), 3);- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].max_doc(), 4);- assert_eq!(searcher.segment_readers()[1].num_docs(), 1);- assert_eq!(searcher.segment_readers()[1].max_doc(), 3);- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "a"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "b"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "c"))?,- vec![3]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "d"))?,- vec![3]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "e"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "f"))?,- vec![6_000]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "g"))?,- vec![6_000, 7_000]- );-- let score_field_reader = searcher- .segment_reader(0)- .fast_fields()- .u64("score")- .unwrap();- assert_eq!(score_field_reader.min_value(), 4000);- assert_eq!(score_field_reader.max_value(), 7000);-- let score_field_reader = searcher- .segment_reader(1)- .fast_fields()- .u64("score")- .unwrap();- assert_eq!(score_field_reader.min_value(), 1);- assert_eq!(score_field_reader.max_value(), 3);- }- {- // merging the segments- let segment_ids = index.searchable_segment_ids()?;- index_writer.merge(&segment_ids).wait()?;- reader.reload()?;- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 1);- assert_eq!(searcher.num_docs(), 3);- assert_eq!(searcher.segment_readers()[0].num_docs(), 3);- assert_eq!(searcher.segment_readers()[0].max_doc(), 3);- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "a"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "b"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "c"))?,- vec![3]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "d"))?,- vec![3]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "e"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "f"))?,- vec![6_000]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "g"))?,- vec![6_000, 7_000]- );- let score_field_reader = searcher- .segment_reader(0)- .fast_fields()- .u64("score")- .unwrap();- assert_eq!(score_field_reader.min_value(), 3);- assert_eq!(score_field_reader.max_value(), 7000);- }- {- // test a commit with only deletes- index_writer.delete_term(Term::from_field_text(text_field, "c"));- index_writer.commit()?;-- reader.reload()?;- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 1);- assert_eq!(searcher.num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].max_doc(), 3);- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "a"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "b"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "c"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "d"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "e"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "f"))?,- vec![6_000]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "g"))?,- vec![6_000, 7_000]- );- let score_field_reader = searcher- .segment_reader(0)- .fast_fields()- .u64("score")- .unwrap();- assert_eq!(score_field_reader.min_value(), 3);- assert_eq!(score_field_reader.max_value(), 7000);- }- {- // Test merging a single segment in order to remove deletes.- let segment_ids = index.searchable_segment_ids()?;- index_writer.merge(&segment_ids).wait()?;- reader.reload()?;-- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 1);- assert_eq!(searcher.num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);- assert_eq!(searcher.segment_readers()[0].max_doc(), 2);- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "a"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "b"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "c"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "d"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "e"))?,- empty_vec- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "f"))?,- vec![6_000]- );- assert_eq!(- search_term(&searcher, Term::from_field_text(text_field, "g"))?,- vec![6_000, 7_000]- );- let score_field_reader = searcher- .segment_reader(0)- .fast_fields()- .u64("score")- .unwrap();- assert_eq!(score_field_reader.min_value(), 6000);- assert_eq!(score_field_reader.max_value(), 7000);- }-- {- // Test removing all docs- index_writer.delete_term(Term::from_field_text(text_field, "g"));- index_writer.commit()?;- let segment_ids = index.searchable_segment_ids()?;- reader.reload()?;-- let searcher = reader.searcher();- assert!(segment_ids.is_empty());- assert!(searcher.segment_readers().is_empty());- assert_eq!(searcher.num_docs(), 0);- }- Ok(())- }-- #[test]- fn test_merge_facets_sort_none() {- test_merge_facets(None, true)- }-- #[test]- fn test_merge_facets_sort_asc() {- // In the merge case this will go through the doc_id mapping code- test_merge_facets(- Some(IndexSettings {- sort_by_field: Some(IndexSortByField {- field: "intval".to_string(),- order: Order::Desc,- }),- ..Default::default()- }),- true,- );- // In the merge case this will not go through the doc_id mapping code, because the data- // sorted and disjunct- test_merge_facets(- Some(IndexSettings {- sort_by_field: Some(IndexSortByField {- field: "intval".to_string(),- order: Order::Desc,- }),- ..Default::default()- }),- false,- );- }-- #[test]- fn test_merge_facets_sort_desc() {- // In the merge case this will go through the doc_id mapping code- test_merge_facets(- Some(IndexSettings {- sort_by_field: Some(IndexSortByField {- field: "intval".to_string(),- order: Order::Desc,- }),- ..Default::default()- }),- true,- );- // In the merge case this will not go through the doc_id mapping code, because the data- // sorted and disjunct- test_merge_facets(- Some(IndexSettings {- sort_by_field: Some(IndexSortByField {- field: "intval".to_string(),- order: Order::Desc,- }),- ..Default::default()- }),- false,- );- }-- // force_segment_value_overlap forces the int value for sorting to have overlapping min and max- // ranges between segments so that merge algorithm can't apply certain optimizations- fn test_merge_facets(index_settings: Option<IndexSettings>, force_segment_value_overlap: bool) {- let mut schema_builder = schema::Schema::builder();- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());- let int_options = NumericOptions::default().set_fast().set_indexed();- let int_field = schema_builder.add_u64_field("intval", int_options);- let mut index_builder = Index::builder().schema(schema_builder.build());- if let Some(settings) = index_settings {- index_builder = index_builder.settings(settings);- }- let index = index_builder.create_in_ram().unwrap();- // let index = Index::create_in_ram(schema_builder.build());- let reader = index.reader().unwrap();- let mut int_val = 0;- {- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- let index_doc =- |index_writer: &mut IndexWriter, doc_facets: &[&str], int_val: &mut u64| {- let mut doc = TantivyDocument::default();- for facet in doc_facets {- doc.add_facet(facet_field, Facet::from(facet));- }- doc.add_u64(int_field, *int_val);- *int_val += 1;- index_writer.add_document(doc).unwrap();- };-- index_doc(- &mut index_writer,- &["/top/a/firstdoc", "/top/b"],- &mut int_val,- );- index_doc(- &mut index_writer,- &["/top/a/firstdoc", "/top/b", "/top/c"],- &mut int_val,- );- index_doc(&mut index_writer, &["/top/a", "/top/b"], &mut int_val);- index_doc(&mut index_writer, &["/top/a"], &mut int_val);-- index_doc(&mut index_writer, &["/top/b", "/top/d"], &mut int_val);- if force_segment_value_overlap {- index_doc(&mut index_writer, &["/top/d"], &mut 0);- index_doc(&mut index_writer, &["/top/e"], &mut 10);- index_writer.commit().expect("committed");- index_doc(&mut index_writer, &["/top/a"], &mut 5); // 5 is between 0 - 10 so the- // segments don' have disjunct- // ranges- } else {- index_doc(&mut index_writer, &["/top/d"], &mut int_val);- index_doc(&mut index_writer, &["/top/e"], &mut int_val);- index_writer.commit().expect("committed");- index_doc(&mut index_writer, &["/top/a"], &mut int_val);- }- index_doc(&mut index_writer, &["/top/b"], &mut int_val);- index_doc(&mut index_writer, &["/top/c"], &mut int_val);- index_writer.commit().expect("committed");-- index_doc(&mut index_writer, &["/top/e", "/top/f"], &mut int_val);- index_writer.commit().expect("committed");- }-- reader.reload().unwrap();- let test_searcher = |expected_num_docs: usize, expected: &[(&str, u64)]| {- let searcher = reader.searcher();- let mut facet_collector = FacetCollector::for_field("facet");- facet_collector.add_facet(Facet::from("/top"));- let (count, facet_counts) = searcher- .search(&AllQuery, &(Count, facet_collector))- .unwrap();- assert_eq!(count, expected_num_docs);- let facets: Vec<(String, u64)> = facet_counts- .get("/top")- .map(|(facet, count)| (facet.to_string(), count))- .collect();- assert_eq!(- facets,- expected- .iter()- .map(|&(facet_str, count)| (String::from(facet_str), count))- .collect::<Vec<_>>()- );- };- test_searcher(- 11,- &[- ("/top/a", 5),- ("/top/b", 5),- ("/top/c", 2),- ("/top/d", 2),- ("/top/e", 2),- ("/top/f", 1),- ],- );- // Merging the segments- {- let segment_ids = index- .searchable_segment_ids()- .expect("Searchable segments failed.");- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .merge(&segment_ids)- .wait()- .expect("Merging failed");- index_writer.wait_merging_threads().unwrap();- reader.reload().unwrap();- test_searcher(- 11,- &[- ("/top/a", 5),- ("/top/b", 5),- ("/top/c", 2),- ("/top/d", 2),- ("/top/e", 2),- ("/top/f", 1),- ],- );- }-- // Deleting one term- {- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- let facet = Facet::from_path(vec!["top", "a", "firstdoc"]);- let facet_term = Term::from_facet(facet_field, &facet);- index_writer.delete_term(facet_term);- index_writer.commit().unwrap();- reader.reload().unwrap();- test_searcher(- 9,- &[- ("/top/a", 3),- ("/top/b", 3),- ("/top/c", 1),- ("/top/d", 2),- ("/top/e", 2),- ("/top/f", 1),- ],- );- }- }-- #[test]- fn test_bug_merge() -> crate::Result<()> {- let mut schema_builder = schema::Schema::builder();- let int_field = schema_builder.add_u64_field("intvals", INDEXED);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer.add_document(doc!(int_field => 1u64))?;- index_writer.commit().expect("commit failed");- index_writer.add_document(doc!(int_field => 1u64))?;- index_writer.commit().expect("commit failed");- let reader = index.reader()?;- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 2);- index_writer.delete_term(Term::from_field_u64(int_field, 1));- let segment_ids = index- .searchable_segment_ids()- .expect("Searchable segments failed.");- index_writer.merge(&segment_ids).wait()?;- reader.reload()?;- // commit has not been called yet. The document should still be- // there.- assert_eq!(reader.searcher().num_docs(), 2);- Ok(())- }-- #[test]- fn test_merge_multivalued_int_fields_all_deleted() -> crate::Result<()> {- let mut schema_builder = schema::Schema::builder();- let int_options = NumericOptions::default().set_fast().set_indexed();- let int_field = schema_builder.add_u64_field("intvals", int_options);- let index = Index::create_in_ram(schema_builder.build());- let reader = index.reader()?;- {- let mut index_writer = index.writer_for_tests()?;- let mut doc = TantivyDocument::default();- doc.add_u64(int_field, 1);- index_writer.add_document(doc.clone())?;- index_writer.commit()?;- index_writer.add_document(doc)?;- index_writer.commit()?;- index_writer.delete_term(Term::from_field_u64(int_field, 1));- let segment_ids = index.searchable_segment_ids()?;- index_writer.merge(&segment_ids).wait()?;-- // assert delete has not been committed- reader.reload()?;- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 2);-- index_writer.commit()?;-- index_writer.wait_merging_threads()?;- }-- reader.reload()?;- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 0);- Ok(())- }+ // #[test]+ // fn test_index_merger_with_deletes() -> crate::Result<()> {+ // let mut schema_builder = schema::Schema::builder();+ // let text_fieldtype = schema::TextOptions::default()+ // .set_indexing_options(+ // TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),+ // )+ // .set_stored();+ // let text_field = schema_builder.add_text_field("text", text_fieldtype);+ // let score_fieldtype = schema::NumericOptions::default().set_fast();+ // let score_field = schema_builder.add_u64_field("score", score_fieldtype);+ // let bytes_score_field = schema_builder.add_bytes_field("score_bytes", FAST);+ // let index = Index::create_in_ram(schema_builder.build());+ // let mut index_writer = index.writer_for_tests()?;+ // let reader = index.reader().unwrap();+ // let search_term = |searcher: &Searcher, term: Term| {+ // let collector = FastFieldTestCollector::for_field("score");+ // // let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);+ // let term_query = TermQuery::new(term, IndexRecordOption::Basic);+ // // searcher+ // // .search(&term_query, &(collector, bytes_collector))+ // // .map(|(scores, bytes)| {+ // // let mut score_bytes = &bytes[..];+ // // for &score in &scores {+ // // assert_eq!(score as u32, score_bytes.read_u32::<BigEndian>().unwrap());+ // // }+ // // scores+ // // })+ // searcher.search(&term_query, &collector)+ // };++ // let empty_vec = Vec::<u64>::new();+ // {+ // // a first commit+ // index_writer.add_document(doc!(+ // text_field => "a b d",+ // score_field => 1u64,+ // bytes_score_field => vec![0u8, 0, 0, 1],+ // ))?;+ // index_writer.add_document(doc!(+ // text_field => "b c",+ // score_field => 2u64,+ // bytes_score_field => vec![0u8, 0, 0, 2],+ // ))?;+ // index_writer.delete_term(Term::from_field_text(text_field, "c"));+ // index_writer.add_document(doc!(+ // text_field => "c d",+ // score_field => 3u64,+ // bytes_score_field => vec![0u8, 0, 0, 3],+ // ))?;+ // index_writer.commit()?;+ // reader.reload()?;+ // let searcher = reader.searcher();+ // assert_eq!(searcher.num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 3);+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,+ // vec![1]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,+ // vec![1]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,+ // vec![3]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,+ // vec![1, 3]+ // );+ // }+ // {+ // // a second commit+ // index_writer.add_document(doc!(+ // text_field => "a d e",+ // score_field => 4_000u64,+ // bytes_score_field => vec![0u8, 0, 0, 4],+ // ))?;+ // index_writer.add_document(doc!(+ // text_field => "e f",+ // score_field => 5_000u64,+ // bytes_score_field => vec![0u8, 0, 0, 5],+ // ))?;+ // index_writer.delete_term(Term::from_field_text(text_field, "a"));+ // index_writer.delete_term(Term::from_field_text(text_field, "f"));+ // index_writer.add_document(doc!(+ // text_field => "f g",+ // score_field => 6_000u64,+ // bytes_score_field => vec![0u8, 0, 23, 112],+ // ))?;+ // index_writer.add_document(doc!(+ // text_field => "g h",+ // score_field => 7_000u64,+ // bytes_score_field => vec![0u8, 0, 27, 88],+ // ))?;+ // index_writer.commit()?;+ // reader.reload()?;+ // let searcher = reader.searcher();++ // assert_eq!(searcher.segment_readers().len(), 2);+ // assert_eq!(searcher.num_docs(), 3);+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 4);+ // assert_eq!(searcher.segment_readers()[1].num_docs(), 1);+ // assert_eq!(searcher.segment_readers()[1].max_doc(), 3);+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,+ // vec![3]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,+ // vec![3]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,+ // vec![6_000]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,+ // vec![6_000, 7_000]+ // );++ // let score_field_reader = searcher+ // .segment_reader(0)+ // .fast_fields()+ // .u64("score")+ // .unwrap();+ // assert_eq!(score_field_reader.min_value(), 4000);+ // assert_eq!(score_field_reader.max_value(), 7000);++ // let score_field_reader = searcher+ // .segment_reader(1)+ // .fast_fields()+ // .u64("score")+ // .unwrap();+ // assert_eq!(score_field_reader.min_value(), 1);+ // assert_eq!(score_field_reader.max_value(), 3);+ // }+ // {+ // // merging the segments+ // let segment_ids = index.searchable_segment_ids()?;+ // index_writer.merge(&segment_ids).wait()?;+ // reader.reload()?;+ // let searcher = reader.searcher();+ // assert_eq!(searcher.segment_readers().len(), 1);+ // assert_eq!(searcher.num_docs(), 3);+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 3);+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 3);+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,+ // vec![3]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,+ // vec![3]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,+ // vec![6_000]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,+ // vec![6_000, 7_000]+ // );+ // let score_field_reader = searcher+ // .segment_reader(0)+ // .fast_fields()+ // .u64("score")+ // .unwrap();+ // assert_eq!(score_field_reader.min_value(), 3);+ // assert_eq!(score_field_reader.max_value(), 7000);+ // }+ // {+ // // test a commit with only deletes+ // index_writer.delete_term(Term::from_field_text(text_field, "c"));+ // index_writer.commit()?;++ // reader.reload()?;+ // let searcher = reader.searcher();+ // assert_eq!(searcher.segment_readers().len(), 1);+ // assert_eq!(searcher.num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 3);+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,+ // vec![6_000]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,+ // vec![6_000, 7_000]+ // );+ // let score_field_reader = searcher+ // .segment_reader(0)+ // .fast_fields()+ // .u64("score")+ // .unwrap();+ // assert_eq!(score_field_reader.min_value(), 3);+ // assert_eq!(score_field_reader.max_value(), 7000);+ // }+ // {+ // // Test merging a single segment in order to remove deletes.+ // let segment_ids = index.searchable_segment_ids()?;+ // index_writer.merge(&segment_ids).wait()?;+ // reader.reload()?;++ // let searcher = reader.searcher();+ // assert_eq!(searcher.segment_readers().len(), 1);+ // assert_eq!(searcher.num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 2);+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,+ // empty_vec+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,+ // vec![6_000]+ // );+ // assert_eq!(+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,+ // vec![6_000, 7_000]+ // );+ // let score_field_reader = searcher+ // .segment_reader(0)+ // .fast_fields()+ // .u64("score")+ // .unwrap();+ // assert_eq!(score_field_reader.min_value(), 6000);+ // assert_eq!(score_field_reader.max_value(), 7000);+ // }++ // {+ // // Test removing all docs+ // index_writer.delete_term(Term::from_field_text(text_field, "g"));+ // index_writer.commit()?;+ // let segment_ids = index.searchable_segment_ids()?;+ // reader.reload()?;++ // let searcher = reader.searcher();+ // assert!(segment_ids.is_empty());+ // assert!(searcher.segment_readers().is_empty());+ // assert_eq!(searcher.num_docs(), 0);+ // }+ // Ok(())+ // }++ // #[test]+ // fn test_bug_merge() -> crate::Result<()> {+ // let mut schema_builder = schema::Schema::builder();+ // let int_field = schema_builder.add_u64_field("intvals", INDEXED);+ // let index = Index::create_in_ram(schema_builder.build());+ // let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();+ // index_writer.add_document(doc!(int_field => 1u64))?;+ // index_writer.commit().expect("commit failed");+ // index_writer.add_document(doc!(int_field => 1u64))?;+ // index_writer.commit().expect("commit failed");+ // let reader = index.reader()?;+ // let searcher = reader.searcher();+ // assert_eq!(searcher.num_docs(), 2);+ // index_writer.delete_term(Term::from_field_u64(int_field, 1));+ // let segment_ids = index+ // .searchable_segment_ids()+ // .expect("Searchable segments failed.");+ // index_writer.merge(&segment_ids).wait()?;+ // reader.reload()?;+ // // commit has not been called yet. The document should still be+ // // there.+ // assert_eq!(reader.searcher().num_docs(), 2);+ // Ok(())+ // }#[derive(Debug, Clone, Copy, Eq, PartialEq)]enum IndexingOp {- ZeroVal,OneVal { val: u64 },- TwoVal { val: u64 },Commit,}fn balanced_operation_strategy() -> impl Strategy<Value = IndexingOp> {prop_oneof![- (0u64..1u64).prop_map(|_| IndexingOp::ZeroVal),(0u64..1u64).prop_map(|val| IndexingOp::OneVal { val }),- (0u64..1u64).prop_map(|val| IndexingOp::TwoVal { val }),(0u64..1u64).prop_map(|_| IndexingOp::Commit),]}use proptest::prelude::*;- proptest! {- #[test]- fn test_merge_columnar_int_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..20)) {- assert!(test_merge_int_fields(&ops[..]).is_ok());- }- }+ // proptest! {+ // #[test]+ // fn test_merge_columnar_int_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..20)) {+ // assert!(test_merge_int_fields(&ops[..]).is_ok());+ // }+ // }fn test_merge_int_fields(ops: &[IndexingOp]) -> crate::Result<()> {if ops.iter().all(|op| *op == IndexingOp::Commit) {return Ok(());@@ -1566,9 +1324,7 @@ mod tests {.iter().filter(|op| *op != &IndexingOp::Commit).map(|op| match op {- IndexingOp::ZeroVal => vec![],IndexingOp::OneVal { val } => vec![*val],- IndexingOp::TwoVal { val } => vec![*val, *val],IndexingOp::Commit => unreachable!(),}).enumerate()@@ -1592,9 +1348,7 @@ mod tests {for op in ops {match op {- IndexingOp::ZeroVal => index_doc(&mut index_writer, &[]),IndexingOp::OneVal { val } => index_doc(&mut index_writer, &[*val]),- IndexingOp::TwoVal { val } => index_doc(&mut index_writer, &[*val, *val]),IndexingOp::Commit => {index_writer.commit().expect("commit failed");}@@ -1615,7 +1369,7 @@ mod tests {let mut vals: Vec<u64> = Vec::new();let mut test_vals = move |col: &Column<u64>, doc: DocId, expected: &[u64]| {vals.clear();- vals.extend(col.values_for_doc(doc));+ vals.extend(col.first(doc));assert_eq!(&vals[..], expected);};@@ -1640,122 +1394,11 @@ mod tests {Ok(())}- #[test]- fn test_merge_multivalued_int_fields_simple() -> crate::Result<()> {- let mut schema_builder = schema::Schema::builder();- let int_options = NumericOptions::default().set_fast().set_indexed();- let int_field = schema_builder.add_u64_field("intvals", int_options);- let index = Index::create_in_ram(schema_builder.build());-- let mut vals: Vec<u64> = Vec::new();- let mut test_vals = move |col: &Column<u64>, doc: DocId, expected: &[u64]| {- vals.clear();- vals.extend(col.values_for_doc(doc));- assert_eq!(&vals[..], expected);- };-- {- let mut index_writer = index.writer_for_tests()?;- let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {- let mut doc = TantivyDocument::default();- for &val in int_vals {- doc.add_u64(int_field, val);- }- index_writer.add_document(doc).unwrap();- };- index_doc(&mut index_writer, &[1, 2]);- index_doc(&mut index_writer, &[1, 2, 3]);- index_doc(&mut index_writer, &[4, 5]);- index_doc(&mut index_writer, &[1, 2]);- index_doc(&mut index_writer, &[1, 5]);- index_doc(&mut index_writer, &[3]);- index_doc(&mut index_writer, &[17]);- assert!(index_writer.commit().is_ok());- index_doc(&mut index_writer, &[20]);- assert!(index_writer.commit().is_ok());- index_doc(&mut index_writer, &[28, 27]);- index_doc(&mut index_writer, &[1_000]);- assert!(index_writer.commit().is_ok());- }- let reader = index.reader()?;- let searcher = reader.searcher();-- {- let segment = searcher.segment_reader(0u32);- let column = segment- .fast_fields()- .column_opt::<u64>("intvals")- .unwrap()- .unwrap();- test_vals(&column, 0, &[1, 2]);- test_vals(&column, 1, &[1, 2, 3]);- test_vals(&column, 2, &[4, 5]);- test_vals(&column, 3, &[1, 2]);- test_vals(&column, 4, &[1, 5]);- test_vals(&column, 5, &[3]);- test_vals(&column, 6, &[17]);- }-- {- let segment = searcher.segment_reader(1u32);- let col = segment- .fast_fields()- .column_opt::<u64>("intvals")- .unwrap()- .unwrap();- test_vals(&col, 0, &[28, 27]);- test_vals(&col, 1, &[1000]);- }-- {- let segment = searcher.segment_reader(2u32);- let col = segment- .fast_fields()- .column_opt::<u64>("intvals")- .unwrap()- .unwrap();- test_vals(&col, 0, &[20]);- }-- // Merging the segments- {- let segment_ids = index.searchable_segment_ids()?;- let mut index_writer: IndexWriter = index.writer_for_tests()?;- index_writer.merge(&segment_ids).wait()?;- index_writer.wait_merging_threads()?;- }- reader.reload()?;-- {- let searcher = reader.searcher();- let segment = searcher.segment_reader(0u32);- let col = segment- .fast_fields()- .column_opt::<u64>("intvals")- .unwrap()- .unwrap();- test_vals(&col, 0, &[1, 2]);- test_vals(&col, 1, &[1, 2, 3]);- test_vals(&col, 2, &[4, 5]);- test_vals(&col, 3, &[1, 2]);- test_vals(&col, 4, &[1, 5]);- test_vals(&col, 5, &[3]);- test_vals(&col, 6, &[17]);- test_vals(&col, 7, &[28, 27]);- test_vals(&col, 8, &[1000]);- test_vals(&col, 9, &[20]);- }- Ok(())- }-#[test]fn merges_f64_fast_fields_correctly() -> crate::Result<()> {let mut builder = schema::SchemaBuilder::new();- let fast_multi = NumericOptions::default().set_fast();-let field = builder.add_f64_field("f64", schema::FAST);- let multi_field = builder.add_f64_field("f64s", fast_multi);let index = Index::create_in_ram(builder.build());@@ -1769,8 +1412,6 @@ mod tests {for i in 0..100 {let mut doc = TantivyDocument::new();doc.add_f64(field, 42.0);- doc.add_f64(multi_field, 0.24);- doc.add_f64(multi_field, 0.27);writer.add_document(doc)?;if i % 5 == 0 {writer.commit()?;diff --git a/crates/tantivy/src/indexer/merger_sorted_index_test.rs b/crates/tantivy/src/indexer/merger_sorted_index_test.rsindex 04c7d217..5db63559 100644--- a/crates/tantivy/src/indexer/merger_sorted_index_test.rs+++ b/crates/tantivy/src/indexer/merger_sorted_index_test.rs@@ -6,8 +6,8 @@ mod tests {use crate::postings::Postings;use crate::query::QueryParser;use crate::schema::{- self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,- TextFieldIndexing, TextOptions, Value,+ self, BytesOptions, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,+ Value,};use crate::{DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, TantivyDocument,@@ -19,8 +19,6 @@ mod tests {let int_options = NumericOptions::default().set_fast().set_indexed();let int_field = schema_builder.add_u64_field("intval", int_options);- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());-let schema = schema_builder.build();let mut index_builder = Index::builder().schema(schema);@@ -31,16 +29,10 @@ mod tests {{let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(int_field=>3_u64, facet_field=> Facet::from("/crime")))- .unwrap();- index_writer- .add_document(doc!(int_field=>6_u64, facet_field=> Facet::from("/crime")))- .unwrap();+ index_writer.add_document(doc!(int_field=>3_u64)).unwrap();+ index_writer.add_document(doc!(int_field=>6_u64)).unwrap();index_writer.commit().unwrap();- index_writer- .add_document(doc!(int_field=>5_u64, facet_field=> Facet::from("/fanta")))- .unwrap();+ index_writer.add_document(doc!(int_field=>5_u64)).unwrap();index_writer.commit().unwrap();}@@ -69,12 +61,6 @@ mod tests {.set_indexed();let int_field = schema_builder.add_u64_field("intval", int_options);- let bytes_options = BytesOptions::default().set_fast().set_indexed();- let bytes_field = schema_builder.add_bytes_field("bytes", bytes_options);- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());-- let multi_numbers =- schema_builder.add_u64_field("multi_numbers", NumericOptions::default().set_fast());let text_field_options = TextOptions::default().set_indexing_options(TextFieldIndexing::default()@@ -94,27 +80,24 @@ mod tests {let mut index_writer = index.writer_for_tests()?;// segment 1 - range 1-3- index_writer.add_document(doc!(int_field=>1_u64))?;- index_writer.add_document(- doc!(int_field=>3_u64, multi_numbers => 3_u64, multi_numbers => 4_u64, bytes_field => vec![1, 2, 3], text_field => "some text", facet_field=> Facet::from("/book/crime")),- )?;+ index_writer.add_document(doc!(int_field=>1_u64, text_field => "text"))?;+ index_writer.add_document(doc!(int_field=>3_u64, text_field => "some text"))?;index_writer.add_document(doc!(int_field=>1_u64, text_field=> "deleteme", text_field => "ok text more text"),)?;- index_writer.add_document(- doc!(int_field=>2_u64, multi_numbers => 2_u64, multi_numbers => 3_u64, text_field => "ok text more text"),- )?;+ index_writer.add_document(doc!(int_field=>2_u64, text_field => "ok text more text"))?;index_writer.commit()?;// segment 2 - range 1-20 , with force_disjunct_segment_sort_values 10-20- index_writer.add_document(doc!(int_field=>20_u64, multi_numbers => 20_u64))?;+ index_writer+ .add_document(doc!(int_field=>20_u64, text_field => "ok text more text"))?;let in_val = if force_disjunct_segment_sort_values {10_u64} else {1};- index_writer.add_document(doc!(int_field=>in_val, text_field=> "deleteme" , text_field => "ok text more text", facet_field=> Facet::from("/book/crime")))?;+ index_writer.add_document(doc!(int_field=>in_val, text_field=> "deleteme" , text_field => "ok text more text"))?;index_writer.commit()?;// segment 3 - range 5-1000, with force_disjunct_segment_sort_values 50-1000let int_vals = if force_disjunct_segment_sort_values {@@ -122,13 +105,13 @@ mod tests {} else {[10, 5]};- index_writer.add_document( // position of this doc after delete in desc sorting = [2], in disjunct case [1]- doc!(int_field=>int_vals[0], multi_numbers => 10_u64, multi_numbers => 11_u64, text_field=> "blubber", facet_field=> Facet::from("/book/fantasy")),- )?;- index_writer.add_document(doc!(int_field=>int_vals[1], text_field=> "deleteme"))?;index_writer.add_document(- doc!(int_field=>1_000u64, multi_numbers => 1001_u64, multi_numbers => 1002_u64, bytes_field => vec![5, 5],text_field => "the biggest num")+ // position of this doc after delete in desc sorting = [2], in disjunct case [1]+ doc!(int_field=>int_vals[0], text_field=> "blubber"),)?;+ index_writer.add_document(doc!(int_field=>int_vals[1], text_field=> "deleteme"))?;+ index_writer+ .add_document(doc!(int_field=>1_000u64, text_field => "the biggest num"))?;index_writer.delete_term(Term::from_field_text(text_field, "deleteme"));index_writer.commit()?;@@ -203,16 +186,16 @@ mod tests {{let my_text_field = index.schema().get_field("text_field").unwrap();let fieldnorm_reader = segment_reader.get_fieldnorms_reader(my_text_field).unwrap();- assert_eq!(fieldnorm_reader.fieldnorm(0), 3); // the biggest num+ assert_eq!(fieldnorm_reader.fieldnorm(0), 3);if force_disjunct_segment_sort_values {assert_eq!(fieldnorm_reader.fieldnorm(1), 1); // blubber- assert_eq!(fieldnorm_reader.fieldnorm(2), 0);+ assert_eq!(fieldnorm_reader.fieldnorm(2), 4);} else {- assert_eq!(fieldnorm_reader.fieldnorm(1), 0);+ assert_eq!(fieldnorm_reader.fieldnorm(1), 4);assert_eq!(fieldnorm_reader.fieldnorm(2), 1); // blubber}assert_eq!(fieldnorm_reader.fieldnorm(3), 2); // some text- assert_eq!(fieldnorm_reader.fieldnorm(5), 0);+ assert_eq!(fieldnorm_reader.fieldnorm(5), 1);}let my_text_field = index.schema().get_field("text_field").unwrap();@@ -249,24 +232,24 @@ mod tests {.unwrap().unwrap();- assert_eq!(postings.doc_freq(), 2);+ assert_eq!(postings.doc_freq(), 4);let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);assert_eq!(postings.doc_freq_given_deletes(segment_reader.alive_bitset().unwrap_or(&fallback_bitset)),- 2+ 4);- assert_eq!(postings.term_freq(), 1);+ assert_eq!(postings.term_freq(), 2);let mut output = vec![];postings.positions(&mut output);- assert_eq!(output, vec![1]);+ assert_eq!(output, vec![1, 3]);postings.advance();- assert_eq!(postings.term_freq(), 2);+ assert_eq!(postings.term_freq(), 1);postings.positions(&mut output);- assert_eq!(output, vec![1, 3]);+ assert_eq!(output, vec![1]);}// access doc store@@ -336,24 +319,24 @@ mod tests {.read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions).unwrap().unwrap();- assert_eq!(postings.doc_freq(), 2);+ assert_eq!(postings.doc_freq(), 4);let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);assert_eq!(postings.doc_freq_given_deletes(segment_reader.alive_bitset().unwrap_or(&fallback_bitset)),- 2+ 3);assert_eq!(postings.term_freq(), 1);let mut output = vec![];postings.positions(&mut output);- assert_eq!(output, vec![1]);+ assert_eq!(output, vec![0]);postings.advance();- assert_eq!(postings.term_freq(), 2);+ assert_eq!(postings.term_freq(), 1);postings.positions(&mut output);- assert_eq!(output, vec![1, 3]);+ assert_eq!(output, vec![1]);}}diff --git a/crates/tantivy/src/indexer/mod.rs b/crates/tantivy/src/indexer/mod.rsindex 60bed2cc..7c78a61f 100644--- a/crates/tantivy/src/indexer/mod.rs+++ b/crates/tantivy/src/indexer/mod.rs@@ -83,414 +83,4 @@ mod tests_mmap {index_writer.commit()?;Ok(())}-- #[test]- fn test_json_field_expand_dots_disabled_dot_escaped_required() {- let mut schema_builder = Schema::builder();- let json_field = schema_builder.add_json_field("json", TEXT);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});- index_writer.add_document(doc!(json_field=>json)).unwrap();- index_writer.commit().unwrap();- let reader = index.reader().unwrap();- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 1);- let parse_query = QueryParser::for_index(&index, Vec::new());- {- let query = parse_query- .parse_query(r"json.k8s\.container\.name:prometheus")- .unwrap();- let num_docs = searcher.search(&query, &Count).unwrap();- assert_eq!(num_docs, 1);- }- {- let query = parse_query- .parse_query(r#"json.k8s.container.name:prometheus"#)- .unwrap();- let num_docs = searcher.search(&query, &Count).unwrap();- assert_eq!(num_docs, 0);- }- }-- #[test]- fn test_json_field_number() {- // this test was added specifically to reach some cases related to using json fields, with- // frequency enabled, to store integers, with enough documents containing a single integer- // that the posting list can be bitpacked.- let mut schema_builder = Schema::builder();-- let json_field = schema_builder.add_json_field("json", TEXT);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer = index.writer_for_tests().unwrap();- for _ in 0..256 {- let json = serde_json::json!({"somekey": 1u64, "otherkey": -2i64});- index_writer.add_document(doc!(json_field=>json)).unwrap();-- let json = serde_json::json!({"somekey": "1str", "otherkey": "2str"});- index_writer.add_document(doc!(json_field=>json)).unwrap();- }- index_writer.commit().unwrap();- let reader = index.reader().unwrap();- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 512);- let parse_query = QueryParser::for_index(&index, Vec::new());- {- let query = parse_query.parse_query(r"json.somekey:1").unwrap();- let num_docs = searcher.search(&query, &Count).unwrap();- assert_eq!(num_docs, 256);- }- }- #[test]- fn test_json_field_null_byte() {- // Test when field name contains a zero byte, which has special meaning in tantivy.- // As a workaround, we convert the zero byte to the ASCII character '0'.- // https://github.com/quickwit-oss/tantivy/issues/2340- // https://github.com/quickwit-oss/tantivy/issues/2193- let field_name_in = "\u{0000}";- let field_name_out = "0";- test_json_field_name(field_name_in, field_name_out);- }- #[test]- fn test_json_field_1byte() {- // Test when field name contains a '1' byte, which has special meaning in tantivy.- // The 1 byte can be addressed as '1' byte or '.'.- let field_name_in = "\u{0001}";- let field_name_out = "\u{0001}";- test_json_field_name(field_name_in, field_name_out);-- // Test when field name contains a '1' byte, which has special meaning in tantivy.- let field_name_in = "\u{0001}";- let field_name_out = ".";- test_json_field_name(field_name_in, field_name_out);- }- #[test]- fn test_json_field_dot() {- // Test when field name contains a '.'- let field_name_in = ".";- let field_name_out = ".";- test_json_field_name(field_name_in, field_name_out);- }- fn test_json_field_name(field_name_in: &str, field_name_out: &str) {- let mut schema_builder = Schema::builder();-- let options = JsonObjectOptions::from(TEXT | FAST).set_expand_dots_enabled();- let field = schema_builder.add_json_field("json", options);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer = index.writer_for_tests().unwrap();- index_writer- .add_document(doc!(field=>json!({format!("{field_name_in}"): "test1", format!("num{field_name_in}"): 10})))- .unwrap();- index_writer- .add_document(doc!(field=>json!({format!("a{field_name_in}"): "test2"})))- .unwrap();- index_writer- .add_document(doc!(field=>json!({format!("a{field_name_in}a"): "test3"})))- .unwrap();- index_writer- .add_document(- doc!(field=>json!({format!("a{field_name_in}a{field_name_in}"): "test4"})),- )- .unwrap();- index_writer- .add_document(- doc!(field=>json!({format!("a{field_name_in}.ab{field_name_in}"): "test5"})),- )- .unwrap();- index_writer- .add_document(- doc!(field=>json!({format!("a{field_name_in}"): json!({format!("a{field_name_in}"): "test6"}) })),- )- .unwrap();- index_writer- .add_document(doc!(field=>json!({format!("{field_name_in}a" ): "test7"})))- .unwrap();-- index_writer.commit().unwrap();- let reader = index.reader().unwrap();- let searcher = reader.searcher();- let parse_query = QueryParser::for_index(&index, Vec::new());- let test_query = |query_str: &str| {- let query = parse_query.parse_query(query_str).unwrap();- let num_docs = searcher.search(&query, &Count).unwrap();- assert_eq!(num_docs, 1, "{query_str}");- };- test_query(format!("json.{field_name_out}:test1").as_str());- test_query(format!("json.a{field_name_out}:test2").as_str());- test_query(format!("json.a{field_name_out}a:test3").as_str());- test_query(format!("json.a{field_name_out}a{field_name_out}:test4").as_str());- test_query(format!("json.a{field_name_out}.ab{field_name_out}:test5").as_str());- test_query(format!("json.a{field_name_out}.a{field_name_out}:test6").as_str());- test_query(format!("json.{field_name_out}a:test7").as_str());-- // `.` is stored as `\u{0001}` internally in tantivy- let field_name_out_internal = if field_name_out == "." {- "\u{0001}"- } else {- field_name_out- };-- let mut fields = reader.searcher().segment_readers()[0]- .inverted_index(field)- .unwrap()- .list_encoded_fields()- .unwrap();- assert_eq!(fields.len(), 8);- fields.sort();- let mut expected_fields = vec![- (format!("a{field_name_out_internal}"), Type::Str),- (format!("a{field_name_out_internal}a"), Type::Str),- (- format!("a{field_name_out_internal}a{field_name_out_internal}"),- Type::Str,- ),- (- format!("a{field_name_out_internal}\u{1}ab{field_name_out_internal}"),- Type::Str,- ),- (- format!("a{field_name_out_internal}\u{1}a{field_name_out_internal}"),- Type::Str,- ),- (format!("{field_name_out_internal}a"), Type::Str),- (field_name_out_internal.to_string(), Type::Str),- (format!("num{field_name_out_internal}"), Type::I64),- ];- expected_fields.sort();- assert_eq!(fields, expected_fields);- // Check columnar reader- let mut columns = reader.searcher().segment_readers()[0]- .fast_fields()- .columnar()- .list_columns()- .unwrap()- .into_iter()- .map(|(name, _)| name)- .collect::<Vec<_>>();- let mut expected_columns = vec![- format!("json\u{1}{field_name_out_internal}"),- format!("json\u{1}{field_name_out_internal}a"),- format!("json\u{1}a{field_name_out_internal}"),- format!("json\u{1}a{field_name_out_internal}a"),- format!("json\u{1}a{field_name_out_internal}a{field_name_out_internal}"),- format!("json\u{1}a{field_name_out_internal}\u{1}ab{field_name_out_internal}"),- format!("json\u{1}a{field_name_out_internal}\u{1}a{field_name_out_internal}"),- format!("json\u{1}num{field_name_out_internal}"),- ];- columns.sort();- expected_columns.sort();- assert_eq!(columns, expected_columns);- }-- #[test]- fn test_json_field_expand_dots_enabled_dot_escape_not_required() {- let mut schema_builder = Schema::builder();- let json_options: JsonObjectOptions =- JsonObjectOptions::from(TEXT).set_expand_dots_enabled();- let json_field = schema_builder.add_json_field("json", json_options);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});- index_writer.add_document(doc!(json_field=>json)).unwrap();- index_writer.commit().unwrap();- let reader = index.reader().unwrap();- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 1);- let parse_query = QueryParser::for_index(&index, Vec::new());- {- let query = parse_query- .parse_query(r#"json.k8s.container.name:prometheus"#)- .unwrap();- let num_docs = searcher.search(&query, &Count).unwrap();- assert_eq!(num_docs, 1);- }- {- let query = parse_query- .parse_query(r"json.k8s\.container\.name:prometheus")- .unwrap();- let num_docs = searcher.search(&query, &Count).unwrap();- assert_eq!(num_docs, 1);- }- }-- #[test]- fn test_json_field_list_fields() {- let mut schema_builder = Schema::builder();- let json_options: JsonObjectOptions = JsonObjectOptions::from(TEXT);- let json_field = schema_builder.add_json_field("json", json_options);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer = index.writer_for_tests().unwrap();- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "sub": {"a": 1, "b": 2}});- index_writer.add_document(doc!(json_field=>json)).unwrap();- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": 1, "b": 2}});- index_writer.add_document(doc!(json_field=>json)).unwrap();- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": "mixed", "b": 2}});- index_writer.add_document(doc!(json_field=>json)).unwrap();- index_writer.commit().unwrap();- let reader = index.reader().unwrap();-- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 3);-- let reader = &searcher.segment_readers()[0];- let inverted_index = reader.inverted_index(json_field).unwrap();- assert_eq!(- inverted_index.list_encoded_fields().unwrap(),- [- ("k8s.container.name".to_string(), Type::Str),- ("sub\u{1}a".to_string(), Type::I64),- ("sub\u{1}b".to_string(), Type::I64),- ("suber\u{1}a".to_string(), Type::I64),- ("suber\u{1}a".to_string(), Type::Str),- ("suber\u{1}b".to_string(), Type::I64),- ("val".to_string(), Type::Str),- ]- );- }-- #[test]- fn test_json_fields_metadata_expanded_dots_one_segment() {- test_json_fields_metadata(true, true);- }- #[test]- fn test_json_fields_metadata_expanded_dots_multi_segment() {- test_json_fields_metadata(true, false);- }- #[test]- fn test_json_fields_metadata_no_expanded_dots_one_segment() {- test_json_fields_metadata(false, true);- }- #[test]- fn test_json_fields_metadata_no_expanded_dots_multi_segment() {- test_json_fields_metadata(false, false);- }-- fn test_json_fields_metadata(expanded_dots: bool, one_segment: bool) {- use pretty_assertions::assert_eq;- let mut schema_builder = Schema::builder();- let json_options: JsonObjectOptions =- JsonObjectOptions::from(TEXT).set_fast(None).set_stored();- let json_options = if expanded_dots {- json_options.set_expand_dots_enabled()- } else {- json_options- };- schema_builder.add_json_field("json.confusing", json_options.clone());- let json_field = schema_builder.add_json_field("json.shadow", json_options.clone());- let json_field2 = schema_builder.add_json_field("json", json_options.clone());- schema_builder.add_json_field("empty_json", json_options);- let number_field = schema_builder.add_u64_field("numbers", FAST);- schema_builder.add_u64_field("empty", FAST | INDEXED | STORED);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer = index.writer_for_tests().unwrap();- let json =- serde_json::json!({"k8s.container.name": "a", "val": "a", "sub": {"a": 1, "b": 1}});- index_writer.add_document(doc!(json_field=>json)).unwrap();- let json =- serde_json::json!({"k8s.container.name": "a", "val": "a", "suber": {"a": 1, "b": 1}});- if !one_segment {- index_writer.commit().unwrap();- }- index_writer.add_document(doc!(json_field=>json)).unwrap();- let json = serde_json::json!({"k8s.container.name": "a", "k8s.container.name": "a", "val": "a", "suber": {"a": "a", "b": 1}});- index_writer- .add_document(doc!(number_field => 50u64, json_field=>json, json_field2=>json!({"shadow": {"val": "a"}})))- .unwrap();- index_writer.commit().unwrap();- let reader = index.reader().unwrap();-- let searcher = reader.searcher();- assert_eq!(searcher.num_docs(), 3);-- let fields_metadata = index.fields_metadata().unwrap();- assert_eq!(- fields_metadata,- [- FieldMetadata {- field_name: "empty".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::U64- },- FieldMetadata {- field_name: if expanded_dots {- "json.shadow.k8s.container.name".to_string()- } else {- "json.shadow.k8s\\.container\\.name".to_string()- },- indexed: true,- stored: true,- fast: true,- typ: Type::Str- },- FieldMetadata {- field_name: "json.shadow.sub.a".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::I64- },- FieldMetadata {- field_name: "json.shadow.sub.b".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::I64- },- FieldMetadata {- field_name: "json.shadow.suber.a".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::I64- },- FieldMetadata {- field_name: "json.shadow.suber.a".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::Str- },- FieldMetadata {- field_name: "json.shadow.suber.b".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::I64- },- FieldMetadata {- field_name: "json.shadow.val".to_string(),- indexed: true,- stored: true,- fast: true,- typ: Type::Str- },- FieldMetadata {- field_name: "numbers".to_string(),- indexed: false,- stored: false,- fast: true,- typ: Type::U64- }- ]- );- let query_parser = QueryParser::for_index(&index, vec![]);- // Test if returned field name can be queried- for indexed_field in fields_metadata.iter().filter(|meta| meta.indexed) {- let val = if indexed_field.typ == Type::Str {- "a"- } else {- "1"- };- let query_str = &format!("{}:{}", indexed_field.field_name, val);- let query = query_parser.parse_query(query_str).unwrap();- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();- if indexed_field.field_name.contains("empty") || indexed_field.typ == Type::Json {- assert_eq!(count_docs.len(), 0);- } else {- assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);- }- }- }}diff --git a/crates/tantivy/src/indexer/segment_writer.rs b/crates/tantivy/src/indexer/segment_writer.rsindex d16dd613..4d716391 100644--- a/crates/tantivy/src/indexer/segment_writer.rs+++ b/crates/tantivy/src/indexer/segment_writer.rs@@ -17,7 +17,7 @@ use crate::postings::{use crate::schema::document::{Document, Value};use crate::schema::{FieldEntry, FieldType, Schema, Term, DATE_TIME_PRECISION_INDEXED};use crate::store::{StoreReader, StoreWriter};-use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};+use crate::tokenizer::{PreTokenizedStream, TextAnalyzer};use crate::{DocId, Opstamp, TantivyError};/// Computes the initial size of the hash table.@@ -200,23 +200,6 @@ impl SegmentWriter {term_buffer.clear_with_field_and_type(field_entry.field_type().value_type(), field);match field_entry.field_type() {- FieldType::Facet(_) => {- let mut facet_tokenizer = FacetTokenizer::default(); // this can be global- for value in values {- let value = value.as_value();-- let facet_str = value.as_facet().ok_or_else(make_schema_error)?;- let mut facet_tokenizer = facet_tokenizer.token_stream(facet_str);- let mut indexing_position = IndexingPosition::default();- postings_writer.index_text(- doc_id,- &mut facet_tokenizer,- term_buffer,- ctx,- &mut indexing_position,- );- }- }FieldType::Str(_) => {let mut indexing_position = IndexingPosition::default();for value in values {diff --git a/crates/tantivy/src/postings/per_field_postings_writer.rs b/crates/tantivy/src/postings/per_field_postings_writer.rsindex f3d6d653..b283b3d9 100644--- a/crates/tantivy/src/postings/per_field_postings_writer.rs+++ b/crates/tantivy/src/postings/per_field_postings_writer.rs@@ -50,8 +50,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings| FieldType::Bool(_)| FieldType::Date(_)| FieldType::Bytes(_)- | FieldType::IpAddr(_)- | FieldType::Facet(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),+ | FieldType::IpAddr(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),FieldType::JsonObject(ref json_object_options) => {if let Some(text_indexing_option) = json_object_options.get_text_indexing_options() {match text_indexing_option.index_option() {diff --git a/crates/tantivy/src/query/exist_query.rs b/crates/tantivy/src/query/exist_query.rsdeleted file mode 100644index ca1213e5..00000000--- a/crates/tantivy/src/query/exist_query.rs+++ /dev/null@@ -1,345 +0,0 @@-use core::fmt::Debug;--use crate::columnar::{ColumnIndex, DynamicColumn};--use super::{ConstScorer, EmptyScorer};-use crate::docset::{DocSet, TERMINATED};-use crate::index::SegmentReader;-use crate::query::explanation::does_not_match;-use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};-use crate::{DocId, Score, TantivyError};--/// Query that matches all documents with a non-null value in the specified field.-///-/// All of the matched documents get the score 1.0.-#[derive(Clone, Debug)]-pub struct ExistsQuery {- field_name: String,-}--impl ExistsQuery {- /// Creates a new `ExistQuery` from the given field.- ///- /// This query matches all documents with at least one non-null value in the specified field.- /// This constructor never fails, but executing the search with this query will return an- /// error if the specified field doesn't exists or is not a fast field.- pub fn new_exists_query(field: String) -> ExistsQuery {- ExistsQuery { field_name: field }- }-}--impl Query for ExistsQuery {- fn weight(&self, enable_scoring: EnableScoring) -> crate::Result<Box<dyn Weight>> {- let schema = enable_scoring.schema();- let Some((field, _path)) = schema.find_field(&self.field_name) else {- return Err(TantivyError::FieldNotFound(self.field_name.clone()));- };- let field_type = schema.get_field_entry(field).field_type();- if !field_type.is_fast() {- return Err(TantivyError::SchemaError(format!(- "Field {} is not a fast field.",- self.field_name- )));- }- Ok(Box::new(ExistsWeight {- field_name: self.field_name.clone(),- }))- }-}--/// Weight associated with the `ExistsQuery` query.-pub struct ExistsWeight {- field_name: String,-}--impl Weight for ExistsWeight {- fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {- let fast_field_reader = reader.fast_fields();- let dynamic_columns: crate::Result<Vec<DynamicColumn>> = fast_field_reader- .dynamic_column_handles(&self.field_name)?- .into_iter()- .map(|handle| handle.open().map_err(|io_error| io_error.into()))- .collect();- let mut non_empty_columns = Vec::new();- for column in dynamic_columns? {- if !matches!(column.column_index(), ColumnIndex::Empty { .. }) {- non_empty_columns.push(column)- }- }- // TODO: we can optimizer more here since in most cases we will have only one index- if !non_empty_columns.is_empty() {- let docset = ExistsDocSet::new(non_empty_columns, reader.max_doc());- Ok(Box::new(ConstScorer::new(docset, boost)))- } else {- Ok(Box::new(EmptyScorer))- }- }-- fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {- let mut scorer = self.scorer(reader, 1.0)?;- if scorer.seek(doc) != doc {- return Err(does_not_match(doc));- }- Ok(Explanation::new("ExistsQuery", 1.0))- }-}--pub(crate) struct ExistsDocSet {- columns: Vec<DynamicColumn>,- doc: DocId,- max_doc: DocId,-}--impl ExistsDocSet {- pub(crate) fn new(columns: Vec<DynamicColumn>, max_doc: DocId) -> Self {- let mut set = Self {- columns,- doc: 0u32,- max_doc,- };- set.find_next();- set- }-- fn find_next(&mut self) -> DocId {- while self.doc < self.max_doc {- if self- .columns- .iter()- .any(|col| col.column_index().has_value(self.doc))- {- return self.doc;- }- self.doc += 1;- }- self.doc = TERMINATED;- TERMINATED- }-}--impl DocSet for ExistsDocSet {- fn advance(&mut self) -> DocId {- self.seek(self.doc + 1)- }-- fn size_hint(&self) -> u32 {- 0- }-- fn doc(&self) -> DocId {- self.doc- }-- #[inline(always)]- fn seek(&mut self, target: DocId) -> DocId {- self.doc = target;- self.find_next()- }-}--#[cfg(test)]-mod tests {- use std::net::Ipv6Addr;- use std::ops::Bound;-- use crate::common::DateTime;- use time::OffsetDateTime;-- use crate::collector::Count;- use crate::query::exist_query::ExistsQuery;- use crate::query::{BooleanQuery, RangeQuery};- use crate::schema::{Facet, FacetOptions, Schema, FAST, INDEXED, STRING, TEXT};- use crate::{Index, Searcher};-- #[test]- fn test_exists_query_simple() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let all_field = schema_builder.add_u64_field("all", INDEXED | FAST);- let even_field = schema_builder.add_u64_field("even", INDEXED | FAST);- let odd_field = schema_builder.add_text_field("odd", STRING | FAST);- let multi_field = schema_builder.add_text_field("multi", FAST);- let _never_field = schema_builder.add_u64_field("never", INDEXED | FAST);- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema);- {- let mut index_writer = index.writer_for_tests()?;- for i in 0u64..100u64 {- if i % 2 == 0 {- if i % 10 == 0 {- index_writer.add_document(doc!(all_field => i, even_field => i, multi_field => i.to_string(), multi_field => (i + 1).to_string()))?;- } else {- index_writer.add_document(doc!(all_field => i, even_field => i))?;- }- } else {- index_writer.add_document(doc!(all_field => i, odd_field => i.to_string()))?;- }- }- index_writer.commit()?;- }- let reader = index.reader()?;- let searcher = reader.searcher();-- assert_eq!(count_existing_fields(&searcher, "all")?, 100);- assert_eq!(count_existing_fields(&searcher, "odd")?, 50);- assert_eq!(count_existing_fields(&searcher, "even")?, 50);- assert_eq!(count_existing_fields(&searcher, "multi")?, 10);- assert_eq!(count_existing_fields(&searcher, "never")?, 0);-- // exercise seek- let query = BooleanQuery::intersection(vec![- Box::new(RangeQuery::new_u64_bounds(- "all".to_string(),- Bound::Included(50),- Bound::Unbounded,- )),- Box::new(ExistsQuery::new_exists_query("even".to_string())),- ]);- assert_eq!(searcher.search(&query, &Count)?, 25);-- let query = BooleanQuery::intersection(vec![- Box::new(RangeQuery::new_u64_bounds(- "all".to_string(),- Bound::Included(0),- Bound::Excluded(50),- )),- Box::new(ExistsQuery::new_exists_query("odd".to_string())),- ]);- assert_eq!(searcher.search(&query, &Count)?, 25);-- Ok(())- }-- #[test]- fn test_exists_query_json() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", TEXT | FAST);- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema);- {- let mut index_writer = index.writer_for_tests()?;- for i in 0u64..100u64 {- if i % 2 == 0 {- index_writer.add_document(doc!(json => json!({"all": i, "even": true})))?;- } else {- index_writer- .add_document(doc!(json => json!({"all": i.to_string(), "odd": true})))?;- }- }- index_writer.commit()?;- }- let reader = index.reader()?;- let searcher = reader.searcher();-- assert_eq!(count_existing_fields(&searcher, "json.all")?, 100);- assert_eq!(count_existing_fields(&searcher, "json.even")?, 50);- assert_eq!(count_existing_fields(&searcher, "json.odd")?, 50);-- // Handling of non-existing fields:- assert_eq!(count_existing_fields(&searcher, "json.absent")?, 0);- assert_eq!(- searcher- .search(- &ExistsQuery::new_exists_query("does_not_exists.absent".to_string()),- &Count- )- .unwrap_err()- .to_string(),- "The field does not exist: 'does_not_exists.absent'"- );-- Ok(())- }-- #[test]- fn test_exists_query_misc_supported_types() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let bool = schema_builder.add_bool_field("bool", FAST);- let bytes = schema_builder.add_bytes_field("bytes", FAST);- let date = schema_builder.add_date_field("date", FAST);- let f64 = schema_builder.add_f64_field("f64", FAST);- let ip_addr = schema_builder.add_ip_addr_field("ip_addr", FAST);- let facet = schema_builder.add_facet_field("facet", FacetOptions::default());- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema);- {- let mut index_writer = index.writer_for_tests()?;- let now = OffsetDateTime::now_utc().unix_timestamp();- for i in 0u8..100u8 {- if i % 2 == 0 {- let date_val = DateTime::from_utc(OffsetDateTime::from_unix_timestamp(- now + i as i64 * 100,- )?);- index_writer.add_document(- doc!(bool => i % 3 == 0, bytes => vec![i, i + 1, i + 2], date => date_val),- )?;- } else {- let ip_addr_v6 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, i.into());- index_writer- .add_document(doc!(f64 => i as f64 * 0.5, ip_addr => ip_addr_v6, facet => Facet::from("/facet/foo"), facet => Facet::from("/facet/bar")))?;- }- }- index_writer.commit()?;- }- let reader = index.reader()?;- let searcher = reader.searcher();-- assert_eq!(count_existing_fields(&searcher, "bool")?, 50);- assert_eq!(count_existing_fields(&searcher, "bytes")?, 50);- assert_eq!(count_existing_fields(&searcher, "date")?, 50);- assert_eq!(count_existing_fields(&searcher, "f64")?, 50);- assert_eq!(count_existing_fields(&searcher, "ip_addr")?, 50);- assert_eq!(count_existing_fields(&searcher, "facet")?, 50);-- Ok(())- }-- #[test]- fn test_exists_query_unsupported_types() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let not_fast = schema_builder.add_text_field("not_fast", TEXT);- let schema = schema_builder.build();-- let index = Index::create_in_ram(schema);- {- let mut index_writer = index.writer_for_tests()?;- index_writer.add_document(doc!(- not_fast => "slow",- ))?;- index_writer.commit()?;- }- let reader = index.reader()?;- let searcher = reader.searcher();-- assert_eq!(- searcher- .search(- &ExistsQuery::new_exists_query("not_fast".to_string()),- &Count- )- .unwrap_err()- .to_string(),- "Schema error: 'Field not_fast is not a fast field.'"- );-- assert_eq!(- searcher- .search(- &ExistsQuery::new_exists_query("does_not_exists".to_string()),- &Count- )- .unwrap_err()- .to_string(),- "The field does not exist: 'does_not_exists'"- );-- Ok(())- }-- fn count_existing_fields(searcher: &Searcher, field: &str) -> crate::Result<usize> {- let query = ExistsQuery::new_exists_query(field.to_string());- searcher.search(&query, &Count)- }-}diff --git a/crates/tantivy/src/query/mod.rs b/crates/tantivy/src/query/mod.rsindex 4083627a..5ab71842 100644--- a/crates/tantivy/src/query/mod.rs+++ b/crates/tantivy/src/query/mod.rs@@ -8,7 +8,6 @@ mod const_score_query;mod disjunction_max_query;mod empty_query;mod exclude;-mod exist_query;mod explanation;mod fuzzy_query;mod intersection;@@ -44,7 +43,6 @@ pub use self::const_score_query::{ConstScoreQuery, ConstScorer};pub use self::disjunction_max_query::DisjunctionMaxQuery;pub use self::empty_query::{EmptyQuery, EmptyScorer, EmptyWeight};pub use self::exclude::Exclude;-pub use self::exist_query::ExistsQuery;pub use self::explanation::Explanation;#[cfg(test)]pub(crate) use self::fuzzy_query::DfaWrapper;@@ -56,7 +54,7 @@ pub use self::phrase_query::PhraseQuery;pub use self::query::{EnableScoring, Query, QueryClone};#[cfg(test)]pub use self::query_parser::{QueryParser, QueryParserError};-pub use self::range_query::{FastFieldRangeWeight, IPFastFieldRangeWeight, RangeQuery};+pub use self::range_query::{FastFieldRangeWeight, RangeQuery};pub use self::reqopt_scorer::RequiredOptionalScorer;pub use self::score_combiner::{DisjunctionMaxCombiner, ScoreCombiner, SumCombiner, SumWithCoordsCombiner,diff --git a/crates/tantivy/src/query/more_like_this/more_like_this.rs b/crates/tantivy/src/query/more_like_this/more_like_this.rsindex 1240be53..9ae5d307 100644--- a/crates/tantivy/src/query/more_like_this/more_like_this.rs+++ b/crates/tantivy/src/query/more_like_this/more_like_this.rs@@ -7,7 +7,7 @@ use crate::query::bm25::idf;use crate::query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery};use crate::schema::document::{Document, Value};use crate::schema::{Field, FieldType, IndexRecordOption, Term};-use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer};+use crate::tokenizer::{PreTokenizedStream, TokenStream};use crate::{DocAddress, Result, Searcher, TantivyDocument, TantivyError};#[derive(Debug, PartialEq)]@@ -176,26 +176,6 @@ impl MoreLikeThis {// extract the raw value, possibly tokenizing & filtering to update the term frequency mapmatch field_entry.field_type() {- FieldType::Facet(_) => {- let facets: Vec<&str> = values- .iter()- .map(|value| {- value.as_facet().ok_or_else(|| {- TantivyError::InvalidArgument("invalid field value".to_string())- })- })- .collect::<Result<Vec<_>>>()?;- for fake_str in facets {- FacetTokenizer::default()- .token_stream(fake_str)- .process(&mut |token| {- if self.is_noise_word(token.text.clone()) {- let term = Term::from_field_text(field, &token.text);- *term_frequencies.entry(term).or_insert(0) += 1;- }- });- }- }FieldType::Str(text_options) => {let mut tokenizer_opt = text_options.get_indexing_options()diff --git a/crates/tantivy/src/query/query_parser/query_parser.rs b/crates/tantivy/src/query/query_parser/query_parser.rsindex ef681603..f37ab2d8 100644--- a/crates/tantivy/src/query/query_parser/query_parser.rs+++ b/crates/tantivy/src/query/query_parser/query_parser.rs@@ -18,8 +18,8 @@ use crate::query::{PhraseQuery, Query, TermQuery, TermSetQuery,};use crate::schema::{- Facet, FacetParseError, Field, FieldType, IndexRecordOption, IntoIpv6Addr, JsonObjectOptions,- Schema, Term, TextFieldIndexing, Type,+ Field, FieldType, IndexRecordOption, IntoIpv6Addr, JsonObjectOptions, Schema, Term,+ TextFieldIndexing, Type,};use crate::time::format_description::well_known::Rfc3339;use crate::time::OffsetDateTime;@@ -96,9 +96,6 @@ pub enum QueryParserError {/// The format for the date field is not RFC 3339 compliant.#[error("The date field has an invalid format")]DateFormatError(#[from] time::error::Parse),- /// The format for the facet field is invalid.- #[error("The facet field is malformed: {0}")]- FacetFormatError(#[from] FacetParseError),/// The format for the ip field is invalid.#[error("The ip field is malformed: {0}")]IpFormatError(#[from] AddrParseError),@@ -493,10 +490,6 @@ impl QueryParser {"Range query are not supported on json field.".to_string(),))}- FieldType::Facet(_) => match Facet::from_text(phrase) {- Ok(facet) => Ok(Term::from_facet(field, &facet)),- Err(e) => Err(QueryParserError::from(e)),- },FieldType::Bytes(_) => {let bytes = BASE64.decode(phrase)@@ -588,13 +581,6 @@ impl QueryParser {&self.tokenizer_manager,json_options,),- FieldType::Facet(_) => match Facet::from_text(phrase) {- Ok(facet) => {- let facet_term = Term::from_facet(field, &facet);- Ok(vec![LogicalLiteral::Term(facet_term)])- }- Err(e) => Err(QueryParserError::from(e)),- },FieldType::Bytes(_) => {let bytes = BASE64.decode(phrase)@@ -1019,886 +1005,3 @@ fn convert_to_query(fuzzy: &FxHashMap<Field, Fuzzy>, logical_ast: LogicalAst) ->None => Box::new(EmptyQuery),}}--#[cfg(test)]-mod test {- use matches::assert_matches;-- use super::super::logical_ast::*;- use super::{QueryParser, QueryParserError};- use crate::query::Query;- use crate::schema::{- FacetOptions, Field, IndexRecordOption, Schema, Term, TextFieldIndexing, TextOptions, FAST,- INDEXED, STORED, STRING, TEXT,- };- use crate::tokenizer::{- LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,- };- use crate::Index;-- fn make_schema() -> Schema {- let mut schema_builder = Schema::builder();- let text_field_indexing = TextFieldIndexing::default()- .set_tokenizer("en_with_stop_words")- .set_index_option(IndexRecordOption::WithFreqsAndPositions);- let text_options = TextOptions::default()- .set_indexing_options(text_field_indexing)- .set_stored();- schema_builder.add_text_field("title", TEXT);- schema_builder.add_text_field("text", TEXT);- schema_builder.add_i64_field("signed", INDEXED);- schema_builder.add_u64_field("unsigned", INDEXED);- schema_builder.add_text_field("notindexed_text", STORED);- schema_builder.add_text_field("notindexed_u64", STORED);- schema_builder.add_text_field("notindexed_i64", STORED);- schema_builder.add_text_field("nottokenized", STRING);- schema_builder.add_text_field("with_stop_words", text_options);- schema_builder.add_date_field("date", INDEXED);- schema_builder.add_f64_field("float", INDEXED);- schema_builder.add_facet_field("facet", FacetOptions::default());- schema_builder.add_bytes_field("bytes", INDEXED);- schema_builder.add_bytes_field("bytes_not_indexed", STORED);- schema_builder.add_json_field("json", TEXT);- schema_builder.add_json_field("json_not_indexed", STORED);- schema_builder.add_bool_field("bool", INDEXED);- schema_builder.add_bool_field("notindexed_bool", STORED);- schema_builder.add_u64_field("u64_ff", FAST);- schema_builder.build()- }-- fn make_query_parser_with_default_fields(default_fields: &[&'static str]) -> QueryParser {- let schema = make_schema();- let default_fields: Vec<Field> = default_fields- .iter()- .flat_map(|field_name| schema.get_field(field_name))- .collect();- let tokenizer_manager = TokenizerManager::default();- tokenizer_manager.register(- "en_with_stop_words",- TextAnalyzer::builder(SimpleTokenizer::default())- .filter(LowerCaser)- .filter(StopWordFilter::remove(vec!["the".to_string()]))- .build(),- );- QueryParser::new(schema, default_fields, tokenizer_manager)- }-- fn make_query_parser() -> QueryParser {- make_query_parser_with_default_fields(&["title", "text"])- }-- fn parse_query_to_logical_ast(- query: &str,- default_conjunction: bool,- ) -> Result<LogicalAst, QueryParserError> {- let mut query_parser = make_query_parser();- if default_conjunction {- query_parser.set_conjunction_by_default();- }- query_parser.parse_query_to_logical_ast(query)- }-- fn test_parse_query_to_logical_ast_helper(- query: &str,- expected: &str,- default_conjunction: bool,- ) {- let query = parse_query_to_logical_ast(query, default_conjunction).unwrap();- let query_str = format!("{query:?}");- assert_eq!(query_str, expected);- }-- #[test]- pub fn test_parse_query_facet() {- let query_parser = make_query_parser();- let query = query_parser.parse_query("facet:/root/branch/leaf").unwrap();- assert_eq!(- format!("{query:?}"),- r#"TermQuery(Term(field=11, type=Facet, Facet(/root/branch/leaf)))"#- );- }-- #[test]- pub fn test_parse_query_with_boost() {- let mut query_parser = make_query_parser();- let schema = make_schema();- let text_field = schema.get_field("text").unwrap();- query_parser.set_field_boost(text_field, 2.0);- let query = query_parser.parse_query("text:hello").unwrap();- assert_eq!(- format!("{query:?}"),- r#"Boost(query=TermQuery(Term(field=1, type=Str, "hello")), boost=2)"#- );- }-- #[test]- pub fn test_parse_query_range_with_boost() {- let query = make_query_parser().parse_query("title:[A TO B]").unwrap();- assert_eq!(- format!("{query:?}"),- "RangeQuery { field: \"title\", value_type: Str, lower_bound: Included([97]), \- upper_bound: Included([98]), limit: None }"- );- }-- #[test]- pub fn test_parse_query_with_default_boost_and_custom_boost() {- let mut query_parser = make_query_parser();- let schema = make_schema();- let text_field = schema.get_field("text").unwrap();- query_parser.set_field_boost(text_field, 2.0);- let query = query_parser.parse_query("text:hello^2").unwrap();- assert_eq!(- format!("{query:?}"),- r#"Boost(query=Boost(query=TermQuery(Term(field=1, type=Str, "hello")), boost=2), boost=2)"#- );- }-- #[test]- pub fn test_parse_nonindexed_field_yields_error() {- let query_parser = make_query_parser();-- let is_not_indexed_err = |query: &str| {- let result: Result<Box<dyn Query>, QueryParserError> = query_parser.parse_query(query);- if let Err(QueryParserError::FieldNotIndexed(field_name)) = result {- Some(field_name)- } else {- None- }- };-- assert_eq!(- is_not_indexed_err("notindexed_text:titi"),- Some(String::from("notindexed_text"))- );- assert_eq!(- is_not_indexed_err("notindexed_u64:23424"),- Some(String::from("notindexed_u64"))- );- assert_eq!(- is_not_indexed_err("notindexed_i64:-234324"),- Some(String::from("notindexed_i64"))- );- assert_eq!(- is_not_indexed_err("notindexed_bool:true"),- Some(String::from("notindexed_bool"))- );- }-- #[test]- pub fn test_parse_query_untokenized() {- test_parse_query_to_logical_ast_helper(- "nottokenized:\"wordone wordtwo\"",- r#"Term(field=7, type=Str, "wordone wordtwo")"#,- false,- );- }-- #[test]- pub fn test_parse_query_empty() {- test_parse_query_to_logical_ast_helper("", "<emptyclause>", false);- test_parse_query_to_logical_ast_helper(" ", "<emptyclause>", false);- let query_parser = make_query_parser();- let query_result = query_parser.parse_query("");- let query = query_result.unwrap();- assert_eq!(format!("{query:?}"), "EmptyQuery");- }-- #[test]- pub fn test_parse_query_ints() {- let query_parser = make_query_parser();- assert!(query_parser.parse_query("signed:2324").is_ok());- assert!(query_parser.parse_query("signed:\"22\"").is_ok());- assert!(query_parser.parse_query("signed:\"-2234\"").is_ok());- assert!(query_parser- .parse_query("signed:\"-9999999999999\"")- .is_ok());- assert!(query_parser.parse_query("signed:\"a\"").is_err());- assert!(query_parser.parse_query("signed:\"2a\"").is_err());- assert!(query_parser- .parse_query("signed:\"18446744073709551615\"")- .is_err());- assert!(query_parser.parse_query("unsigned:\"2\"").is_ok());- assert!(query_parser.parse_query("unsigned:\"-2\"").is_err());- assert!(query_parser- .parse_query("unsigned:\"18446744073709551615\"")- .is_ok());- assert!(query_parser.parse_query("float:\"3.1\"").is_ok());- assert!(query_parser.parse_query("float:\"-2.4\"").is_ok());- assert!(query_parser.parse_query("float:\"2.1.2\"").is_err());- assert!(query_parser.parse_query("float:\"2.1a\"").is_err());- assert!(query_parser- .parse_query("float:\"18446744073709551615.0\"")- .is_ok());- test_parse_query_to_logical_ast_helper(- "unsigned:2324",- "Term(field=3, type=U64, 2324)",- false,- );-- test_parse_query_to_logical_ast_helper(- "signed:-2324",- &format!(- "{:?}",- Term::from_field_i64(Field::from_field_id(2u32), -2324)- ),- false,- );-- test_parse_query_to_logical_ast_helper(- "float:2.5",- &format!(- "{:?}",- Term::from_field_f64(Field::from_field_id(10u32), 2.5)- ),- false,- );- }-- #[test]- fn test_parse_bytes() {- test_parse_query_to_logical_ast_helper(- "bytes:YnVidQ==",- "Term(field=12, type=Bytes, [98, 117, 98, 117])",- false,- );- }-- #[test]- fn test_parse_bool() {- test_parse_query_to_logical_ast_helper(- "bool:true",- &format!(- "{:?}",- Term::from_field_bool(Field::from_field_id(16u32), true),- ),- false,- );- }-- #[test]- fn test_parse_bytes_not_indexed() {- let error = parse_query_to_logical_ast("bytes_not_indexed:aaa", false).unwrap_err();- assert!(matches!(error, QueryParserError::FieldNotIndexed(_)));- }-- #[test]- fn test_json_field() {- test_parse_query_to_logical_ast_helper(- "json.titi:hello",- "Term(field=14, type=Json, path=titi, type=Str, \"hello\")",- false,- );- }-- fn extract_query_term_json_path(query: &str) -> String {- let LogicalAst::Leaf(literal) = parse_query_to_logical_ast(query, false).unwrap() else {- panic!();- };- let LogicalLiteral::Term(term) = *literal else {- panic!();- };- std::str::from_utf8(term.serialized_value_bytes())- .unwrap()- .to_string()- }-- #[test]- fn test_json_field_query_with_espaced_dot() {- assert_eq!(- extract_query_term_json_path(r#"json.k8s.node.name:hello"#),- "k8s\u{1}node\u{1}name\0shello"- );- assert_eq!(- extract_query_term_json_path(r"json.k8s\.node\.name:hello"),- "k8s.node.name\0shello"- );- }-- #[test]- fn test_json_field_possibly_a_number() {- test_parse_query_to_logical_ast_helper(- "json.titi:5",- r#"(Term(field=14, type=Json, path=titi, type=I64, 5) Term(field=14, type=Json, path=titi, type=Str, "5"))"#,- true,- );- test_parse_query_to_logical_ast_helper(- "json.titi:-5",- r#"(Term(field=14, type=Json, path=titi, type=I64, -5) Term(field=14, type=Json, path=titi, type=Str, "5"))"#, //< Yes this is a bit weird after going through the tokenizer we lose the "-".- true,- );- test_parse_query_to_logical_ast_helper(- "json.titi:10000000000000000000",- r#"(Term(field=14, type=Json, path=titi, type=U64, 10000000000000000000) Term(field=14, type=Json, path=titi, type=Str, "10000000000000000000"))"#,- true,- );- test_parse_query_to_logical_ast_helper(- "json.titi:-5.2",- r#"(Term(field=14, type=Json, path=titi, type=F64, -5.2) "[(0, Term(field=14, type=Json, path=titi, type=Str, "5")), (1, Term(field=14, type=Json, path=titi, type=Str, "2"))]")"#,- true,- );- }-- #[test]- fn test_json_field_possibly_a_date() {- test_parse_query_to_logical_ast_helper(- r#"json.date:"2019-10-12T07:20:50.52Z""#,- r#"(Term(field=14, type=Json, path=date, type=Date, 2019-10-12T07:20:50Z) "[(0, Term(field=14, type=Json, path=date, type=Str, "2019")), (1, Term(field=14, type=Json, path=date, type=Str, "10")), (2, Term(field=14, type=Json, path=date, type=Str, "12t07")), (3, Term(field=14, type=Json, path=date, type=Str, "20")), (4, Term(field=14, type=Json, path=date, type=Str, "50")), (5, Term(field=14, type=Json, path=date, type=Str, "52z"))]")"#,- true,- );- }-- #[test]- fn test_json_field_possibly_a_bool() {- test_parse_query_to_logical_ast_helper(- "json.titi:true",- r#"(Term(field=14, type=Json, path=titi, type=Bool, true) Term(field=14, type=Json, path=titi, type=Str, "true"))"#,- true,- );- }-- #[test]- fn test_json_field_not_indexed() {- let error = parse_query_to_logical_ast("json_not_indexed.titi:hello", false).unwrap_err();- assert!(matches!(error, QueryParserError::FieldNotIndexed(_)));- }-- fn test_query_to_logical_ast_with_default_json(- query: &str,- expected: &str,- default_conjunction: bool,- ) {- let mut query_parser = make_query_parser_with_default_fields(&["json"]);- if default_conjunction {- query_parser.set_conjunction_by_default();- }- let ast = query_parser.parse_query_to_logical_ast(query).unwrap();- let ast_str = format!("{ast:?}");- assert_eq!(ast_str, expected);- }-- #[test]- fn test_json_default() {- test_query_to_logical_ast_with_default_json(- "titi:4",- "(Term(field=14, type=Json, path=titi, type=I64, 4) Term(field=14, type=Json, \- path=titi, type=Str, \"4\"))",- false,- );- }-- #[test]- fn test_json_default_with_different_field() {- for conjunction in [false, true] {- test_query_to_logical_ast_with_default_json(- "text:4",- r#"Term(field=1, type=Str, "4")"#,- conjunction,- );- }- }-- #[test]- fn test_json_default_with_same_field() {- for conjunction in [false, true] {- test_query_to_logical_ast_with_default_json(- "json:4",- r#"(Term(field=14, type=Json, path=, type=I64, 4) Term(field=14, type=Json, path=, type=Str, "4"))"#,- conjunction,- );- }- }-- #[test]- fn test_parse_bytes_phrase() {- test_parse_query_to_logical_ast_helper(- "bytes:\"YnVidQ==\"",- "Term(field=12, type=Bytes, [98, 117, 98, 117])",- false,- );- }-- #[test]- fn test_parse_bytes_invalid_base64() {- let base64_err: QueryParserError =- parse_query_to_logical_ast("bytes:aa", false).unwrap_err();- assert!(matches!(base64_err, QueryParserError::ExpectedBase64(_)));- }-- #[test]- fn test_parse_query_to_ast_ab_c() {- test_parse_query_to_logical_ast_helper(- "(+title:a +title:b) title:c",- r#"((+Term(field=0, type=Str, "a") +Term(field=0, type=Str, "b")) Term(field=0, type=Str, "c"))"#,- false,- );- test_parse_query_to_logical_ast_helper(- "(+title:a +title:b) title:c",- r#"(+(+Term(field=0, type=Str, "a") +Term(field=0, type=Str, "b")) +Term(field=0, type=Str, "c"))"#,- true,- );- }-- #[test]- pub fn test_parse_query_to_ast_single_term() {- test_parse_query_to_logical_ast_helper(- "title:toto",- r#"Term(field=0, type=Str, "toto")"#,- false,- );- test_parse_query_to_logical_ast_helper(- "+title:toto",- r#"Term(field=0, type=Str, "toto")"#,- false,- );- test_parse_query_to_logical_ast_helper(- "+title:toto -titi",- r#"(+Term(field=0, type=Str, "toto") -(Term(field=0, type=Str, "titi") Term(field=1, type=Str, "titi")))"#,- false,- );- }-- #[test]- fn test_single_negative_term() {- assert_matches!(- parse_query_to_logical_ast("-title:toto", false),- Err(QueryParserError::AllButQueryForbidden)- );- }-- #[test]- pub fn test_parse_query_to_ast_two_terms() {- test_parse_query_to_logical_ast_helper(- "title:a b",- r#"(Term(field=0, type=Str, "a") (Term(field=0, type=Str, "b") Term(field=1, type=Str, "b")))"#,- false,- );- test_parse_query_to_logical_ast_helper(- r#"title:"a b""#,- r#""[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]""#,- false,- );- }- #[test]- pub fn test_parse_query_all_query() {- let logical_ast = parse_query_to_logical_ast("*", false).unwrap();- assert_eq!(format!("{logical_ast:?}"), "*");- }-- #[test]- pub fn test_parse_query_range_require_a_target_field() {- let query_parser_error = parse_query_to_logical_ast("[A TO B]", false).err().unwrap();- assert_eq!(- query_parser_error.to_string(),- "Unsupported query: Range query need to target a specific field."- );- }-- #[test]- pub fn test_parse_query_to_ast_ranges() {- test_parse_query_to_logical_ast_helper(- "title:[a TO b]",- r#"(Included(Term(field=0, type=Str, "a")) TO Included(Term(field=0, type=Str, "b")))"#,- false,- );- test_parse_query_to_logical_ast_helper(- "title:{titi TO toto}",- r#"(Excluded(Term(field=0, type=Str, "titi")) TO Excluded(Term(field=0, type=Str, "toto")))"#,- false,- );- test_parse_query_to_logical_ast_helper(- "title:{* TO toto}",- r#"(Unbounded TO Excluded(Term(field=0, type=Str, "toto")))"#,- false,- );- test_parse_query_to_logical_ast_helper(- "title:{titi TO *}",- r#"(Excluded(Term(field=0, type=Str, "titi")) TO Unbounded)"#,- false,- );- test_parse_query_to_logical_ast_helper(- "signed:{-5 TO 3}",- r#"(Excluded(Term(field=2, type=I64, -5)) TO Excluded(Term(field=2, type=I64, 3)))"#,- false,- );- test_parse_query_to_logical_ast_helper(- "float:{-1.5 TO 1.5}",- r#"(Excluded(Term(field=10, type=F64, -1.5)) TO Excluded(Term(field=10, type=F64, 1.5)))"#,- false,- );- test_parse_query_to_logical_ast_helper(- "u64_ff:[7 TO 77]",- r#"(Included(Term(field=18, type=U64, 7)) TO Included(Term(field=18, type=U64, 77)))"#,- false,- );- }-- #[test]- pub fn test_query_parser_field_does_not_exist() {- let query_parser = make_query_parser();- assert_eq!(- query_parser- .parse_query("boujou:\"18446744073709551615\"")- .unwrap_err(),- QueryParserError::FieldDoesNotExist("boujou".to_string())- );- }-- #[test]- pub fn test_query_parser_field_not_indexed() {- let query_parser = make_query_parser();- assert_matches!(- query_parser.parse_query("notindexed_text:\"18446744073709551615\""),- Err(QueryParserError::FieldNotIndexed(_))- );- }-- #[test]- pub fn test_unknown_tokenizer() {- let mut schema_builder = Schema::builder();- let text_field_indexing = TextFieldIndexing::default()- .set_tokenizer("nonexistingtokenizer")- .set_index_option(IndexRecordOption::Basic);- let text_options = TextOptions::default().set_indexing_options(text_field_indexing);- let title = schema_builder.add_text_field("title", text_options);- let schema = schema_builder.build();- let default_fields = vec![title];- let tokenizer_manager = TokenizerManager::default();- let query_parser = QueryParser::new(schema, default_fields, tokenizer_manager);-- assert_matches!(- query_parser.parse_query("title:\"happy tax payer\""),- Err(QueryParserError::UnknownTokenizer { .. })- );- }-- #[test]- pub fn test_query_parser_no_positions() {- let mut schema_builder = Schema::builder();- let text_field_indexing = TextFieldIndexing::default()- .set_tokenizer("customtokenizer")- .set_index_option(IndexRecordOption::Basic);- let text_options = TextOptions::default().set_indexing_options(text_field_indexing);- let title = schema_builder.add_text_field("title", text_options);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- index- .tokenizers()- .register("customtokenizer", SimpleTokenizer::default());- let query_parser = QueryParser::for_index(&index, vec![title]);- assert_eq!(- query_parser.parse_query("title:\"happy tax\"").unwrap_err(),- QueryParserError::FieldDoesNotHavePositionsIndexed("title".to_string())- );- }-- #[test]- pub fn test_query_parser_expected_int() {- let query_parser = make_query_parser();- assert_matches!(- query_parser.parse_query("unsigned:18a"),- Err(QueryParserError::ExpectedInt(_))- );- assert!(query_parser.parse_query("unsigned:\"18\"").is_ok());- assert_matches!(- query_parser.parse_query("signed:18b"),- Err(QueryParserError::ExpectedInt(_))- );- assert!(query_parser.parse_query("float:\"1.8\"").is_ok());- assert_matches!(- query_parser.parse_query("float:1.8a"),- Err(QueryParserError::ExpectedFloat(_))- );- }-- #[test]- pub fn test_query_parser_expected_bool() {- let query_parser = make_query_parser();- assert_matches!(- query_parser.parse_query("bool:brie"),- Err(QueryParserError::ExpectedBool(_))- );- assert!(query_parser.parse_query("bool:\"true\"").is_ok());- assert!(query_parser.parse_query("bool:\"false\"").is_ok());- }-- #[test]- pub fn test_query_parser_expected_date() {- let query_parser = make_query_parser();- assert_matches!(- query_parser.parse_query("date:18a"),- Err(QueryParserError::DateFormatError(_))- );- test_parse_query_to_logical_ast_helper(- r#"date:"2010-11-21T09:55:06.000000000+02:00""#,- r#"Term(field=9, type=Date, 2010-11-21T07:55:06Z)"#,- true,- );- test_parse_query_to_logical_ast_helper(- r#"date:"1985-04-12T23:20:50.52Z""#,- r#"Term(field=9, type=Date, 1985-04-12T23:20:50Z)"#,- true,- );- }-- #[test]- pub fn test_query_parser_expected_facet() {- let query_parser = make_query_parser();- match query_parser.parse_query("facet:INVALID") {- Ok(_) => panic!("should never succeed"),- Err(e) => assert_eq!(- "The facet field is malformed: Failed to parse the facet string: 'INVALID'",- format!("{e}")- ),- }- assert!(query_parser.parse_query("facet:\"/foo/bar\"").is_ok());- }-- #[test]- pub fn test_query_parser_not_empty_but_no_tokens() {- let query_parser = make_query_parser();- assert!(query_parser.parse_query(" !, ").is_ok());- assert!(query_parser.parse_query("with_stop_words:the").is_ok());- }-- #[test]- pub fn test_parse_query_single_negative_term_through_error() {- assert_matches!(- parse_query_to_logical_ast("-title:toto", true),- Err(QueryParserError::AllButQueryForbidden)- );- assert_matches!(- parse_query_to_logical_ast("-title:toto", false),- Err(QueryParserError::AllButQueryForbidden)- );- }-- #[test]- pub fn test_parse_query_to_ast_conjunction() {- test_parse_query_to_logical_ast_helper(- "title:toto",- r#"Term(field=0, type=Str, "toto")"#,- true,- );- test_parse_query_to_logical_ast_helper(- "+title:toto",- r#"Term(field=0, type=Str, "toto")"#,- true,- );- test_parse_query_to_logical_ast_helper(- "+title:toto -titi",- r#"(+Term(field=0, type=Str, "toto") -(Term(field=0, type=Str, "titi") Term(field=1, type=Str, "titi")))"#,- true,- );- test_parse_query_to_logical_ast_helper(- "title:a b",- r#"(+Term(field=0, type=Str, "a") +(Term(field=0, type=Str, "b") Term(field=1, type=Str, "b")))"#,- true,- );- test_parse_query_to_logical_ast_helper(- "title:\"a b\"",- r#""[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]""#,- true,- );- }-- #[test]- pub fn test_query_parser_hyphen() {- test_parse_query_to_logical_ast_helper(- "title:www-form-encoded",- r#""[(0, Term(field=0, type=Str, "www")), (1, Term(field=0, type=Str, "form")), (2, Term(field=0, type=Str, "encoded"))]""#,- false,- );- }-- #[test]- fn test_and_default_regardless_of_default_conjunctive() {- for &default_conjunction in &[false, true] {- test_parse_query_to_logical_ast_helper(- "title:a AND title:b",- r#"(+Term(field=0, type=Str, "a") +Term(field=0, type=Str, "b"))"#,- default_conjunction,- );- }- }-- #[test]- fn test_or_default_conjunctive() {- for &default_conjunction in &[false, true] {- test_parse_query_to_logical_ast_helper(- "title:a OR title:b",- r#"(Term(field=0, type=Str, "a") Term(field=0, type=Str, "b"))"#,- default_conjunction,- );- }- }-- #[test]- fn test_escaped_field() {- let mut schema_builder = Schema::builder();- schema_builder.add_text_field(r"a\.b", STRING);- let schema = schema_builder.build();- let query_parser = QueryParser::new(schema, Vec::new(), TokenizerManager::default());- let query = query_parser.parse_query(r"a\.b:hello").unwrap();- assert_eq!(- format!("{query:?}"),- "TermQuery(Term(field=0, type=Str, \"hello\"))"- );- }-- #[test]- fn test_split_full_path() {- let mut schema_builder = Schema::builder();- schema_builder.add_text_field("second", STRING);- schema_builder.add_text_field("first", STRING);- schema_builder.add_text_field("first.toto", STRING);- schema_builder.add_text_field("first.toto.titi", STRING);- schema_builder.add_text_field("third.a.b.c", STRING);- let schema = schema_builder.build();- let query_parser =- QueryParser::new(schema.clone(), Vec::new(), TokenizerManager::default());- assert_eq!(- query_parser.split_full_path("first.toto"),- Some((schema.get_field("first.toto").unwrap(), ""))- );- assert_eq!(- query_parser.split_full_path("first.toto.bubu"),- Some((schema.get_field("first.toto").unwrap(), "bubu"))- );- assert_eq!(- query_parser.split_full_path("first.toto.titi"),- Some((schema.get_field("first.toto.titi").unwrap(), ""))- );- assert_eq!(- query_parser.split_full_path("first.titi"),- Some((schema.get_field("first").unwrap(), "titi"))- );- assert_eq!(query_parser.split_full_path("third"), None);- assert_eq!(query_parser.split_full_path("hello.toto"), None);- assert_eq!(query_parser.split_full_path(""), None);- assert_eq!(query_parser.split_full_path("firsty"), None);- }-- #[test]- pub fn test_phrase_slop() {- test_parse_query_to_logical_ast_helper(- "\"a b\"~0",- r#"("[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]" "[(0, Term(field=1, type=Str, "a")), (1, Term(field=1, type=Str, "b"))]")"#,- false,- );- test_parse_query_to_logical_ast_helper(- "\"a b\"~2",- r#"("[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]"~2 "[(0, Term(field=1, type=Str, "a")), (1, Term(field=1, type=Str, "b"))]"~2)"#,- false,- );- test_parse_query_to_logical_ast_helper(- "title:\"a b~4\"~2",- r#""[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b")), (2, Term(field=0, type=Str, "4"))]"~2"#,- false,- );- }-- #[test]- pub fn test_phrase_prefix() {- test_parse_query_to_logical_ast_helper(- "\"big bad wo\"*",- r#"("[(0, Term(field=0, type=Str, "big")), (1, Term(field=0, type=Str, "bad")), (2, Term(field=0, type=Str, "wo"))]"* "[(0, Term(field=1, type=Str, "big")), (1, Term(field=1, type=Str, "bad")), (2, Term(field=1, type=Str, "wo"))]"*)"#,- false,- );-- let query_parser = make_query_parser();- let query = query_parser.parse_query("\"big bad wo\"*").unwrap();- assert_eq!(- format!("{query:?}"),- "BooleanQuery { subqueries: [(Should, PhrasePrefixQuery { field: Field(0), \- phrase_terms: [(0, Term(field=0, type=Str, \"big\")), (1, Term(field=0, type=Str, \- \"bad\"))], prefix: (2, Term(field=0, type=Str, \"wo\")), max_expansions: 50 }), \- (Should, PhrasePrefixQuery { field: Field(1), phrase_terms: [(0, Term(field=1, \- type=Str, \"big\")), (1, Term(field=1, type=Str, \"bad\"))], prefix: (2, \- Term(field=1, type=Str, \"wo\")), max_expansions: 50 })] }"- );- }-- #[test]- pub fn test_phrase_prefix_too_short() {- let err = parse_query_to_logical_ast("\"wo\"*", true).unwrap_err();- assert_eq!(- err,- QueryParserError::PhrasePrefixRequiresAtLeastTwoTerms {- phrase: "wo".to_owned(),- tokenizer: "default".to_owned()- }- );-- let err = parse_query_to_logical_ast("\"\"*", true).unwrap_err();- assert_eq!(- err,- QueryParserError::PhrasePrefixRequiresAtLeastTwoTerms {- phrase: "".to_owned(),- tokenizer: "default".to_owned()- }- );- }-- #[test]- pub fn test_term_set_query() {- test_parse_query_to_logical_ast_helper(- "title: IN [a b cd]",- r#"IN [Term(field=0, type=Str, "a"), Term(field=0, type=Str, "b"), Term(field=0, type=Str, "cd")]"#,- false,- );- test_parse_query_to_logical_ast_helper(- "bytes: IN [AA== ABA= ABCD]",- r#"IN [Term(field=12, type=Bytes, [0]), Term(field=12, type=Bytes, [0, 16]), Term(field=12, type=Bytes, [0, 16, 131])]"#,- false,- );- test_parse_query_to_logical_ast_helper(- "signed: IN [1 2 -3]",- r#"IN [Term(field=2, type=I64, 1), Term(field=2, type=I64, 2), Term(field=2, type=I64, -3)]"#,- false,- );-- test_parse_query_to_logical_ast_helper(- "float: IN [1.1 2.2 -3.3]",- r#"IN [Term(field=10, type=F64, 1.1), Term(field=10, type=F64, 2.2), Term(field=10, type=F64, -3.3)]"#,- false,- );- }-- #[test]- pub fn test_set_field_fuzzy() {- {- let mut query_parser = make_query_parser();- query_parser.set_field_fuzzy(- query_parser.schema.get_field("title").unwrap(),- false,- 1,- true,- );- let query = query_parser.parse_query("abc").unwrap();- assert_eq!(- format!("{query:?}"),- "BooleanQuery { subqueries: [(Should, FuzzyTermQuery { term: Term(field=0, \- type=Str, \"abc\"), distance: 1, transposition_cost_one: true, prefix: false }), \- (Should, TermQuery(Term(field=1, type=Str, \"abc\")))] }"- );- }-- {- let mut query_parser = make_query_parser();- query_parser.set_field_fuzzy(- query_parser.schema.get_field("text").unwrap(),- true,- 2,- false,- );- let query = query_parser.parse_query("abc").unwrap();- assert_eq!(- format!("{query:?}"),- "BooleanQuery { subqueries: [(Should, TermQuery(Term(field=0, type=Str, \- \"abc\"))), (Should, FuzzyTermQuery { term: Term(field=1, type=Str, \"abc\"), \- distance: 2, transposition_cost_one: false, prefix: true })] }"- );- }- }-}diff --git a/crates/tantivy/src/query/range_query/mod.rs b/crates/tantivy/src/query/range_query/mod.rsindex 75b47626..6214c1e5 100644--- a/crates/tantivy/src/query/range_query/mod.rs+++ b/crates/tantivy/src/query/range_query/mod.rs@@ -4,11 +4,9 @@ use crate::schema::Type;mod fast_field_range_query;mod range_query;-mod range_query_ip_fastfield;mod range_query_u64_fastfield;pub use self::range_query::RangeQuery;-pub use self::range_query_ip_fastfield::IPFastFieldRangeWeight;pub use self::range_query_u64_fastfield::FastFieldRangeWeight;// TODO is this correct?@@ -16,7 +14,7 @@ pub(crate) fn is_type_valid_for_fastfield_range_query(typ: Type) -> bool {match typ {Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,Type::IpAddr => true,- Type::Str | Type::Facet | Type::Bytes | Type::Json => false,+ Type::Str | Type::Bytes | Type::Json => false,}}@@ -28,15 +26,3 @@ fn map_bound<TFrom, TTo>(bound: &Bound<TFrom>, transform: impl Fn(&TFrom) -> TToUnbounded => Unbounded,}}--fn map_bound_res<TFrom, TTo, Err>(- bound: &Bound<TFrom>,- transform: impl Fn(&TFrom) -> Result<TTo, Err>,-) -> Result<Bound<TTo>, Err> {- use self::Bound::*;- Ok(match bound {- Excluded(ref from_val) => Excluded(transform(from_val)?),- Included(ref from_val) => Included(transform(from_val)?),- Unbounded => Unbounded,- })-}diff --git a/crates/tantivy/src/query/range_query/range_query.rs b/crates/tantivy/src/query/range_query/range_query.rsindex 2b46a30e..351c1888 100644--- a/crates/tantivy/src/query/range_query/range_query.rs+++ b/crates/tantivy/src/query/range_query/range_query.rs@@ -2,7 +2,6 @@ use std::io;use std::net::Ipv6Addr;use std::ops::{Bound, Range};-use crate::columnar::MonotonicallyMappableToU128;use crate::common::{BinarySerializable, BitSet};use super::map_bound;@@ -10,8 +9,7 @@ use super::range_query_u64_fastfield::FastFieldRangeWeight;use crate::error::TantivyError;use crate::index::SegmentReader;use crate::query::explanation::does_not_match;-use crate::query::range_query::range_query_ip_fastfield::IPFastFieldRangeWeight;-use crate::query::range_query::{is_type_valid_for_fastfield_range_query, map_bound_res};+use crate::query::range_query::is_type_valid_for_fastfield_range_query;use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};use crate::schema::{Field, IndexRecordOption, Term, Type};use crate::termdict::{TermDictionary, TermStreamer};@@ -324,7 +322,7 @@ pub(crate) fn maps_to_u64_fastfield(typ: Type) -> bool {match typ {Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,Type::IpAddr => false,- Type::Str | Type::Facet | Type::Bytes | Type::Json => false,+ Type::Str | Type::Bytes | Type::Json => false,}}@@ -345,39 +343,20 @@ impl Query for RangeQuery {}if field_type.is_fast() && is_type_valid_for_fastfield_range_query(self.value_type) {- if field_type.is_ip_addr() {- let parse_ip_from_bytes = |data: &Vec<u8>| {- let ip_u128_bytes: [u8; 16] = data.as_slice().try_into().map_err(|_| {- crate::TantivyError::InvalidArgument(- "Expected 8 bytes for ip address".to_string(),- )- })?;- let ip_u128 = u128::from_be_bytes(ip_u128_bytes);- crate::Result::<Ipv6Addr>::Ok(Ipv6Addr::from_u128(ip_u128))- };- let lower_bound = map_bound_res(&self.lower_bound, parse_ip_from_bytes)?;- let upper_bound = map_bound_res(&self.upper_bound, parse_ip_from_bytes)?;- Ok(Box::new(IPFastFieldRangeWeight::new(- self.field.to_string(),- lower_bound,- upper_bound,- )))- } else {- // We run the range query on u64 value space for performance reasons and simpicity- // assert the type maps to u64- assert!(maps_to_u64_fastfield(self.value_type));- let parse_from_bytes = |data: &Vec<u8>| {- u64::from_be(BinarySerializable::deserialize(&mut &data[..]).unwrap())- };-- let lower_bound = map_bound(&self.lower_bound, parse_from_bytes);- let upper_bound = map_bound(&self.upper_bound, parse_from_bytes);- Ok(Box::new(FastFieldRangeWeight::new_u64_lenient(- self.field.to_string(),- lower_bound,- upper_bound,- )))- }+ // We run the range query on u64 value space for performance reasons and simpicity+ // assert the type maps to u64+ assert!(maps_to_u64_fastfield(self.value_type));+ let parse_from_bytes = |data: &Vec<u8>| {+ u64::from_be(BinarySerializable::deserialize(&mut &data[..]).unwrap())+ };++ let lower_bound = map_bound(&self.lower_bound, parse_from_bytes);+ let upper_bound = map_bound(&self.upper_bound, parse_from_bytes);+ Ok(Box::new(FastFieldRangeWeight::new_u64_lenient(+ self.field.to_string(),+ lower_bound,+ upper_bound,+ )))} else {Ok(Box::new(RangeWeight {field: self.field.to_string(),@@ -695,165 +674,4 @@ mod tests {assert_eq!(top_docs.len(), 1);Ok(())}-- #[test]- fn search_ip_range_test_posting_list() {- search_ip_range_test_opt(false);- }-- #[test]- fn search_ip_range_test() {- search_ip_range_test_opt(true);- }-- fn search_ip_range_test_opt(with_fast_field: bool) {- let mut schema_builder = Schema::builder();- let ip_field = if with_fast_field {- schema_builder.add_ip_addr_field("ip", INDEXED | STORED | FAST)- } else {- schema_builder.add_ip_addr_field("ip", INDEXED | STORED)- };- let text_field = schema_builder.add_text_field("text", TEXT | STORED);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let ip_addr_1 = IpAddr::from_str("127.0.0.10").unwrap().into_ipv6_addr();- let ip_addr_2 = IpAddr::from_str("127.0.0.20").unwrap().into_ipv6_addr();-- {- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- for _ in 0..1_000 {- index_writer- .add_document(doc!(- ip_field => ip_addr_1,- text_field => "BLUBBER"- ))- .unwrap();- }- for _ in 0..1_000 {- index_writer- .add_document(doc!(- ip_field => ip_addr_2,- text_field => "BLOBBER"- ))- .unwrap();- }- index_writer.commit().unwrap();- }- let reader = index.reader().unwrap();- let searcher = reader.searcher();- assert_eq!(searcher.segment_readers().len(), 1);-- let get_num_hits = |query| {- let (_top_docs, count) = searcher- .search(&query, &(TopDocs::with_limit(10), Count))- .unwrap();- count- };- let query_from_text = |text: &str| {- QueryParser::for_index(&index, vec![])- .parse_query(text)- .unwrap()- };-- // Inclusive range- assert_eq!(- get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20]")),- 2000- );-- assert_eq!(- get_num_hits(query_from_text("ip:[127.0.0.10 TO 127.0.0.20]")),- 2000- );-- assert_eq!(- get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.20]")),- 1000- );-- assert_eq!(- get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.19]")),- 0- );-- assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.11 TO *]")), 1000);- assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.21 TO *]")), 0);- assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.9]")), 0);- assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.10]")), 1000);-- // Exclusive range- assert_eq!(- get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20}")),- 1000- );-- assert_eq!(- get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.21}")),- 2000- );-- assert_eq!(- get_num_hits(query_from_text("ip:{127.0.0.10 TO 127.0.0.20}")),- 0- );-- assert_eq!(- get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.20}")),- 0- );-- assert_eq!(- get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.19}")),- 0- );-- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.11 TO *}")), 1000);- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.10 TO *}")), 1000);- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.21 TO *}")), 0);- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.20 TO *}")), 0);- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.19 TO *}")), 1000);- assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.9}")), 0);- assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.10}")), 0);- assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.11}")), 1000);-- // Inclusive/Exclusive range- assert_eq!(- get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20}")),- 1000- );-- assert_eq!(- get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20]")),- 2000- );-- // Intersection- assert_eq!(- get_num_hits(query_from_text(- "text:BLUBBER AND ip:[127.0.0.10 TO 127.0.0.10]"- )),- 1000- );-- assert_eq!(- get_num_hits(query_from_text(- "text:BLOBBER AND ip:[127.0.0.10 TO 127.0.0.10]"- )),- 0- );-- assert_eq!(- get_num_hits(query_from_text(- "text:BLOBBER AND ip:[127.0.0.20 TO 127.0.0.20]"- )),- 1000- );-- assert_eq!(- get_num_hits(query_from_text(- "text:BLUBBER AND ip:[127.0.0.20 TO 127.0.0.20]"- )),- 0- );- }}diff --git a/crates/tantivy/src/query/range_query/range_query_ip_fastfield.rs b/crates/tantivy/src/query/range_query/range_query_ip_fastfield.rsdeleted file mode 100644index 60b8bcfa..00000000--- a/crates/tantivy/src/query/range_query/range_query_ip_fastfield.rs+++ /dev/null@@ -1,273 +0,0 @@-//! IP Fastfields support efficient scanning for range queries.-//! We use this variant only if the fastfield exists, otherwise the default in `range_query` is-//! used, which uses the term dictionary + postings.--use std::net::Ipv6Addr;-use std::ops::{Bound, RangeInclusive};--use crate::columnar::{Column, MonotonicallyMappableToU128};--use crate::query::range_query::fast_field_range_query::RangeDocSet;-use crate::query::{ConstScorer, EmptyScorer, Explanation, Scorer, Weight};-use crate::{DocId, DocSet, Score, SegmentReader, TantivyError};--/// `IPFastFieldRangeWeight` uses the ip address fast field to execute range queries.-pub struct IPFastFieldRangeWeight {- field: String,- lower_bound: Bound<Ipv6Addr>,- upper_bound: Bound<Ipv6Addr>,-}--impl IPFastFieldRangeWeight {- /// Creates a new IPFastFieldRangeWeight.- pub fn new(field: String, lower_bound: Bound<Ipv6Addr>, upper_bound: Bound<Ipv6Addr>) -> Self {- Self {- field,- lower_bound,- upper_bound,- }- }-}--impl Weight for IPFastFieldRangeWeight {- fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {- let Some(ip_addr_column): Option<Column<Ipv6Addr>> =- reader.fast_fields().column_opt(&self.field)?- else {- return Ok(Box::new(EmptyScorer));- };- let value_range = bound_to_value_range(- &self.lower_bound,- &self.upper_bound,- ip_addr_column.min_value(),- ip_addr_column.max_value(),- );- let docset = RangeDocSet::new(value_range, ip_addr_column);- Ok(Box::new(ConstScorer::new(docset, boost)))- }-- fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {- let mut scorer = self.scorer(reader, 1.0)?;- if scorer.seek(doc) != doc {- return Err(TantivyError::InvalidArgument(format!(- "Document #({doc}) does not match"- )));- }- let explanation = Explanation::new("Const", scorer.score());- Ok(explanation)- }-}--fn bound_to_value_range(- lower_bound: &Bound<Ipv6Addr>,- upper_bound: &Bound<Ipv6Addr>,- min_value: Ipv6Addr,- max_value: Ipv6Addr,-) -> RangeInclusive<Ipv6Addr> {- let start_value = match lower_bound {- Bound::Included(ip_addr) => *ip_addr,- Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() + 1),- Bound::Unbounded => min_value,- };-- let end_value = match upper_bound {- Bound::Included(ip_addr) => *ip_addr,- Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() - 1),- Bound::Unbounded => max_value,- };- start_value..=end_value-}--#[cfg(test)]-pub mod tests {- use proptest::prelude::ProptestConfig;- use proptest::strategy::Strategy;- use proptest::{prop_oneof, proptest};-- use super::*;- use crate::collector::Count;- use crate::query::QueryParser;- use crate::schema::{Schema, FAST, INDEXED, STORED, STRING};- use crate::{Index, IndexWriter};-- #[derive(Clone, Debug)]- pub struct Doc {- pub id: String,- pub ip: Ipv6Addr,- }-- fn operation_strategy() -> impl Strategy<Value = Doc> {- prop_oneof![- (0u64..10_000u64).prop_map(doc_from_id_1),- (1u64..10_000u64).prop_map(doc_from_id_2),- ]- }-- pub fn doc_from_id_1(id: u64) -> Doc {- let id = id * 1000;- Doc {- // ip != id- id: id.to_string(),- ip: Ipv6Addr::from_u128(id as u128),- }- }- fn doc_from_id_2(id: u64) -> Doc {- let id = id * 1000;- Doc {- // ip != id- id: (id - 1).to_string(),- ip: Ipv6Addr::from_u128(id as u128),- }- }-- proptest! {- #![proptest_config(ProptestConfig::with_cases(10))]- #[test]- fn test_ip_range_for_docs_prop(ops in proptest::collection::vec(operation_strategy(), 1..1000)) {- assert!(test_ip_range_for_docs(&ops).is_ok());- }- }-- #[test]- fn test_ip_range_regression1() {- let ops = &[doc_from_id_1(0)];- assert!(test_ip_range_for_docs(ops).is_ok());- }-- #[test]- fn test_ip_range_regression2() {- let ops = &[- doc_from_id_1(52),- doc_from_id_1(63),- doc_from_id_1(12),- doc_from_id_2(91),- doc_from_id_2(33),- ];- assert!(test_ip_range_for_docs(ops).is_ok());- }-- #[test]- fn test_ip_range_regression3() {- let ops = &[doc_from_id_1(1), doc_from_id_1(2), doc_from_id_1(3)];- assert!(test_ip_range_for_docs(ops).is_ok());- }-- #[test]- fn test_ip_range_regression3_simple() {- let mut schema_builder = Schema::builder();- let ips_field = schema_builder.add_ip_addr_field("ips", FAST | INDEXED);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut writer: IndexWriter = index.writer_for_tests().unwrap();- let ip_addrs: Vec<Ipv6Addr> = [1000, 2000, 3000]- .into_iter()- .map(Ipv6Addr::from_u128)- .collect();- for &ip_addr in &ip_addrs {- writer- .add_document(doc!(ips_field=>ip_addr, ips_field=>ip_addr))- .unwrap();- }- writer.commit().unwrap();- let searcher = index.reader().unwrap().searcher();- let range_weight = IPFastFieldRangeWeight {- field: "ips".to_string(),- lower_bound: Bound::Included(ip_addrs[1]),- upper_bound: Bound::Included(ip_addrs[2]),- };- let count = range_weight.count(searcher.segment_reader(0)).unwrap();- assert_eq!(count, 2);- }-- pub fn create_index_from_docs(docs: &[Doc]) -> Index {- let mut schema_builder = Schema::builder();- let ip_field = schema_builder.add_ip_addr_field("ip", STORED | FAST);- let ips_field = schema_builder.add_ip_addr_field("ips", FAST | INDEXED);- let text_field = schema_builder.add_text_field("id", STRING | STORED);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- {- let mut index_writer = index.writer_with_num_threads(2, 60_000_000).unwrap();- for doc in docs.iter() {- index_writer- .add_document(doc!(- ips_field => doc.ip,- ips_field => doc.ip,- ip_field => doc.ip,- text_field => doc.id.to_string(),- ))- .unwrap();- }-- index_writer.commit().unwrap();- }- index- }-- fn test_ip_range_for_docs(docs: &[Doc]) -> crate::Result<()> {- let index = create_index_from_docs(docs);- let reader = index.reader().unwrap();- let searcher = reader.searcher();-- let get_num_hits = |query| searcher.search(&query, &Count).unwrap();- let query_from_text = |text: &str| {- QueryParser::for_index(&index, vec![])- .parse_query(text)- .unwrap()- };-- let gen_query_inclusive = |field: &str, ip_range: &RangeInclusive<Ipv6Addr>| {- format!("{field}:[{} TO {}]", ip_range.start(), ip_range.end())- };-- let test_sample = |sample_docs: &[Doc]| {- let mut ips: Vec<Ipv6Addr> = sample_docs.iter().map(|doc| doc.ip).collect();- ips.sort();- let ip_range = ips[0]..=ips[1];- let expected_num_hits = docs- .iter()- .filter(|doc| (ips[0]..=ips[1]).contains(&doc.ip))- .count();-- let query = gen_query_inclusive("ip", &ip_range);- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);-- let query = gen_query_inclusive("ips", &ip_range);- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);-- // Intersection search- let id_filter = sample_docs[0].id.to_string();- let expected_num_hits = docs- .iter()- .filter(|doc| ip_range.contains(&doc.ip) && doc.id == id_filter)- .count();- let query = format!(- "{} AND id:{}",- gen_query_inclusive("ip", &ip_range),- &id_filter- );- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);-- // Intersection search on multivalue ip field- let id_filter = sample_docs[0].id.to_string();- let query = format!(- "{} AND id:{}",- gen_query_inclusive("ips", &ip_range),- &id_filter- );- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);- };-- test_sample(&[docs[0].clone(), docs[0].clone()]);- if docs.len() > 1 {- test_sample(&[docs[0].clone(), docs[1].clone()]);- test_sample(&[docs[1].clone(), docs[1].clone()]);- }- if docs.len() > 2 {- test_sample(&[docs[1].clone(), docs[2].clone()]);- }-- Ok(())- }-}diff --git a/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs b/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rsindex 2e9981bb..13c8ca4a 100644--- a/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs+++ b/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs@@ -140,7 +140,7 @@ pub mod tests {use crate::collector::Count;use crate::query::range_query::range_query_u64_fastfield::FastFieldRangeWeight;use crate::query::{QueryParser, Weight};- use crate::schema::{NumericOptions, Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING};+ use crate::schema::{Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING};use crate::{Index, IndexWriter, TERMINATED};#[derive(Clone, Debug)]@@ -179,12 +179,6 @@ pub mod tests {}}- #[test]- fn range_regression1_test() {- let ops = vec![doc_from_id_1(0)];- assert!(test_id_range_for_docs(ops).is_ok());- }-#[test]fn test_range_regression2() {let ops = vec![@@ -197,12 +191,6 @@ pub mod tests {assert!(test_id_range_for_docs(ops).is_ok());}- #[test]- fn test_range_regression3() {- let ops = vec![doc_from_id_1(9), doc_from_id_1(0), doc_from_id_1(13)];- assert!(test_id_range_for_docs(ops).is_ok());- }-#[test]fn test_range_regression_simplified() {let mut schema_builder = SchemaBuilder::new();@@ -225,34 +213,16 @@ pub mod tests {}#[test]- fn range_regression3_test() {+ fn test_range_regression3() {let ops = vec![doc_from_id_1(1), doc_from_id_1(2), doc_from_id_1(3)];assert!(test_id_range_for_docs(ops).is_ok());}- #[test]- fn range_regression4_test() {- let ops = vec![doc_from_id_2(100)];- assert!(test_id_range_for_docs(ops).is_ok());- }-pub fn create_index_from_docs(docs: &[Doc]) -> Index {let mut schema_builder = Schema::builder();let id_u64_field = schema_builder.add_u64_field("id", INDEXED | STORED | FAST);- let ids_u64_field =- schema_builder.add_u64_field("ids", NumericOptions::default().set_fast().set_indexed());-let id_f64_field = schema_builder.add_f64_field("id_f64", INDEXED | STORED | FAST);- let ids_f64_field = schema_builder.add_f64_field(- "ids_f64",- NumericOptions::default().set_fast().set_indexed(),- );-let id_i64_field = schema_builder.add_i64_field("id_i64", INDEXED | STORED | FAST);- let ids_i64_field = schema_builder.add_i64_field(- "ids_i64",- NumericOptions::default().set_fast().set_indexed(),- );let text_field = schema_builder.add_text_field("id_name", STRING | STORED);let schema = schema_builder.build();@@ -263,12 +233,6 @@ pub mod tests {for doc in docs.iter() {index_writer.add_document(doc!(- ids_i64_field => doc.id as i64,- ids_i64_field => doc.id as i64,- ids_f64_field => doc.id as f64,- ids_f64_field => doc.id as f64,- ids_u64_field => doc.id,- ids_u64_field => doc.id,id_u64_field => doc.id,id_f64_field => doc.id as f64,id_i64_field => doc.id as i64,@@ -314,9 +278,6 @@ pub mod tests {let query = gen_query_inclusive("id", ids[0]..=ids[1]);assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);- let query = gen_query_inclusive("ids", ids[0]..=ids[1]);- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);-// Exclusive rangelet expected_num_hits = docs.iter()@@ -328,9 +289,6 @@ pub mod tests {let query = gen_query_exclusive("id", ids[0]..=ids[1]);assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);- let query = gen_query_exclusive("ids", ids[0]..=ids[1]);- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);-// Intersection searchlet id_filter = sample_docs[0].id_name.to_string();let expected_num_hits = docs@@ -355,27 +313,6 @@ pub mod tests {&id_filter);assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);-- // Intersection search on multivalue id field- let id_filter = sample_docs[0].id_name.to_string();- let query = format!(- "{} AND id_name:{}",- gen_query_inclusive("ids", ids[0]..=ids[1]),- &id_filter- );- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);- let query = format!(- "{} AND id_name:{}",- gen_query_inclusive("ids_f64", ids[0]..=ids[1]),- &id_filter- );- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);- let query = format!(- "{} AND id_name:{}",- gen_query_inclusive("ids_i64", ids[0]..=ids[1]),- &id_filter- );- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);};test_sample(vec![docs[0].clone(), docs[0].clone()]);diff --git a/crates/tantivy/src/schema/document/de.rs b/crates/tantivy/src/schema/document/de.rsindex a7d5ef8c..22c1c8b1 100644--- a/crates/tantivy/src/schema/document/de.rs+++ b/crates/tantivy/src/schema/document/de.rs@@ -21,7 +21,7 @@ use crate::common::{u64_to_f64, BinarySerializable, DateTime, VInt};use super::se::BinaryObjectSerializer;use super::{OwnedValue, Value};use crate::schema::document::type_codes;-use crate::schema::{Facet, Field};+use crate::schema::Field;use crate::tokenizer::PreTokenizedString;#[derive(Debug, thiserror::Error, Clone)]@@ -112,9 +112,6 @@ pub trait ValueDeserializer<'de> {/// Attempts to deserialize a datetime value from the deserializer.fn deserialize_datetime(self) -> Result<DateTime, DeserializeError>;- /// Attempts to deserialize a facet value from the deserializer.- fn deserialize_facet(self) -> Result<Facet, DeserializeError>;-/// Attempts to deserialize a bytes value from the deserializer.fn deserialize_bytes(self) -> Result<Vec<u8>, DeserializeError>;@@ -148,8 +145,6 @@ pub enum ValueType {F64,/// A datetime value.DateTime,- /// A facet value.- Facet,/// A bytes value.Bytes,/// A IP address value.@@ -224,12 +219,6 @@ pub trait ValueVisitor {Err(DeserializeError::UnsupportedType(ValueType::IpAddr))}- #[inline]- /// Called when the deserializer visits a facet value.- fn visit_facet(&self, _val: Facet) -> Result<Self::Value, DeserializeError> {- Err(DeserializeError::UnsupportedType(ValueType::Facet))- }-#[inline]/// Called when the deserializer visits a bytes value.fn visit_bytes(&self, _val: Vec<u8>) -> Result<Self::Value, DeserializeError> {@@ -370,7 +359,6 @@ wheretype_codes::F64_CODE => ValueType::F64,type_codes::BOOL_CODE => ValueType::Bool,type_codes::DATE_CODE => ValueType::DateTime,- type_codes::HIERARCHICAL_FACET_CODE => ValueType::Facet,type_codes::BYTES_CODE => ValueType::Bytes,type_codes::EXT_CODE => {let ext_type_code = <u8 as BinarySerializable>::deserialize(reader)?;@@ -452,11 +440,6 @@ where<DateTime as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)}- fn deserialize_facet(self) -> Result<Facet, DeserializeError> {- self.validate_type(ValueType::Facet)?;- <Facet as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)- }-fn deserialize_bytes(self) -> Result<Vec<u8>, DeserializeError> {self.validate_type(ValueType::Bytes)?;<Vec<u8> as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)@@ -506,10 +489,6 @@ wherelet val = self.deserialize_datetime()?;visitor.visit_datetime(val)}- ValueType::Facet => {- let val = self.deserialize_facet()?;- visitor.visit_facet(val)- }ValueType::Bytes => {let val = self.deserialize_bytes()?;visitor.visit_bytes(val)@@ -726,16 +705,6 @@ impl ValueDeserialize for Ipv6Addr {}}-impl ValueDeserialize for Facet {- #[inline]- fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>- where- D: ValueDeserializer<'de>,- {- deserializer.deserialize_facet()- }-}-impl ValueDeserialize for Vec<u8> {#[inline]fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>@@ -923,11 +892,6 @@ mod tests {crate::schema::OwnedValue::Date(DateTime::from_timestamp_micros(100)));- let facet = Facet::from_text("/hello/world").unwrap();- let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());- let value = deserialize_value(result);- assert_eq!(value, crate::schema::OwnedValue::Facet(facet));-let pre_tok_str = PreTokenizedString {text: "hello, world".to_string(),tokens: vec![Token::default(), Token::default()],diff --git a/crates/tantivy/src/schema/document/default_document.rs b/crates/tantivy/src/schema/document/default_document.rsindex 40df237f..119c50a2 100644--- a/crates/tantivy/src/schema/document/default_document.rs+++ b/crates/tantivy/src/schema/document/default_document.rs@@ -14,7 +14,7 @@ use crate::schema::document::{DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,};use crate::schema::field_type::ValueParsingError;-use crate::schema::{Facet, Field, NamedFieldDocument, OwnedValue, Schema};+use crate::schema::{Field, NamedFieldDocument, OwnedValue, Schema};use crate::tokenizer::PreTokenizedString;#[repr(packed)]@@ -68,15 +68,6 @@ impl CompactDoc {self.field_values.len()}- /// Adding a facet to the document.- pub fn add_facet<F>(&mut self, field: Field, path: F)- where- Facet: From<F>,- {- let facet = Facet::from(path);- self.add_leaf_field_value(field, ReferenceValueLeaf::Facet(facet.encoded_str()));- }-/// Add a text field.pub fn add_text<S: AsRef<str>>(&mut self, field: Field, text: S) {self.add_leaf_field_value(field, ReferenceValueLeaf::Str(text.as_ref()));@@ -245,9 +236,6 @@ impl CompactDoc {ReferenceValueLeaf::Str(bytes) => {write_bytes_into(&mut self.node_data, bytes.as_bytes())}- ReferenceValueLeaf::Facet(bytes) => {- write_bytes_into(&mut self.node_data, bytes.as_bytes())- }ReferenceValueLeaf::Bytes(bytes) => write_bytes_into(&mut self.node_data, bytes),ReferenceValueLeaf::U64(num) => write_into(&mut self.node_data, num),ReferenceValueLeaf::I64(num) => write_into(&mut self.node_data, num),@@ -430,10 +418,6 @@ impl<'a> CompactDocValue<'a> {let str_ref = self.container.extract_str(addr);Ok(ReferenceValueLeaf::Str(str_ref).into())}- ValueType::Facet => {- let str_ref = self.container.extract_str(addr);- Ok(ReferenceValueLeaf::Facet(str_ref).into())- }ValueType::Bytes => {let data = self.container.extract_bytes(addr);Ok(ReferenceValueLeaf::Bytes(data).into())@@ -534,8 +518,6 @@ pub enum ValueType {F64 = 4,/// Date/time with nanoseconds precisionDate = 5,- /// Facet- Facet = 6,/// Arbitrarily sized byte arrayBytes = 7,/// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.@@ -591,7 +573,6 @@ impl<'a> From<&ReferenceValueLeaf<'a>> for ValueType {ReferenceValueLeaf::Date(_) => ValueType::Date,ReferenceValueLeaf::IpAddr(_) => ValueType::IpAddr,ReferenceValueLeaf::PreTokStr(_) => ValueType::PreTokStr,- ReferenceValueLeaf::Facet(_) => ValueType::Facet,ReferenceValueLeaf::Bytes(_) => ValueType::Bytes,}}diff --git a/crates/tantivy/src/schema/document/existing_type_impls.rs b/crates/tantivy/src/schema/document/existing_type_impls.rsindex d3187f6d..45d00974 100644--- a/crates/tantivy/src/schema/document/existing_type_impls.rs+++ b/crates/tantivy/src/schema/document/existing_type_impls.rs@@ -13,7 +13,6 @@ use serde_json::Number;use time::format_description::well_known::Rfc3339;use time::OffsetDateTime;-use super::facet::Facet;use super::ReferenceValueLeaf;use crate::schema::document::{ArrayAccess, DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,@@ -83,15 +82,6 @@ impl<'a> Value<'a> for &'a String {}}-impl<'a> Value<'a> for &'a Facet {- type ArrayIter = Empty<&'a Facet>;- type ObjectIter = Empty<(&'a str, &'a Facet)>;- #[inline]- fn as_value(&self) -> ReferenceValue<'a, Self> {- ReferenceValue::Leaf(ReferenceValueLeaf::Facet(self.encoded_str()))- }-}-impl<'a> Value<'a> for &'a u64 {type ArrayIter = Empty<&'a u64>;type ObjectIter = Empty<(&'a str, &'a u64)>;diff --git a/crates/tantivy/src/schema/document/mod.rs b/crates/tantivy/src/schema/document/mod.rsindex 91ce894c..71275bad 100644--- a/crates/tantivy/src/schema/document/mod.rs+++ b/crates/tantivy/src/schema/document/mod.rs@@ -184,11 +184,13 @@ use super::*;pub trait Document: Send + Sync + 'static {/// The value of the field.type Value<'a>: Value<'a> + Clone- where Self: 'a;+ where+ Self: 'a;/// The iterator over all of the fields and values within the doc.type FieldsValuesIter<'a>: Iterator<Item = (Field, Self::Value<'a>)>- where Self: 'a;+ where+ Self: 'a;/// Get an iterator iterating over all fields and values in a document.fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_>;@@ -255,7 +257,6 @@ pub(crate) mod type_codes {pub const TEXT_CODE: u8 = 0;pub const U64_CODE: u8 = 1;pub const I64_CODE: u8 = 2;- pub const HIERARCHICAL_FACET_CODE: u8 = 3;pub const BYTES_CODE: u8 = 4;pub const DATE_CODE: u8 = 5;pub const F64_CODE: u8 = 6;diff --git a/crates/tantivy/src/schema/document/owned_value.rs b/crates/tantivy/src/schema/document/owned_value.rsindex 9fbf1f8c..95240af2 100644--- a/crates/tantivy/src/schema/document/owned_value.rs+++ b/crates/tantivy/src/schema/document/owned_value.rs@@ -14,7 +14,6 @@ use crate::schema::document::{ArrayAccess, DeserializeError, ObjectAccess, ReferenceValue, Value, ValueDeserialize,ValueDeserializer, ValueVisitor,};-use crate::schema::Facet;use crate::tokenizer::PreTokenizedString;use crate::DateTime;@@ -39,8 +38,6 @@ pub enum OwnedValue {Bool(bool),/// Date/time with nanoseconds precisionDate(DateTime),- /// Facet- Facet(Facet),/// Arbitrarily sized byte arrayBytes(Vec<u8>),/// A set of values.@@ -72,7 +69,6 @@ impl<'a> Value<'a> for &'a OwnedValue {OwnedValue::F64(val) => ReferenceValueLeaf::F64(*val).into(),OwnedValue::Bool(val) => ReferenceValueLeaf::Bool(*val).into(),OwnedValue::Date(val) => ReferenceValueLeaf::Date(*val).into(),- OwnedValue::Facet(val) => ReferenceValueLeaf::Facet(val.encoded_str()).into(),OwnedValue::Bytes(val) => ReferenceValueLeaf::Bytes(val).into(),OwnedValue::IpAddr(val) => ReferenceValueLeaf::IpAddr(*val).into(),OwnedValue::Array(array) => ReferenceValue::Array(array.iter()),@@ -83,7 +79,9 @@ impl<'a> Value<'a> for &'a OwnedValue {impl ValueDeserialize for OwnedValue {fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>- where D: ValueDeserializer<'de> {+ where+ D: ValueDeserializer<'de>,+ {struct Visitor;impl ValueVisitor for Visitor {@@ -121,10 +119,6 @@ impl ValueDeserialize for OwnedValue {Ok(OwnedValue::IpAddr(val))}- fn visit_facet(&self, val: Facet) -> Result<Self::Value, DeserializeError> {- Ok(OwnedValue::Facet(val))- }-fn visit_bytes(&self, val: Vec<u8>) -> Result<Self::Value, DeserializeError> {Ok(OwnedValue::Bytes(val))}@@ -137,7 +131,9 @@ impl ValueDeserialize for OwnedValue {}fn visit_array<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>- where A: ArrayAccess<'de> {+ where+ A: ArrayAccess<'de>,+ {let mut elements = Vec::with_capacity(access.size_hint());while let Some(value) = access.next_element()? {@@ -148,7 +144,9 @@ impl ValueDeserialize for OwnedValue {}fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>- where A: ObjectAccess<'de> {+ where+ A: ObjectAccess<'de>,+ {let mut elements = Vec::with_capacity(access.size_hint());while let Some((key, value)) = access.next_entry()? {@@ -167,7 +165,9 @@ impl Eq for OwnedValue {}impl serde::Serialize for OwnedValue {fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>- where S: serde::Serializer {+ where+ S: serde::Serializer,+ {use serde::ser::SerializeMap;match *self {OwnedValue::Null => serializer.serialize_unit(),@@ -180,7 +180,6 @@ impl serde::Serialize for OwnedValue {OwnedValue::Date(ref date) => {time::serde::rfc3339::serialize(&date.into_utc(), serializer)}- OwnedValue::Facet(ref facet) => facet.serialize(serializer),OwnedValue::Bytes(ref bytes) => serializer.serialize_str(&BASE64.encode(bytes)),OwnedValue::Object(ref obj) => {let mut map = serializer.serialize_map(Some(obj.len()))?;@@ -204,7 +203,9 @@ impl serde::Serialize for OwnedValue {impl<'de> serde::Deserialize<'de> for OwnedValue {fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>- where D: serde::Deserializer<'de> {+ where+ D: serde::Deserializer<'de>,+ {struct ValueVisitor;impl<'de> serde::de::Visitor<'de> for ValueVisitor {@@ -239,12 +240,16 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {}fn visit_unit<E>(self) -> Result<Self::Value, E>- where E: serde::de::Error {+ where+ E: serde::de::Error,+ {Ok(OwnedValue::Null)}fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>- where A: SeqAccess<'de> {+ where+ A: SeqAccess<'de>,+ {let mut elements = Vec::with_capacity(seq.size_hint().unwrap_or_default());while let Some(value) = seq.next_element()? {@@ -255,7 +260,9 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {}fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>- where A: MapAccess<'de> {+ where+ A: MapAccess<'de>,+ {let mut object = map.size_hint().map(Vec::with_capacity).unwrap_or_default();while let Some((key, value)) = map.next_entry()? {object.push((key, value));@@ -278,9 +285,6 @@ impl<'a, V: Value<'a>> From<ReferenceValue<'a, V>> for OwnedValue {ReferenceValueLeaf::I64(val) => OwnedValue::I64(val),ReferenceValueLeaf::F64(val) => OwnedValue::F64(val),ReferenceValueLeaf::Date(val) => OwnedValue::Date(val),- ReferenceValueLeaf::Facet(val) => {- OwnedValue::Facet(Facet::from_encoded_string(val.to_string()))- }ReferenceValueLeaf::Bytes(val) => OwnedValue::Bytes(val.to_vec()),ReferenceValueLeaf::IpAddr(val) => OwnedValue::IpAddr(val),ReferenceValueLeaf::Bool(val) => OwnedValue::Bool(val),@@ -351,12 +355,6 @@ impl<'a> From<&'a [u8]> for OwnedValue {}}-impl From<Facet> for OwnedValue {- fn from(facet: Facet) -> OwnedValue {- OwnedValue::Facet(facet)- }-}-impl From<Vec<u8>> for OwnedValue {fn from(bytes: Vec<u8>) -> OwnedValue {OwnedValue::Bytes(bytes)diff --git a/crates/tantivy/src/schema/document/se.rs b/crates/tantivy/src/schema/document/se.rsindex 18d49467..a0bb2661 100644--- a/crates/tantivy/src/schema/document/se.rs+++ b/crates/tantivy/src/schema/document/se.rs@@ -119,11 +119,6 @@ whereself.write_type_code(type_codes::DATE_CODE)?;val.serialize(self.writer)}- ReferenceValueLeaf::Facet(val) => {- self.write_type_code(type_codes::HIERARCHICAL_FACET_CODE)?;-- Cow::Borrowed(val).serialize(self.writer)- }ReferenceValueLeaf::Bytes(val) => {self.write_type_code(type_codes::BYTES_CODE)?;@@ -312,7 +307,7 @@ mod tests {use super::*;use crate::schema::document::existing_type_impls::JsonObjectIter;- use crate::schema::{Facet, Field, FAST, STORED, TEXT};+ use crate::schema::{Field, FAST, STORED, TEXT};use crate::tokenizer::PreTokenizedString;fn serialize_value<'a>(value: ReferenceValue<'a, &'a serde_json::Value>) -> Vec<u8> {@@ -427,16 +422,6 @@ mod tests {"Expected serialized value to match the binary representation");- let facet = Facet::from_text("/hello/world").unwrap();- let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());- let expected = binary_repr!(- type_codes::HIERARCHICAL_FACET_CODE => Facet::from_text("/hello/world").unwrap(),- );- assert_eq!(- result, expected,- "Expected serialized value to match the binary representation"- );-let pre_tok_str = PreTokenizedString {text: "hello, world".to_string(),tokens: vec![Token::default(), Token::default()],diff --git a/crates/tantivy/src/schema/document/value.rs b/crates/tantivy/src/schema/document/value.rsindex fc3023da..086ccefa 100644--- a/crates/tantivy/src/schema/document/value.rs+++ b/crates/tantivy/src/schema/document/value.rs@@ -83,12 +83,6 @@ pub trait Value<'a>: Send + Sync + Debug {self.as_leaf().and_then(|leaf| leaf.as_bytes())}- #[inline]- /// If the Value is a facet, returns the associated facet. Returns None otherwise.- fn as_facet(&self) -> Option<&'a str> {- self.as_leaf().and_then(|leaf| leaf.as_facet())- }-#[inline]/// Returns the iterator over the array if the Value is an array.fn as_array(&self) -> Option<Self::ArrayIter> {@@ -125,9 +119,6 @@ pub enum ReferenceValueLeaf<'a> {F64(f64),/// Date/time with nanoseconds precisionDate(DateTime),- /// Facet string needs to match the format of- /// [Facet::encoded_str](crate::schema::Facet::encoded_str).- Facet(&'a str),/// Arbitrarily sized byte arrayBytes(&'a [u8]),/// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.@@ -211,7 +202,6 @@ impl<'a, T: Value<'a> + ?Sized> From<ReferenceValueLeaf<'a>> for ReferenceValue<ReferenceValueLeaf::I64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::I64(val)),ReferenceValueLeaf::F64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::F64(val)),ReferenceValueLeaf::Date(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Date(val)),- ReferenceValueLeaf::Facet(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Facet(val)),ReferenceValueLeaf::Bytes(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(val)),ReferenceValueLeaf::IpAddr(val) => {ReferenceValue::Leaf(ReferenceValueLeaf::IpAddr(val))@@ -321,16 +311,6 @@ impl<'a> ReferenceValueLeaf<'a> {None}}-- #[inline]- /// If the Value is a facet, returns the associated facet. Returns None otherwise.- pub fn as_facet(&self) -> Option<&'a str> {- if let Self::Facet(val) = self {- Some(val)- } else {- None- }- }}/// A enum representing a value for tantivy to index.@@ -433,12 +413,6 @@ whereself.as_leaf().and_then(|leaf| leaf.as_bytes())}- #[inline]- /// If the Value is a facet, returns the associated facet. Returns None otherwise.- pub fn as_facet(&self) -> Option<&'a str> {- self.as_leaf().and_then(|leaf| leaf.as_facet())- }-#[inline]/// Returns true if the Value is an array.pub fn is_array(&self) -> bool {diff --git a/crates/tantivy/src/schema/facet.rs b/crates/tantivy/src/schema/facet.rsdeleted file mode 100644index 515c41df..00000000--- a/crates/tantivy/src/schema/facet.rs+++ /dev/null@@ -1,354 +0,0 @@-use std::borrow::{Borrow, Cow};-use std::fmt::{self, Debug, Display, Formatter};-use std::io::{self, Read, Write};-use std::str;-use std::string::FromUtf8Error;--use crate::common::BinarySerializable;-use once_cell::sync::Lazy;-use regex::Regex;-use serde::de::Error as _;-use serde::{Deserialize, Deserializer, Serialize, Serializer};--const SLASH_BYTE: u8 = b'/';-const ESCAPE_BYTE: u8 = b'\\';--/// BYTE used as a level separation in the binary-/// representation of facets.-pub const FACET_SEP_BYTE: u8 = 0u8;--/// `char` used as a level separation in the binary-/// representation of facets. (It is the null codepoint.)-pub const FACET_SEP_CHAR: char = '\u{0}';--/// An error enum for facet parser.-#[derive(Debug, PartialEq, Eq, Error)]-pub enum FacetParseError {- /// The facet text representation is unparsable.- #[error("Failed to parse the facet string: '{0}'")]- FacetParseError(String),-}--/// A Facet represent a point in a given hierarchy.-///-/// They are typically represented similarly to a filepath.-/// For instance, an e-commerce website could-/// have a `Facet` for `/electronics/tv_and_video/led_tv`.-///-/// A document can be associated with any number of facets.-/// The hierarchy implicitly imply that a document-/// belonging to a facet also belongs to the ancestor of-/// its facet. In the example above, `/electronics/tv_and_video/`-/// and `/electronics`.-#[derive(Clone, Default, Eq, Hash, PartialEq, Ord, PartialOrd)]-pub struct Facet(pub(crate) String);--impl Facet {- /// Returns a new instance of the "root facet"- /// Equivalent to `/`.- pub fn root() -> Facet {- Facet("".to_string())- }-- /// Returns true if the facet is the root facet `/`.- pub fn is_root(&self) -> bool {- self.encoded_str().is_empty()- }-- /// Returns a binary representation of the facet.- ///- /// In this representation, `0u8` is used as a separator- /// and the string parts of the facet are unescaped.- /// (The first `/` is not encoded at all).- ///- /// This representation has the benefit of making it possible to- /// express "being a child of a given facet" as a range over- /// the term ordinals.- pub fn encoded_str(&self) -> &str {- &self.0- }-- pub(crate) fn from_encoded_string(facet_string: String) -> Facet {- Facet(facet_string)- }-- /// Creates a `Facet` from its binary representation.- pub fn from_encoded(encoded_bytes: Vec<u8>) -> Result<Facet, FromUtf8Error> {- // facet bytes validation. `0u8` is used a separator but that is still legal utf-8- String::from_utf8(encoded_bytes).map(Facet)- }-- /// Parse a text representation of a facet.- ///- /// If one of the segments of this path- /// contains a `/`, it should be escaped- /// using an anti-slash `\`.- pub fn from_text<T>(path: &T) -> Result<Facet, FacetParseError>- where- T: ?Sized + AsRef<str>,- {- #[derive(Copy, Clone)]- enum State {- Escaped,- Idle,- }- let path_ref = path.as_ref();- if path_ref.is_empty() {- return Err(FacetParseError::FacetParseError(path_ref.to_string()));- }- if !path_ref.starts_with('/') {- return Err(FacetParseError::FacetParseError(path_ref.to_string()));- }- let mut facet_encoded = String::new();- let mut state = State::Idle;- let path_bytes = path_ref.as_bytes();- let mut last_offset = 1;- for i in 1..path_bytes.len() {- let c = path_bytes[i];- match (state, c) {- (State::Idle, ESCAPE_BYTE) => {- facet_encoded.push_str(&path_ref[last_offset..i]);- last_offset = i + 1;- state = State::Escaped- }- (State::Idle, SLASH_BYTE) => {- facet_encoded.push_str(&path_ref[last_offset..i]);- facet_encoded.push(FACET_SEP_CHAR);- last_offset = i + 1;- }- (State::Escaped, _escaped_char) => {- state = State::Idle;- }- (State::Idle, _any_char) => {}- }- }- facet_encoded.push_str(&path_ref[last_offset..]);- Ok(Facet(facet_encoded))- }-- /// Returns a `Facet` from an iterator over the different- /// steps of the facet path.- ///- /// The steps are expected to be unescaped.- pub fn from_path<Path>(path: Path) -> Facet- where- Path: IntoIterator,- Path::Item: AsRef<str>,- {- let mut facet_string: String = String::with_capacity(100);- let mut step_it = path.into_iter();- if let Some(step) = step_it.next() {- facet_string.push_str(step.as_ref());- }- for step in step_it {- facet_string.push(FACET_SEP_CHAR);- facet_string.push_str(step.as_ref());- }- Facet(facet_string)- }-- /// Returns `true` if other is a `strict` subfacet of `self`.- ///- /// Disclaimer: By strict we mean that the relation is not reflexive.- /// `/happy` is not a prefix of `/happy`.- pub fn is_prefix_of(&self, other: &Facet) -> bool {- let self_str = self.encoded_str();- let other_str = other.encoded_str();-- // Fast path, but also required to ensure that / is not a prefix of /.- if other_str.len() <= self_str.len() {- return false;- }-- // Root is a prefix of every other path.- // This is not just an optimisation. It is necessary for correctness.- if self.is_root() {- return true;- }-- other_str.starts_with(self_str) && other_str.as_bytes()[self_str.len()] == FACET_SEP_BYTE- }-- /// Extract path from the `Facet`.- pub fn to_path(&self) -> Vec<&str> {- self.encoded_str().split(|c| c == FACET_SEP_CHAR).collect()- }-- /// This function is the inverse of Facet::from(&str).- pub fn to_path_string(&self) -> String {- format!("{self}")- }-}--impl Borrow<str> for Facet {- fn borrow(&self) -> &str {- self.encoded_str()- }-}--impl<'a, T: ?Sized + AsRef<str>> From<&'a T> for Facet {- fn from(path_asref: &'a T) -> Facet {- Facet::from_text(path_asref).unwrap()- }-}--impl BinarySerializable for Facet {- fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {- <String as BinarySerializable>::serialize(&self.0, writer)- }-- fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {- Ok(Facet(<String as BinarySerializable>::deserialize(reader)?))- }-}--impl Display for Facet {- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {- for step in self.0.split(FACET_SEP_CHAR) {- write!(f, "/")?;- write!(f, "{}", escape_slashes(step))?;- }- Ok(())- }-}--fn escape_slashes(s: &str) -> Cow<'_, str> {- static SLASH_PTN: Lazy<Regex> = Lazy::new(|| Regex::new(r"[\\/]").unwrap());- SLASH_PTN.replace_all(s, "\\/")-}--impl Serialize for Facet {- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>- where- S: Serializer,- {- serializer.serialize_str(&self.to_string())- }-}--impl<'de> Deserialize<'de> for Facet {- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>- where- D: Deserializer<'de>,- {- <Cow<'de, str> as Deserialize<'de>>::deserialize(deserializer).and_then(|path| {- Facet::from_text(&*path).map_err(|err| D::Error::custom(err.to_string()))- })- }-}--impl Debug for Facet {- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {- write!(f, "Facet({self})")?;- Ok(())- }-}--#[cfg(test)]-mod tests {-- use super::{Facet, FacetParseError};-- #[test]- fn test_root() {- assert_eq!(Facet::root(), Facet::from("/"));- assert_eq!(format!("{}", Facet::root()), "/");- assert!(Facet::root().is_root());- assert_eq!(Facet::root().encoded_str(), "");- }-- #[test]- fn test_from_path() {- assert_eq!(- Facet::from_path(vec!["top", "a", "firstdoc"]),- Facet::from("/top/a/firstdoc")- );- }-- #[test]- fn test_facet_display() {- {- let v = ["first", "second", "third"];- let facet = Facet::from_path(v.iter());- assert_eq!(format!("{facet}"), "/first/second/third");- }- {- let v = ["first", "sec/ond", "third"];- let facet = Facet::from_path(v.iter());- assert_eq!(format!("{facet}"), "/first/sec\\/ond/third");- }- }-- #[test]- fn test_facet_debug() {- let v = ["first", "second", "third"];- let facet = Facet::from_path(v.iter());- assert_eq!(format!("{facet:?}"), "Facet(/first/second/third)");- }-- #[test]- fn test_to_path() {- let v = ["first", "second", "third\\/not_fourth"];- let facet = Facet::from_path(v.iter());- assert_eq!(facet.to_path(), v);- }-- #[test]- fn test_to_path_string() {- let v = ["first", "second", "third/not_fourth"];- let facet = Facet::from_path(v.iter());- assert_eq!(- facet.to_path_string(),- String::from("/first/second/third\\/not_fourth")- );- }-- #[test]- fn test_to_path_string_empty() {- let v: Vec<&str> = vec![];- let facet = Facet::from_path(v.iter());- assert_eq!(facet.to_path_string(), "/");- }-- #[test]- fn test_from_text() {- assert_eq!(- Err(FacetParseError::FacetParseError("INVALID".to_string())),- Facet::from_text("INVALID")- );- }-- #[test]- fn only_proper_prefixes() {- assert!(Facet::from("/foo").is_prefix_of(&Facet::from("/foo/bar")));-- assert!(!Facet::from("/foo/bar").is_prefix_of(&Facet::from("/foo/bar")));- }-- #[test]- fn root_is_a_prefix() {- assert!(Facet::from("/").is_prefix_of(&Facet::from("/foobar")));- assert!(!Facet::from("/").is_prefix_of(&Facet::from("/")));- }-- #[test]- fn deserialize_from_borrowed_string() {- let facet = serde_json::from_str::<Facet>(r#""/foo/bar""#).unwrap();- assert_eq!(facet, Facet::from_path(["foo", "bar"]));- }-- #[test]- fn deserialize_from_owned_string() {- let facet = serde_json::from_str::<Facet>(r#""/foo/\u263A""#).unwrap();- assert_eq!(facet, Facet::from_path(["foo", "☺"]));- }-- #[test]- fn deserialize_from_invalid_string() {- let error = serde_json::from_str::<Facet>(r#""foo/bar""#).unwrap_err();- assert_eq!(- error.to_string(),- "Failed to parse the facet string: 'foo/bar'"- );- }-}diff --git a/crates/tantivy/src/schema/facet_options.rs b/crates/tantivy/src/schema/facet_options.rsdeleted file mode 100644index 77a8714e..00000000--- a/crates/tantivy/src/schema/facet_options.rs+++ /dev/null@@ -1,82 +0,0 @@-use std::ops::BitOr;--use serde::{Deserialize, Serialize};--use crate::schema::flags::{IndexedFlag, SchemaFlagList, StoredFlag};--/// Define how a facet field should be handled by tantivy.-///-/// Note that a Facet is always indexed and stored as a fastfield.-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]-pub struct FacetOptions {- stored: bool,-}--impl FacetOptions {- /// Returns true if the value is stored.- #[inline]- pub fn is_stored(&self) -> bool {- self.stored- }-- /// Set the field as stored.- ///- /// Only the fields that are set as *stored* are- /// persisted into the Tantivy's store.- #[must_use]- pub fn set_stored(mut self) -> FacetOptions {- self.stored = true;- self- }-}--impl From<()> for FacetOptions {- fn from(_: ()) -> FacetOptions {- FacetOptions::default()- }-}--impl From<StoredFlag> for FacetOptions {- fn from(_: StoredFlag) -> Self {- FacetOptions { stored: true }- }-}--impl<T: Into<FacetOptions>> BitOr<T> for FacetOptions {- type Output = FacetOptions;-- fn bitor(self, other: T) -> FacetOptions {- let other = other.into();- FacetOptions {- stored: self.stored | other.stored,- }- }-}--impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for FacetOptions-where- Head: Clone,- Tail: Clone,- Self: BitOr<Output = Self> + From<Head> + From<Tail>,-{- fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {- Self::from(head_tail.head) | Self::from(head_tail.tail)- }-}--impl From<IndexedFlag> for FacetOptions {- fn from(_: IndexedFlag) -> Self {- FacetOptions { stored: false }- }-}--#[cfg(test)]-mod tests {- use crate::schema::{FacetOptions, INDEXED};-- #[test]- fn test_from_index_flag() {- let facet_option = FacetOptions::from(INDEXED);- assert_eq!(facet_option, FacetOptions::default());- }-}diff --git a/crates/tantivy/src/schema/field_entry.rs b/crates/tantivy/src/schema/field_entry.rsindex 932a2972..67cc894d 100644--- a/crates/tantivy/src/schema/field_entry.rs+++ b/crates/tantivy/src/schema/field_entry.rs@@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize};use super::ip_options::IpAddrOptions;use crate::schema::bytes_options::BytesOptions;use crate::schema::{- is_valid_field_name, DateOptions, FacetOptions, FieldType, JsonObjectOptions, NumericOptions,- TextOptions,+ is_valid_field_name, DateOptions, FieldType, JsonObjectOptions, NumericOptions, TextOptions,};/// A `FieldEntry` represents a field and its configuration.@@ -66,11 +65,6 @@ impl FieldEntry {Self::new(field_name, FieldType::IpAddr(ip_options))}- /// Creates a field entry for a facet.- pub fn new_facet(field_name: String, facet_options: FacetOptions) -> FieldEntry {- Self::new(field_name, FieldType::Facet(facet_options))- }-/// Creates a field entry for a bytes fieldpub fn new_bytes(field_name: String, bytes_options: BytesOptions) -> FieldEntry {Self::new(field_name, FieldType::Bytes(bytes_options))@@ -126,7 +120,6 @@ impl FieldEntry {| FieldType::Bool(ref options) => options.is_stored(),FieldType::Date(ref options) => options.is_stored(),FieldType::Str(ref options) => options.is_stored(),- FieldType::Facet(ref options) => options.is_stored(),FieldType::Bytes(ref options) => options.is_stored(),FieldType::JsonObject(ref options) => options.is_stored(),FieldType::IpAddr(ref options) => options.is_stored(),diff --git a/crates/tantivy/src/schema/field_type.rs b/crates/tantivy/src/schema/field_type.rsindex d057a499..472ab4aa 100644--- a/crates/tantivy/src/schema/field_type.rs+++ b/crates/tantivy/src/schema/field_type.rs@@ -11,9 +11,8 @@ use thiserror::Error;use super::ip_options::IpAddrOptions;use super::IntoIpv6Addr;use crate::schema::bytes_options::BytesOptions;-use crate::schema::facet_options::FacetOptions;use crate::schema::{- DateOptions, Facet, IndexRecordOption, JsonObjectOptions, NumericOptions, OwnedValue,+ DateOptions, IndexRecordOption, JsonObjectOptions, NumericOptions, OwnedValue,TextFieldIndexing, TextOptions,};use crate::time::format_description::well_known::Rfc3339;@@ -63,8 +62,6 @@ pub enum Type {Bool = b'o',/// `date(i64) timestamp`Date = b'd',- /// `tantivy::schema::Facet`. Passed as a string in JSON.- Facet = b'h',/// `Vec<u8>`Bytes = b'b',/// Leaf in a Json object.@@ -76,26 +73,23 @@ pub enum Type {impl From<ColumnType> for Type {fn from(value: ColumnType) -> Self {match value {- ColumnType::Str => Type::Str,ColumnType::U64 => Type::U64,ColumnType::I64 => Type::I64,ColumnType::F64 => Type::F64,ColumnType::Bool => Type::Bool,ColumnType::DateTime => Type::Date,ColumnType::Bytes => Type::Bytes,- ColumnType::IpAddr => Type::IpAddr,}}}-const ALL_TYPES: [Type; 10] = [+const ALL_TYPES: [Type; 9] = [Type::Str,Type::U64,Type::I64,Type::F64,Type::Bool,Type::Date,- Type::Facet,Type::Bytes,Type::Json,Type::IpAddr,@@ -123,7 +117,6 @@ impl Type {Type::F64 => "F64",Type::Bool => "Bool",Type::Date => "Date",- Type::Facet => "Facet",Type::Bytes => "Bytes",Type::Json => "Json",Type::IpAddr => "IpAddr",@@ -141,7 +134,6 @@ impl Type {b'f' => Some(Type::F64),b'o' => Some(Type::Bool),b'd' => Some(Type::Date),- b'h' => Some(Type::Facet),b'b' => Some(Type::Bytes),b'j' => Some(Type::Json),b'p' => Some(Type::IpAddr),@@ -169,8 +161,6 @@ pub enum FieldType {Bool(NumericOptions),/// Signed 64-bits Date 64 field type configuration,Date(DateOptions),- /// Hierarchical Facet- Facet(FacetOptions),/// Bytes (one per document)Bytes(BytesOptions),/// Json object@@ -189,7 +179,6 @@ impl FieldType {FieldType::F64(_) => Type::F64,FieldType::Bool(_) => Type::Bool,FieldType::Date(_) => Type::Date,- FieldType::Facet(_) => Type::Facet,FieldType::Bytes(_) => Type::Bytes,FieldType::JsonObject(_) => Type::Json,FieldType::IpAddr(_) => Type::IpAddr,@@ -215,7 +204,6 @@ impl FieldType {| FieldType::F64(ref int_options)| FieldType::Bool(ref int_options) => int_options.is_indexed(),FieldType::Date(ref date_options) => date_options.is_indexed(),- FieldType::Facet(ref _facet_options) => true,FieldType::Bytes(ref bytes_options) => bytes_options.is_indexed(),FieldType::JsonObject(ref json_object_options) => json_object_options.is_indexed(),FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_indexed(),@@ -254,7 +242,6 @@ impl FieldType {| FieldType::Bool(ref int_options) => int_options.is_fast(),FieldType::Date(ref date_options) => date_options.is_fast(),FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(),- FieldType::Facet(_) => true,FieldType::JsonObject(ref json_object_options) => json_object_options.is_fast(),}}@@ -271,7 +258,6 @@ impl FieldType {| FieldType::F64(ref int_options)| FieldType::Bool(ref int_options) => int_options.fieldnorms(),FieldType::Date(ref date_options) => date_options.fieldnorms(),- FieldType::Facet(_) => false,FieldType::Bytes(ref bytes_options) => bytes_options.fieldnorms(),FieldType::JsonObject(ref _json_object_options) => false,FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.fieldnorms(),@@ -308,7 +294,6 @@ impl FieldType {None}}- FieldType::Facet(ref _facet_options) => Some(IndexRecordOption::Basic),FieldType::Bytes(ref bytes_options) => {if bytes_options.is_indexed() {Some(IndexRecordOption::Basic)@@ -408,7 +393,6 @@ impl FieldType {})}}- FieldType::Facet(_) => Ok(OwnedValue::Facet(Facet::from(&field_text))),FieldType::Bytes(_) => BASE64.decode(&field_text).map(OwnedValue::Bytes)@@ -474,7 +458,7 @@ impl FieldType {})}}- FieldType::Facet(_) | FieldType::Bytes(_) => Err(ValueParsingError::TypeError {+ FieldType::Bytes(_) => Err(ValueParsingError::TypeError {expected: "a string",json: JsonValue::Number(field_val_num),}),diff --git a/crates/tantivy/src/schema/mod.rs b/crates/tantivy/src/schema/mod.rsindex 0d4b4b1a..2c6e98d8 100644--- a/crates/tantivy/src/schema/mod.rs+++ b/crates/tantivy/src/schema/mod.rs@@ -8,7 +8,7 @@//! - the field name (may contain any characted, can't start with a `-` and can't be empty. Some//! characters may require escaping when using the query parser).//! - the type of the field (currently `text`, `u64`, `i64`, `f64`, `bool`, `date`, `IpAddr`,-//! facets, bytes and json are supported)+//! bytes and json are supported)//! - how the field should be indexed / stored.//!//! This very last point is critical as it will enable / disable some of the functionality@@ -107,8 +107,6 @@//! ```pub mod document;-mod facet;-mod facet_options;mod schema;pub(crate) mod term;@@ -131,9 +129,6 @@ use crate::columnar::ColumnType;pub use self::bytes_options::BytesOptions;pub use self::date_time_options::{DateOptions, DateTimePrecision, DATE_TIME_PRECISION_INDEXED};pub use self::document::{DocParsingError, Document, OwnedValue, TantivyDocument, Value};-pub(crate) use self::facet::FACET_SEP_BYTE;-pub use self::facet::{Facet, FacetParseError};-pub use self::facet_options::FacetOptions;pub use self::field::Field;pub use self::field_entry::FieldEntry;pub use self::field_type::{FieldType, Type};@@ -158,16 +153,13 @@ pub fn is_valid_field_name(field_name: &str) -> bool {pub(crate) fn value_type_to_column_type(typ: Type) -> Option<ColumnType> {match typ {- Type::Str => Some(ColumnType::Str),Type::U64 => Some(ColumnType::U64),Type::I64 => Some(ColumnType::I64),Type::F64 => Some(ColumnType::F64),Type::Bool => Some(ColumnType::Bool),Type::Date => Some(ColumnType::DateTime),- Type::Facet => Some(ColumnType::Str),Type::Bytes => Some(ColumnType::Bytes),- Type::IpAddr => Some(ColumnType::IpAddr),- Type::Json => None,+ Type::Json | Type::Str | Type::IpAddr => None,}}diff --git a/crates/tantivy/src/schema/schema.rs b/crates/tantivy/src/schema/schema.rsindex a91fb671..1c377dc8 100644--- a/crates/tantivy/src/schema/schema.rs+++ b/crates/tantivy/src/schema/schema.rs@@ -155,16 +155,6 @@ impl SchemaBuilder {self.add_field(field_entry)}- /// Adds a facet field to the schema.- pub fn add_facet_field(- &mut self,- field_name: &str,- facet_options: impl Into<FacetOptions>,- ) -> Field {- let field_entry = FieldEntry::new_facet(field_name.to_string(), facet_options.into());- self.add_field(field_entry)- }-/// Adds a fast bytes field to the schema.////// Bytes field are not searchable and are only used@@ -372,7 +362,9 @@ impl Schema {impl Serialize for Schema {fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>- where S: Serializer {+ where+ S: Serializer,+ {let mut seq = serializer.serialize_seq(Some(self.0.fields.len()))?;for e in &self.0.fields {seq.serialize_element(e)?;@@ -383,7 +375,9 @@ impl Serialize for Schema {impl<'de> Deserialize<'de> for Schema {fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>- where D: Deserializer<'de> {+ where+ D: Deserializer<'de>,+ {struct SchemaVisitor;impl<'de> Visitor<'de> for SchemaVisitor {@@ -394,7 +388,9 @@ impl<'de> Deserialize<'de> for Schema {}fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>- where A: SeqAccess<'de> {+ where+ A: SeqAccess<'de>,+ {let mut schema = SchemaBuilder {fields: Vec::with_capacity(seq.size_hint().unwrap_or(0)),fields_map: HashMap::with_capacity(seq.size_hint().unwrap_or(0)),diff --git a/crates/tantivy/src/schema/term.rs b/crates/tantivy/src/schema/term.rsindex 62261f53..21c1b1e4 100644--- a/crates/tantivy/src/schema/term.rs+++ b/crates/tantivy/src/schema/term.rs@@ -10,7 +10,7 @@ use super::date_time_options::DATE_TIME_PRECISION_INDEXED;use super::Field;use crate::fastfield::FastValue;use crate::json_utils::split_json_path;-use crate::schema::{Facet, Type};+use crate::schema::Type;use crate::DateTime;/// Term represents the value that the token can take.@@ -124,12 +124,6 @@ impl Term {Term::from_fast_value(field, &val.truncate(DATE_TIME_PRECISION_INDEXED))}- /// Creates a `Term` given a facet.- pub fn from_facet(field: Field, facet: &Facet) -> Term {- let facet_encoded_str = facet.encoded_str();- Term::with_bytes_and_field_and_payload(Type::Facet, field, facet_encoded_str.as_bytes())- }-/// Builds a term given a field, and a string valuepub fn from_field_text(field: Field, text: &str) -> Term {Term::with_bytes_and_field_and_payload(Type::Str, field, text.as_bytes())@@ -401,18 +395,6 @@ wherestr::from_utf8(self.value_bytes()).ok()}- /// Returns the facet associated with the term.- ///- /// Returns `None` if the field is not of facet type- /// or if the bytes are not valid utf-8.- pub fn as_facet(&self) -> Option<Facet> {- if self.typ() != Type::Facet {- return None;- }- let facet_encode_str = str::from_utf8(self.value_bytes()).ok()?;- Some(Facet::from_encoded_string(facet_encode_str.to_string()))- }-/// Returns the bytes associated with the term.////// Returns `None` if the field is not of bytes type.@@ -506,9 +488,6 @@ whereType::Date => {write_opt(f, self.as_date())?;}- Type::Facet => {- write_opt(f, self.as_facet())?;- }Type::Bytes => {write_opt(f, self.as_bytes())?;}diff --git a/crates/tantivy/src/tokenizer/facet_tokenizer.rs b/crates/tantivy/src/tokenizer/facet_tokenizer.rsdeleted file mode 100644index db0df294..00000000--- a/crates/tantivy/src/tokenizer/facet_tokenizer.rs+++ /dev/null@@ -1,128 +0,0 @@-use super::{Token, TokenStream, Tokenizer};-use crate::schema::FACET_SEP_BYTE;--/// The `FacetTokenizer` process a `Facet` binary representation-/// and emits a token for all of its parent.-///-/// For instance, `/america/north_america/canada`-/// will emit the three following tokens-/// - `/america/north_america/canada`-/// - `/america/north_america`-/// - `/america`-#[derive(Clone, Default)]-pub struct FacetTokenizer {- token: Token,-}--#[derive(Debug)]-enum State {- RootFacetNotEmitted,- UpToPosition(usize), //< we already emitted facet prefix up to &text[..cursor]- Terminated,-}--pub struct FacetTokenStream<'a> {- text: &'a str,- state: State,- token: &'a mut Token,-}--impl Tokenizer for FacetTokenizer {- type TokenStream<'a> = FacetTokenStream<'a>;- fn token_stream<'a>(&'a mut self, text: &'a str) -> FacetTokenStream<'a> {- self.token.reset();- self.token.position = 0;- FacetTokenStream {- text,- state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.- token: &mut self.token,- }- }-}--impl<'a> TokenStream for FacetTokenStream<'a> {- fn advance(&mut self) -> bool {- match self.state {- State::RootFacetNotEmitted => {- self.state = if self.text.is_empty() {- State::Terminated- } else {- State::UpToPosition(0)- };- true- }- State::UpToPosition(cursor) => {- let bytes: &[u8] = self.text.as_bytes();- if let Some(next_sep_pos) = bytes[cursor + 1..]- .iter()- .cloned()- .position(|b| b == FACET_SEP_BYTE)- .map(|pos| cursor + 1 + pos)- {- let facet_part = &self.text[cursor..next_sep_pos];- self.token.text.push_str(facet_part);- self.state = State::UpToPosition(next_sep_pos);- } else {- let facet_part = &self.text[cursor..];- self.token.text.push_str(facet_part);- self.state = State::Terminated;- }- true- }- State::Terminated => false,- }- }-- fn token(&self) -> &Token {- self.token- }-- fn token_mut(&mut self) -> &mut Token {- self.token- }-}--#[cfg(test)]-mod tests {-- use super::FacetTokenizer;- use crate::schema::Facet;- use crate::tokenizer::{Token, TokenStream, Tokenizer};-- #[test]- fn test_facet_tokenizer() {- let facet = Facet::from_path(vec!["top", "a", "b"]);- let mut tokens = vec![];- {- let mut add_token = |token: &Token| {- let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap();- tokens.push(format!("{facet}"));- };- FacetTokenizer::default()- .token_stream(facet.encoded_str())- .process(&mut add_token);- }- assert_eq!(tokens.len(), 4);- assert_eq!(tokens[0], "/");- assert_eq!(tokens[1], "/top");- assert_eq!(tokens[2], "/top/a");- assert_eq!(tokens[3], "/top/a/b");- }-- #[test]- fn test_facet_tokenizer_root_facets() {- let facet = Facet::root();- let mut tokens = vec![];- {- let mut add_token = |token: &Token| {- let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap(); // ok test- tokens.push(format!("{facet}"));- };- FacetTokenizer::default()- .token_stream(facet.encoded_str()) // ok test- .process(&mut add_token);- }- assert_eq!(tokens.len(), 1);- assert_eq!(tokens[0], "/");- }-}diff --git a/crates/tantivy/src/tokenizer/mod.rs b/crates/tantivy/src/tokenizer/mod.rsindex dd79ac10..b2d30319 100644--- a/crates/tantivy/src/tokenizer/mod.rs+++ b/crates/tantivy/src/tokenizer/mod.rs@@ -124,7 +124,6 @@mod alphanum_only;mod ascii_folding_filter;mod empty_tokenizer;-mod facet_tokenizer;mod lower_caser;mod ngram_tokenizer;mod raw_tokenizer;@@ -143,7 +142,6 @@ pub use crate::tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream,pub use self::alphanum_only::AlphaNumOnlyFilter;pub use self::ascii_folding_filter::AsciiFoldingFilter;-pub use self::facet_tokenizer::FacetTokenizer;pub use self::lower_caser::LowerCaser;pub use self::ngram_tokenizer::NgramTokenizer;pub use self::raw_tokenizer::RawTokenizer;