Skip to content

web search done right

remove optional and multivalued columns from tantivy as we only use full columnar indices

Mikkel Denkerba14aaab68f4
diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rs
index 1e1019dc..0af963d3 100644
--- a/crates/core/examples/indexer.rs
+++ b/crates/core/examples/indexer.rs
@@ -40,7 +40,6 @@ fn main() -> anyhow::Result<()> {
.to_string(),
}),
host_centrality_threshold: None,
- topics_path: None,
host_centrality_store_path: data_path.join("centrality/").to_str().unwrap().to_string(),
page_centrality_store_path: Some(
data_path
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index f1ba4798..83036daf 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -33,7 +33,6 @@ pub struct IndexerConfig {
pub warc_source: WarcSource,
pub page_webgraph: Option<IndexerGraphConfig>,
pub host_centrality_threshold: Option<f64>,
- pub topics_path: Option<String>,
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
pub safety_classifier_path: Option<String>,
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index 0594da26..a0e408f9 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -23,8 +23,8 @@ use crate::config::{
defaults, IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LocalConfig,
WebSpellConfig,
};
+use crate::entrypoint::indexer;
use crate::entrypoint::indexer::JobSettings;
-use crate::entrypoint::{dmoz_parser, indexer};
use crate::Result;
use std::fs::{self};
use std::path::Path;
@@ -44,7 +44,6 @@ fn download_files() {
"queries_us.csv",
"sample.warc.gz",
"bangs.json",
- "content.rdf.u8.gz",
"english-wordnet-2022-subset.ttl",
"lambdamart.txt",
"test.zim",
@@ -188,13 +187,6 @@ fn create_inverted_index() -> Result<()> {
page_webgraph: Some(IndexerGraphConfig::Local {
path: webgraph_path.to_str().unwrap().to_string(),
}),
- topics_path: Some(
- Path::new(DATA_PATH)
- .join("human_annotations")
- .to_str()
- .unwrap()
- .to_string(),
- ),
output_path: out_path.to_str().unwrap().to_string(),
limit_warc_files: None,
skip_warc_files: None,
@@ -232,17 +224,9 @@ fn create_entity_index() -> Result<()> {
Ok(())
}
-fn parse_topics() -> Result<()> {
- let dmoz_path = Path::new(DATA_PATH).join("content.rdf.u8.gz");
-
- let topics = dmoz_parser::parse(dmoz_path)?;
- topics.save(Path::new(DATA_PATH).join("human_annotations"))
-}
-
fn index_files() -> Result<()> {
create_webgraph()?;
calculate_centrality();
- parse_topics()?;
create_inverted_index()?;
create_entity_index()?;
build_spellchecker()?;
diff --git a/crates/core/src/entrypoint/dmoz_parser.rs b/crates/core/src/entrypoint/dmoz_parser.rs
deleted file mode 100644
index e54a35e4..00000000
--- a/crates/core/src/entrypoint/dmoz_parser.rs
+++ /dev/null
@@ -1,163 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use flate2::read::MultiGzDecoder;
-use quick_xml::events::Event;
-use url::Url;
-
-use crate::{
- human_website_annotations::{Info, Topic},
- Result,
-};
-use std::{
- collections::HashMap,
- fs::File,
- io::{BufRead, BufReader},
- path::Path,
-};
-
-use crate::human_website_annotations;
-
-struct PageIterator<R: BufRead> {
- reader: quick_xml::Reader<R>,
- buf: Vec<u8>,
-}
-
-impl<R: BufRead> From<R> for PageIterator<R> {
- fn from(reader: R) -> Self {
- Self {
- reader: quick_xml::Reader::from_reader(reader),
- buf: Vec::new(),
- }
- }
-}
-
-impl<R: BufRead> Iterator for PageIterator<R> {
- type Item = Page;
-
- fn next(&mut self) -> Option<Self::Item> {
- let mut current_page = None;
- let mut inside_topic = false;
- let mut inside_desc = false;
-
- while let Ok(event) = self.reader.read_event_into(&mut self.buf) {
- match event {
- Event::Start(ref e) if e.name().as_ref() == b"ExternalPage" => {
- let url_attr = e
- .attributes()
- .filter(std::result::Result::is_ok)
- .find(|attr| attr.as_ref().unwrap().key.as_ref() == b"about")
- .unwrap()
- .unwrap();
- let url = url_attr.decode_and_unescape_value(&self.reader).unwrap();
- current_page = Some(Page::new(url.into_owned()));
- inside_desc = false;
- inside_topic = false;
- }
- Event::End(ref e) if e.name().as_ref() == b"ExternalPage" => break,
- Event::Start(ref e) if e.name().as_ref() == b"topic" && current_page.is_some() => {
- inside_topic = true;
- }
- Event::End(ref e) if e.name().as_ref() == b"topic" && current_page.is_some() => {
- inside_topic = false;
- }
- Event::Start(ref e)
- if e.name().as_ref() == b"d:Description" && current_page.is_some() =>
- {
- inside_desc = true;
- }
- Event::End(ref e)
- if e.name().as_ref() == b"d:Description" && current_page.is_some() =>
- {
- inside_desc = false;
- }
- Event::Text(ref e) => {
- if inside_topic {
- if let Some(page) = &mut current_page {
- let topic = e.unescape().unwrap();
- page.topic.push_str(topic.as_ref());
- }
- } else if inside_desc {
- if let Some(page) = &mut current_page {
- let desc = e.unescape().unwrap();
- page.description.push_str(desc.as_ref());
- }
- }
- }
- Event::Eof => break,
- _ => (),
- }
- }
-
- current_page
- }
-}
-
-#[derive(Debug, Default)]
-struct Page {
- url: String,
- description: String,
- topic: String,
-}
-
-impl Page {
- fn new(url: String) -> Self {
- Self {
- url,
- ..Default::default()
- }
- }
-}
-
-pub fn parse<P: AsRef<Path>>(dmoz_file: P) -> Result<human_website_annotations::Mapper> {
- let file = File::open(dmoz_file)?;
- let reader = BufReader::new(file);
- let reader = BufReader::new(MultiGzDecoder::new(reader));
- let mut map = HashMap::new();
-
- for page in PageIterator::from(reader) {
- let url = match Url::parse(&page.url) {
- Ok(url) => url,
- Err(_) => continue,
- };
-
- if url.path() != "/" {
- continue;
- }
-
- if page.topic.contains("World") {
- continue;
- }
-
- let topic = Topic::from_string("/".to_string() + page.topic.as_str());
-
- let info = Info {
- description: page.description,
- topic,
- };
-
- map.insert(url.host_str().unwrap().to_string(), info);
- }
-
- Ok(map.into())
-}
-
-pub fn run<P: AsRef<Path>>(dmoz_file: P, output_path: P) -> Result<()> {
- let mapper = parse(dmoz_file)?;
- mapper.save(output_path)?;
-
- Ok(())
-}
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 67df1e4d..60a018bd 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -31,7 +31,6 @@ use crate::models::dual_encoder::DualEncoder as DualEncoderModel;
use crate::webgraph::remote::RemoteWebgraph;
use crate::Result;
-use crate::human_website_annotations;
use crate::index::Index;
use crate::rake::RakeModel;
use crate::ranking::SignalComputer;
@@ -42,7 +41,6 @@ pub struct Config {
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
pub page_webgraph: Option<IndexerGraphConfig>,
- pub topics_path: Option<String>,
pub safety_classifier_path: Option<String>,
pub dual_encoder: Option<IndexerDualEncoderConfig>,
}
@@ -53,7 +51,6 @@ impl From<IndexerConfig> for Config {
host_centrality_store_path: config.host_centrality_store_path,
page_centrality_store_path: config.page_centrality_store_path,
page_webgraph: config.page_webgraph,
- topics_path: config.topics_path,
safety_classifier_path: config.safety_classifier_path,
dual_encoder: config.dual_encoder,
}
@@ -66,7 +63,6 @@ impl From<LiveIndexConfig> for Config {
host_centrality_store_path: config.host_centrality_store_path,
page_centrality_store_path: config.page_centrality_store_path,
page_webgraph: config.page_webgraph,
- topics_path: None,
safety_classifier_path: config.safety_classifier_path,
dual_encoder: None,
}
@@ -166,7 +162,6 @@ pub struct IndexingWorker {
page_centrality_store: Option<speedy_kv::Db<NodeID, f64>>,
page_centrality_rank_store: Option<speedy_kv::Db<NodeID, u64>>,
page_webgraph: Option<Webgraph>,
- topics: Option<human_website_annotations::Mapper>,
safety_classifier: Option<safety_classifier::Model>,
job_settings: Option<JobSettings>,
rake: RakeModel,
@@ -198,10 +193,6 @@ impl IndexingWorker {
speedy_kv::Db::open_or_create(Path::new(&p).join("harmonic_rank")).unwrap()
}),
page_webgraph: config.page_webgraph.as_ref().map(Webgraph::new),
- topics: config
- .topics_path
- .as_ref()
- .map(|path| human_website_annotations::Mapper::open(path).unwrap()),
safety_classifier: config
.safety_classifier_path
.as_ref()
@@ -355,16 +346,6 @@ impl IndexingWorker {
}
}
- fn set_dmoz_description(&self, page: &mut Webpage) {
- if let Some(mapper) = self.topics.as_ref() {
- if let Some(info) =
- mapper.get(&page.html.url().host_str().unwrap_or_default().to_string())
- {
- page.dmoz_description = Some(info.description.clone())
- }
- }
- }
-
fn set_keywords(&self, page: &mut Webpage) {
page.keywords = page.html.keywords(&self.rake);
}
@@ -468,7 +449,6 @@ impl IndexingWorker {
}
self.set_page_centralities(&mut prepared);
- self.set_dmoz_description(&mut prepared);
self.set_keywords(&mut prepared);
self.set_safety_classification(&mut prepared);
@@ -516,7 +496,6 @@ mod tests {
host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
page_centrality_store_path: None,
page_webgraph: None,
- topics_path: None,
safety_classifier_path: None,
dual_encoder: Some(IndexerDualEncoderConfig {
model_path: data_path.to_str().unwrap().to_string(),
diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rs
index e2f6e73a..59969878 100644
--- a/crates/core/src/entrypoint/mod.rs
+++ b/crates/core/src/entrypoint/mod.rs
@@ -23,7 +23,6 @@ mod centrality;
#[cfg(feature = "dev")]
pub mod configure;
pub mod crawler;
-pub mod dmoz_parser;
mod entity;
pub mod entity_search_server;
pub mod feed_indexer;
diff --git a/crates/core/src/human_website_annotations.rs b/crates/core/src/human_website_annotations.rs
deleted file mode 100644
index 416af3ab..00000000
--- a/crates/core/src/human_website_annotations.rs
+++ /dev/null
@@ -1,151 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use itertools::Itertools;
-
-use tantivy::schema::Facet;
-
-use crate::Result;
-use std::{
- collections::{HashMap, HashSet},
- fs::File,
- io::{BufReader, Write},
- path::Path,
-};
-
-#[derive(Debug, Hash, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)]
-pub struct Topic<const N: usize = 2> {
- detailed_topics: Vec<String>,
-}
-
-/// bincode seems to not like default const generics, therefore manual impl
-impl<const N: usize> bincode::Encode for Topic<N> {
- fn encode<E: bincode::enc::Encoder>(
- &self,
- encoder: &mut E,
- ) -> Result<(), bincode::error::EncodeError> {
- self.detailed_topics.encode(encoder)
- }
-}
-
-impl<const N: usize> bincode::Decode for Topic<N> {
- fn decode<D: bincode::de::Decoder>(
- decoder: &mut D,
- ) -> Result<Self, bincode::error::DecodeError> {
- Ok(Self {
- detailed_topics: Vec::decode(decoder)?,
- })
- }
-}
-
-impl<'de, const N: usize> bincode::BorrowDecode<'de> for Topic<N> {
- fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(
- decoder: &mut D,
- ) -> Result<Self, bincode::error::DecodeError> {
- Ok(Self {
- detailed_topics: Vec::borrow_decode(decoder)?,
- })
- }
-}
-
-impl<const N: usize> From<Facet> for Topic<N> {
- fn from(facet: Facet) -> Self {
- Self {
- detailed_topics: facet
- .to_path()
- .into_iter()
- .take(N)
- .map(String::from)
- .collect(),
- }
- }
-}
-
-impl<const N: usize> Topic<N> {
- pub fn from_string(s: String) -> Self {
- Self {
- detailed_topics: s
- .split('/')
- .skip(2)
- .take(N)
- .map(String::from)
- .collect::<Vec<_>>(),
- }
- }
-}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct Info {
- pub description: String,
- pub topic: Topic,
-}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct Mapper(HashMap<String, Info>);
-
-impl From<HashMap<String, Info>> for Mapper {
- fn from(map: HashMap<String, Info>) -> Self {
- Self(map)
- }
-}
-
-impl Mapper {
- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
- let mut file = File::options()
- .create(true)
- .truncate(true)
- .write(true)
- .open(path)?;
-
- let bytes = bincode::encode_to_vec(self, bincode::config::standard())?;
- file.write_all(&bytes)?;
-
- Ok(())
- }
-
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mut reader = BufReader::new(File::open(path)?);
-
- Ok(bincode::decode_from_std_read(
- &mut reader,
- bincode::config::standard(),
- )?)
- }
-
- pub fn get(&self, host: &String) -> Option<&Info> {
- self.0.get(host)
- }
-
- pub fn all_topics(&self) -> HashSet<Topic> {
- self.0.values().map(|info| info.topic.clone()).collect()
- }
-
- pub fn top_topics(&self, top_n: usize) -> Vec<Topic> {
- let mut topics: HashMap<Topic, usize> = HashMap::new();
-
- for topic in self.0.values().map(|info| info.topic.clone()) {
- *topics.entry(topic).or_default() += 1;
- }
-
- topics
- .into_iter()
- .sorted_by_key(|(_, count)| *count)
- .rev()
- .map(|(topic, _)| topic)
- .take(top_n)
- .collect()
- }
-}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 5b349225..18162b30 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -57,7 +57,6 @@ mod external_sort;
mod fastfield_reader;
pub mod feed;
mod highlighted;
-mod human_website_annotations;
pub mod hyperloglog;
pub mod image_store;
mod improvement;
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 1341cab5..496f4e36 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -59,13 +59,6 @@ enum Commands {
mode: CentralityMode,
},
- /// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories.
- /// It can be used to calculate the topic centrality for websites or augments website descriptions during indexing.
- DmozParser {
- dmoz_file: String,
- output_path: String,
- },
-
/// Webgraph specific commands.
Webgraph {
#[clap(subcommand)]
@@ -394,10 +387,6 @@ fn main() -> Result<()> {
Commands::Configure { skip_download } => {
configure::run(skip_download)?;
}
- Commands::DmozParser {
- dmoz_file,
- output_path,
- } => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),
Commands::Crawler { options } => match options {
Crawler::Worker { config_path } => {
let config: config::CrawlerConfig = load_toml_config(config_path);
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 1f28b9ad..0fc1d7d1 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -748,7 +748,6 @@ mod tests {
host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
page_centrality_store_path: None,
page_webgraph: None,
- topics_path: None,
safety_classifier_path: None,
dual_encoder: Some(IndexerDualEncoderConfig {
model_path: data_path.to_str().unwrap().to_string(),
diff --git a/crates/tantivy/src/collector/facet_collector.rs b/crates/tantivy/src/collector/facet_collector.rs
deleted file mode 100644
index 0c706faa..00000000
--- a/crates/tantivy/src/collector/facet_collector.rs
+++ /dev/null
@@ -1,827 +0,0 @@
-use std::cmp::Ordering;
-use std::collections::{btree_map, BTreeMap, BTreeSet, BinaryHeap};
-use std::io;
-use std::ops::Bound;
-
-use crate::collector::{Collector, SegmentCollector};
-use crate::fastfield::FacetReader;
-use crate::schema::Facet;
-use crate::{DocId, Score, SegmentOrdinal, SegmentReader};
-
-struct Hit<'a> {
- count: u64,
- facet: &'a Facet,
-}
-
-impl<'a> Eq for Hit<'a> {}
-
-impl<'a> PartialEq<Hit<'a>> for Hit<'a> {
- fn eq(&self, other: &Hit<'_>) -> bool {
- self.count == other.count
- }
-}
-
-impl<'a> PartialOrd<Hit<'a>> for Hit<'a> {
- fn partial_cmp(&self, other: &Hit<'_>) -> Option<Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl<'a> Ord for Hit<'a> {
- fn cmp(&self, other: &Self) -> Ordering {
- other
- .count
- .cmp(&self.count)
- .then(self.facet.cmp(other.facet))
- }
-}
-
-fn facet_depth(facet_bytes: &[u8]) -> usize {
- if facet_bytes.is_empty() {
- 0
- } else {
- facet_bytes.iter().cloned().filter(|b| *b == 0u8).count() + 1
- }
-}
-
-/// Collector for faceting
-///
-/// The collector collects all facets. You need to configure it
-/// beforehand with the facet you want to extract.
-///
-/// This is done by calling `.add_facet(...)` with the root of the
-/// facet you want to extract as argument.
-///
-/// Facet counts will only be computed for the facet that are direct children
-/// of such a root facet.
-///
-/// For instance, if your index represents books, your hierarchy of facets
-/// may contain `category`, `language`.
-///
-/// The category facet may include `subcategories`. For instance, a book
-/// could belong to `/category/fiction/fantasy`.
-///
-/// If you request the facet counts for `/category`, the result will be
-/// the breakdown of counts for the direct children of `/category`
-/// (e.g. `/category/fiction`, `/category/biography`, `/category/personal_development`).
-///
-/// Once collection is finished, you can harvest its results in the form
-/// of a [`FacetCounts`] object, and extract your facet counts from it.
-///
-/// This implementation assumes you are working with a number of facets that
-/// is many hundreds of times smaller than your number of documents.
-///
-///
-/// ```rust
-/// use tantivy::collector::FacetCollector;
-/// use tantivy::query::AllQuery;
-/// use tantivy::schema::{Facet, Schema, FacetOptions, TEXT};
-/// use tantivy::{doc, Index};
-///
-/// fn example() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-///
-/// // Facet have their own specific type.
-/// // It is not a bad practise to put all of your
-/// // facet information in the same field.
-/// let facet = schema_builder.add_facet_field("facet", FacetOptions::default());
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-/// {
-/// let mut index_writer = index.writer(15_000_000)?;
-/// // a document can be associated with any number of facets
-/// index_writer.add_document(doc!(
-/// title => "The Name of the Wind",
-/// facet => Facet::from("/lang/en"),
-/// facet => Facet::from("/category/fiction/fantasy")
-/// ))?;
-/// index_writer.add_document(doc!(
-/// title => "Dune",
-/// facet => Facet::from("/lang/en"),
-/// facet => Facet::from("/category/fiction/sci-fi")
-/// ))?;
-/// index_writer.add_document(doc!(
-/// title => "La Vénus d'Ille",
-/// facet => Facet::from("/lang/fr"),
-/// facet => Facet::from("/category/fiction/fantasy"),
-/// facet => Facet::from("/category/fiction/horror")
-/// ))?;
-/// index_writer.add_document(doc!(
-/// title => "The Diary of a Young Girl",
-/// facet => Facet::from("/lang/en"),
-/// facet => Facet::from("/category/biography")
-/// ))?;
-/// index_writer.commit()?;
-/// }
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// {
-/// let mut facet_collector = FacetCollector::for_field("facet");
-/// facet_collector.add_facet("/lang");
-/// facet_collector.add_facet("/category");
-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-///
-/// // This lists all of the facet counts
-/// let facets: Vec<(&Facet, u64)> = facet_counts
-/// .get("/category")
-/// .collect();
-/// assert_eq!(facets, vec![
-/// (&Facet::from("/category/biography"), 1),
-/// (&Facet::from("/category/fiction"), 3)
-/// ]);
-/// }
-///
-/// {
-/// let mut facet_collector = FacetCollector::for_field("facet");
-/// facet_collector.add_facet("/category/fiction");
-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-///
-/// // This lists all of the facet counts
-/// let facets: Vec<(&Facet, u64)> = facet_counts
-/// .get("/category/fiction")
-/// .collect();
-/// assert_eq!(facets, vec![
-/// (&Facet::from("/category/fiction/fantasy"), 2),
-/// (&Facet::from("/category/fiction/horror"), 1),
-/// (&Facet::from("/category/fiction/sci-fi"), 1)
-/// ]);
-/// }
-///
-/// {
-/// let mut facet_collector = FacetCollector::for_field("facet");
-/// facet_collector.add_facet("/category/fiction");
-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-///
-/// // This lists all of the facet counts
-/// let facets: Vec<(&Facet, u64)> = facet_counts.top_k("/category/fiction", 1);
-/// assert_eq!(facets, vec![
-/// (&Facet::from("/category/fiction/fantasy"), 2)
-/// ]);
-/// }
-///
-/// {
-/// let mut facet_collector = FacetCollector::for_field("facet");
-/// facet_collector.add_facet("/");
-/// let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-///
-/// // This lists all of the facet counts
-/// let facets: Vec<(&Facet, u64)> = facet_counts
-/// .get("/")
-/// .collect();
-/// assert_eq!(facets, vec![
-/// (&Facet::from("/category"), 4),
-/// (&Facet::from("/lang"), 4)
-/// ]);
-/// }
-///
-/// Ok(())
-/// }
-/// # assert!(example().is_ok());
-/// ```
-pub struct FacetCollector {
- field_name: String,
- facets: BTreeSet<Facet>,
-}
-
-pub struct FacetSegmentCollector {
- reader: FacetReader,
- // collapse facet_id -> count
- counts: Vec<u64>,
- // facet_ord -> compressed collapse facet_id
- compressed_collapse_mapping: Vec<usize>,
- // compressed collapse facet_id -> facet_ord
- unique_facet_ords: Vec<(u64, usize)>,
-}
-
-impl FacetCollector {
- /// Create a facet collector to collect the facets
- /// from a specific facet `Field`.
- ///
- /// This function does not check whether the field
- /// is of the proper type.
- pub fn for_field(field_name: impl ToString) -> FacetCollector {
- FacetCollector {
- field_name: field_name.to_string(),
- facets: BTreeSet::default(),
- }
- }
-
- /// Adds a facet that we want to record counts
- ///
- /// Adding facet `Facet::from("/country")` for instance,
- /// will record the counts of all of the direct children of the facet country
- /// (e.g. `/country/FR`, `/country/UK`).
- ///
- /// Adding two facets within which one is the prefix of the other is forbidden.
- /// If you need the correct number of unique documents for two such facets,
- /// just add them in a separate `FacetCollector`.
- pub fn add_facet<T>(&mut self, facet_from: T)
- where
- Facet: From<T>,
- {
- let facet = Facet::from(facet_from);
- for old_facet in &self.facets {
- assert!(
- !old_facet.is_prefix_of(&facet),
- "Tried to add a facet which is a descendant of an already added facet."
- );
- assert!(
- !facet.is_prefix_of(old_facet),
- "Tried to add a facet which is an ancestor of an already added facet."
- );
- }
- self.facets.insert(facet);
- }
-}
-
-fn compress_mapping(mapping: &[(u64, usize)]) -> (Vec<usize>, Vec<(u64, usize)>) {
- // facet_ord -> collapse facet_id
- let mut compressed_collapse_mapping: Vec<usize> = Vec::with_capacity(mapping.len());
- // collapse facet_id -> facet_ord
- let mut unique_facet_ords: Vec<(u64, usize)> = Vec::new();
- if mapping.is_empty() {
- return (Vec::new(), Vec::new());
- }
- compressed_collapse_mapping.push(0);
- unique_facet_ords.push(mapping[0]);
- let mut last_facet_ord = mapping[0];
- let mut last_facet_id = 0;
- for &facet_ord in &mapping[1..] {
- if facet_ord != last_facet_ord {
- last_facet_id += 1;
- last_facet_ord = facet_ord;
- unique_facet_ords.push(facet_ord);
- }
- compressed_collapse_mapping.push(last_facet_id);
- }
- (compressed_collapse_mapping, unique_facet_ords)
-}
-
-impl Collector for FacetCollector {
- type Fruit = FacetCounts;
-
- type Child = FacetSegmentCollector;
-
- fn for_segment(
- &self,
- _: SegmentOrdinal,
- reader: &SegmentReader,
- ) -> crate::Result<FacetSegmentCollector> {
- let facet_reader = reader.facet_reader(&self.field_name)?;
- let facet_dict = facet_reader.facet_dict();
- let collapse_mapping: Vec<(u64, usize)> =
- compute_collapse_mapping(facet_dict, &self.facets)?;
- let (compressed_collapse_mapping, unique_facet_ords) = compress_mapping(&collapse_mapping);
- let counts = vec![0u64; unique_facet_ords.len()];
- Ok(FacetSegmentCollector {
- reader: facet_reader,
- compressed_collapse_mapping,
- counts,
- unique_facet_ords,
- })
- }
-
- fn requires_scoring(&self) -> bool {
- false
- }
-
- fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> crate::Result<FacetCounts> {
- let mut facet_counts: BTreeMap<Facet, u64> = BTreeMap::new();
- for segment_facet_counts in segments_facet_counts {
- for (facet, count) in segment_facet_counts.facet_counts {
- *(facet_counts.entry(facet).or_insert(0)) += count;
- }
- }
- Ok(FacetCounts { facet_counts })
- }
-}
-
-fn is_child_facet(parent_facet: &[u8], possible_child_facet: &[u8]) -> bool {
- if !possible_child_facet.starts_with(parent_facet) {
- return false;
- }
- if parent_facet.is_empty() {
- return true;
- }
- possible_child_facet.get(parent_facet.len()).copied() == Some(0u8)
-}
-
-fn compute_collapse_mapping_one(
- facet_terms: &mut crate::columnar::Streamer,
- facet_bytes: &[u8],
- collapsed: &mut [(u64, usize)],
-) -> io::Result<bool> {
- let mut facet_child: Vec<u8> = Vec::new();
- let mut term_ord = 0;
- let offset = facet_bytes.len() + 1;
- let depth = facet_depth(facet_bytes);
- loop {
- match facet_terms.key().cmp(facet_bytes) {
- Ordering::Less | Ordering::Equal => {}
- Ordering::Greater => {
- if !is_child_facet(facet_bytes, facet_terms.key()) {
- return Ok(true);
- }
- let suffix = &facet_terms.key()[offset..];
- if facet_child.is_empty() || !is_child_facet(&facet_child, suffix) {
- facet_child.clear();
- term_ord = facet_terms.term_ord();
- let end = suffix
- .iter()
- .position(|b| *b == 0u8)
- .unwrap_or(suffix.len());
- facet_child.extend(&suffix[..end]);
- }
- collapsed[facet_terms.term_ord() as usize] = (term_ord, depth);
- }
- }
- if !facet_terms.advance() {
- return Ok(false);
- }
- }
-}
-
-fn compute_collapse_mapping(
- facet_dict: &crate::columnar::Dictionary,
- facets: &BTreeSet<Facet>,
-) -> io::Result<Vec<(u64, usize)>> {
- let mut collapsed = vec![(u64::MAX, 0); facet_dict.num_terms()];
- if facets.is_empty() {
- return Ok(collapsed);
- }
- let mut facet_terms: crate::columnar::Streamer = facet_dict.range().into_stream()?;
- if !facet_terms.advance() {
- return Ok(collapsed);
- }
- let mut facet_bytes = Vec::new();
- for facet in facets {
- facet_bytes.clear();
- facet_bytes.extend(facet.encoded_str().as_bytes());
- if !compute_collapse_mapping_one(&mut facet_terms, &facet_bytes, &mut collapsed[..])? {
- break;
- }
- }
- Ok(collapsed)
-}
-
-impl SegmentCollector for FacetSegmentCollector {
- type Fruit = FacetCounts;
-
- fn collect(&mut self, doc: DocId, _: Score) {
- let mut previous_collapsed_ord: usize = usize::MAX;
- for facet_ord in self.reader.facet_ords(doc) {
- let collapsed_ord = self.compressed_collapse_mapping[facet_ord as usize];
- self.counts[collapsed_ord] += u64::from(collapsed_ord != previous_collapsed_ord);
- previous_collapsed_ord = collapsed_ord;
- }
- }
-
- /// Returns the results of the collection.
- ///
- /// This method does not just return the counters,
- /// it also translates the facet ordinals of the last segment.
- fn harvest(self) -> FacetCounts {
- let mut facet_counts = BTreeMap::new();
- let facet_dict = self.reader.facet_dict();
- for (collapsed_facet_ord, count) in self.counts.iter().cloned().enumerate() {
- if count == 0 {
- continue;
- }
- let mut facet = vec![];
- let (facet_ord, facet_depth) = self.unique_facet_ords[collapsed_facet_ord];
- // TODO handle errors.
- if facet_dict.ord_to_term(facet_ord, &mut facet).is_ok() {
- if let Some((end_collapsed_facet, _)) = facet
- .iter()
- .enumerate()
- .filter(|(_pos, &b)| b == 0u8)
- .nth(facet_depth)
- {
- facet.truncate(end_collapsed_facet);
- }
- if let Ok(facet) = Facet::from_encoded(facet) {
- facet_counts.insert(facet, count);
- }
- }
- }
- FacetCounts { facet_counts }
- }
-}
-
-/// Intermediary result of the `FacetCollector` that stores
-/// the facet counts for all the segments.
-#[derive(Default, Clone)]
-pub struct FacetCounts {
- facet_counts: BTreeMap<Facet, u64>,
-}
-
-pub struct FacetChildIterator<'a> {
- underlying: btree_map::Range<'a, Facet, u64>,
-}
-
-impl<'a> Iterator for FacetChildIterator<'a> {
- type Item = (&'a Facet, u64);
-
- fn next(&mut self) -> Option<Self::Item> {
- self.underlying.next().map(|(facet, count)| (facet, *count))
- }
-}
-
-impl FacetCounts {
- /// Returns an iterator over all of the facet count pairs inside this result.
- /// See the documentation for [`FacetCollector`] for a usage example.
- pub fn get<T>(&self, facet_from: T) -> FacetChildIterator<'_>
- where
- Facet: From<T>,
- {
- let facet = Facet::from(facet_from);
- let lower_bound = Bound::Excluded(facet.clone());
- let upper_bound = if facet.is_root() {
- Bound::Unbounded
- } else {
- let mut facet_after_bytes: String = facet.encoded_str().to_owned();
- facet_after_bytes.push('\u{1}');
- let facet_after = Facet::from_encoded_string(facet_after_bytes);
- Bound::Excluded(facet_after)
- };
- let underlying: btree_map::Range<'_, _, _> =
- self.facet_counts.range((lower_bound, upper_bound));
- FacetChildIterator { underlying }
- }
-
- /// Returns a vector of top `k` facets with their counts, sorted highest-to-lowest by counts.
- /// See the documentation for [`FacetCollector`] for a usage example.
- pub fn top_k<T>(&self, facet: T, k: usize) -> Vec<(&Facet, u64)>
- where
- Facet: From<T>,
- {
- let mut heap = BinaryHeap::with_capacity(k);
- let mut it = self.get(facet);
-
- // push the first k elements to first bring the heap
- // to capacity
- for (facet, count) in (&mut it).take(k) {
- heap.push(Hit { count, facet });
- }
-
- let mut lowest_count: u64 = heap.peek().map(|hit| hit.count).unwrap_or(u64::MIN); //< the `unwrap_or` case may be triggered but the value
- // is never used in that case.
-
- for (facet, count) in it {
- if count > lowest_count {
- if let Some(mut head) = heap.peek_mut() {
- *head = Hit { count, facet };
- }
- // the heap gets reconstructed at this point
- if let Some(head) = heap.peek() {
- lowest_count = head.count;
- }
- }
- }
- heap.into_sorted_vec()
- .into_iter()
- .map(|hit| (hit.facet, hit.count))
- .collect::<Vec<_>>()
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::collections::BTreeSet;
- use std::iter;
-
- use crate::columnar::Dictionary;
- use rand::distributions::Uniform;
- use rand::prelude::SliceRandom;
- use rand::{thread_rng, Rng};
-
- use super::{FacetCollector, FacetCounts};
- use crate::collector::facet_collector::compress_mapping;
- use crate::collector::Count;
- use crate::index::Index;
- use crate::query::{AllQuery, QueryParser, TermQuery};
- use crate::schema::{Facet, FacetOptions, IndexRecordOption, Schema, TantivyDocument};
- use crate::{IndexWriter, Term};
-
- fn test_collapse_mapping_aux(
- facet_terms: &[&str],
- facet_params: &[&str],
- expected_collapsed_mapping: &[(u64, usize)],
- ) {
- let mut facets: Vec<Facet> = facet_terms.iter().map(Facet::from).collect();
- facets.sort();
- let facet_terms: Vec<&str> = facets.iter().map(|facet| facet.encoded_str()).collect();
- let dictionary = Dictionary::build_for_tests(&facet_terms);
- let facet_params: BTreeSet<Facet> = facet_params.iter().map(Facet::from).collect();
- let collapse_mapping = super::compute_collapse_mapping(&dictionary, &facet_params).unwrap();
- assert_eq!(&collapse_mapping[..], expected_collapsed_mapping);
- }
-
- #[test]
- fn test_collapse_simple() {
- test_collapse_mapping_aux(&["/facet/a", "/facet/b"], &["/facet"], &[(0, 1), (1, 1)]);
- test_collapse_mapping_aux(
- &["/facet/a", "/facet/a2", "/facet/b"],
- &["/facet"],
- &[(0, 1), (1, 1), (2, 1)],
- );
- test_collapse_mapping_aux(&["/facet/a", "/facet/a/2"], &["/facet"], &[(0, 1), (0, 1)]);
- test_collapse_mapping_aux(
- &["/facet/a", "/facet/a/2", "/facet/b"],
- &["/facet"],
- &[(0, 1), (0, 1), (2, 1)],
- );
- }
-
- fn test_compress_mapping_aux(
- collapsed_mapping: &[(u64, usize)],
- expected_compressed_collapsed_mapping: &[usize],
- expected_unique_facet_ords: &[(u64, usize)],
- ) {
- let (compressed_collapsed_mapping, unique_facet_ords) = compress_mapping(collapsed_mapping);
- assert_eq!(
- compressed_collapsed_mapping,
- expected_compressed_collapsed_mapping
- );
- assert_eq!(unique_facet_ords, expected_unique_facet_ords);
- }
-
- #[test]
- fn test_compress_mapping() {
- test_compress_mapping_aux(&[], &[], &[]);
- test_compress_mapping_aux(&[(1, 2)], &[0], &[(1, 2)]);
- test_compress_mapping_aux(&[(1, 2), (1, 2)], &[0, 0], &[(1, 2)]);
- test_compress_mapping_aux(
- &[(1, 2), (5, 2), (5, 2), (6, 3), (8, 3)],
- &[0, 1, 1, 2, 3],
- &[(1, 2), (5, 2), (6, 3), (8, 3)],
- );
- }
-
- #[test]
- fn test_facet_collector_simple() {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(facet_field=>Facet::from("/facet/a")))
- .unwrap();
- index_writer
- .add_document(doc!(facet_field=>Facet::from("/facet/b")))
- .unwrap();
- index_writer
- .add_document(doc!(facet_field=>Facet::from("/facet/b")))
- .unwrap();
- index_writer
- .add_document(doc!(facet_field=>Facet::from("/facet/c")))
- .unwrap();
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet("/facet");
- let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector).unwrap();
- let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 1);
- assert_eq!(facets, vec![(&Facet::from("/facet/b"), 2)]);
- }
-
- #[test]
- fn test_facet_collector_drilldown() {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- let num_facets: usize = 3 * 4 * 5;
- let facets: Vec<Facet> = (0..num_facets)
- .map(|mut n| {
- let top = n % 3;
- n /= 3;
- let mid = n % 4;
- n /= 4;
- let leaf = n % 5;
- Facet::from(&format!("/top{top}/mid{mid}/leaf{leaf}"))
- })
- .collect();
- for i in 0..num_facets * 10 {
- let mut doc = TantivyDocument::new();
- doc.add_facet(facet_field, facets[i % num_facets].clone());
- index_writer.add_document(doc).unwrap();
- }
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet(Facet::from("/top1"));
- let counts = searcher.search(&AllQuery, &facet_collector).unwrap();
-
- {
- let facets: Vec<(String, u64)> = counts
- .get("/top1")
- .map(|(facet, count)| (facet.to_string(), count))
- .collect();
- assert_eq!(
- facets,
- [
- ("/top1/mid0", 50),
- ("/top1/mid1", 50),
- ("/top1/mid2", 50),
- ("/top1/mid3", 50),
- ]
- .iter()
- .map(|&(facet_str, count)| (String::from(facet_str), count))
- .collect::<Vec<_>>()
- );
- }
- }
-
- #[test]
- #[should_panic(
- expected = "Tried to add a facet which is a descendant of an already added facet."
- )]
- fn test_misused_facet_collector() {
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet(Facet::from("/country"));
- facet_collector.add_facet(Facet::from("/country/europe"));
- }
-
- #[test]
- fn test_doc_unsorted_multifacet() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facets", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer = index.writer_for_tests()?;
- index_writer.add_document(doc!(
- facet_field => Facet::from_text(&"/subjects/A/a").unwrap(),
- facet_field => Facet::from_text(&"/subjects/B/a").unwrap(),
- facet_field => Facet::from_text(&"/subjects/A/b").unwrap(),
- facet_field => Facet::from_text(&"/subjects/B/b").unwrap(),
- ))?;
- index_writer.commit()?;
- let reader = index.reader()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 1);
- let mut facet_collector = FacetCollector::for_field("facets");
- facet_collector.add_facet("/subjects");
- let counts = searcher.search(&AllQuery, &facet_collector)?;
- let facets: Vec<(&Facet, u64)> = counts.get("/subjects").collect();
- assert_eq!(facets[0].1, 1);
- Ok(())
- }
-
- #[test]
- fn test_doc_search_by_facet() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer = index.writer_for_tests()?;
- index_writer.add_document(doc!(
- facet_field => Facet::from_text(&"/A/A").unwrap(),
- ))?;
- index_writer.add_document(doc!(
- facet_field => Facet::from_text(&"/A/B").unwrap(),
- ))?;
- index_writer.add_document(doc!(
- facet_field => Facet::from_text(&"/A/C/A").unwrap(),
- ))?;
- index_writer.add_document(doc!(
- facet_field => Facet::from_text(&"/D/C/A").unwrap(),
- ))?;
- index_writer.commit()?;
- let reader = index.reader()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 4);
-
- let count_facet = |facet_str: &str| {
- let term = Term::from_facet(facet_field, &Facet::from_text(facet_str).unwrap());
- searcher
- .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)
- .unwrap()
- };
-
- assert_eq!(count_facet("/"), 4);
- assert_eq!(count_facet("/A"), 3);
- assert_eq!(count_facet("/A/B"), 1);
- assert_eq!(count_facet("/A/C"), 1);
- assert_eq!(count_facet("/A/C/A"), 1);
- assert_eq!(count_facet("/C/A"), 0);
-
- let query_parser = QueryParser::for_index(&index, vec![]);
- {
- let query = query_parser.parse_query("facet:/A/B")?;
- assert_eq!(1, searcher.search(&query, &Count).unwrap());
- }
- {
- let query = query_parser.parse_query("facet:/A")?;
- assert_eq!(3, searcher.search(&query, &Count)?);
- }
- Ok(())
- }
-
- #[test]
- fn test_non_used_facet_collector() {
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet(Facet::from("/country"));
- facet_collector.add_facet(Facet::from("/countryeurope"));
- }
-
- #[test]
- fn test_facet_collector_topk() {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- let uniform = Uniform::new_inclusive(1, 100_000);
- let mut docs: Vec<TantivyDocument> =
- vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
- .into_iter()
- .flat_map(|(c, count)| {
- let facet = Facet::from(&format!("/facet/{c}"));
- let doc = doc!(facet_field => facet);
- iter::repeat(doc).take(count)
- })
- .map(|mut doc| {
- doc.add_facet(
- facet_field,
- &format!("/facet/{}", thread_rng().sample(uniform)),
- );
- doc
- })
- .collect();
- docs[..].shuffle(&mut thread_rng());
-
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- for doc in docs {
- index_writer.add_document(doc).unwrap();
- }
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
-
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet("/facet");
- let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector).unwrap();
-
- {
- let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 3);
- assert_eq!(
- facets,
- vec![
- (&Facet::from("/facet/b"), 100),
- (&Facet::from("/facet/e"), 21),
- (&Facet::from("/facet/d"), 12),
- ]
- );
- }
- }
-
- #[test]
- fn test_facet_collector_topk_tie_break() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]
- .into_iter()
- .flat_map(|(c, count)| {
- let facet = Facet::from(&format!("/facet/{c}"));
- let doc = doc!(facet_field => facet);
- iter::repeat(doc).take(count)
- })
- .collect();
-
- let mut index_writer = index.writer_for_tests()?;
- for doc in docs {
- index_writer.add_document(doc)?;
- }
- index_writer.commit()?;
-
- let searcher = index.reader()?.searcher();
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet("/facet");
- let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector)?;
-
- let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 2);
- assert_eq!(
- facets,
- vec![(&Facet::from("/facet/c"), 4), (&Facet::from("/facet/a"), 2)]
- );
- Ok(())
- }
-
- #[test]
- fn is_child_facet() {
- assert!(super::is_child_facet(&b"foo"[..], &b"foo\0bar"[..]));
- assert!(super::is_child_facet(&b""[..], &b"foo\0bar"[..]));
- assert!(super::is_child_facet(&b""[..], &b"foo"[..]));
- assert!(!super::is_child_facet(&b"foo\0bar"[..], &b"foo"[..]));
- assert!(!super::is_child_facet(&b"foo"[..], &b"foobar\0baz"[..]));
- }
-}
diff --git a/crates/tantivy/src/collector/filter_collector_wrapper.rs b/crates/tantivy/src/collector/filter_collector_wrapper.rs
index df121222..e68ffe37 100644
--- a/crates/tantivy/src/collector/filter_collector_wrapper.rs
+++ b/crates/tantivy/src/collector/filter_collector_wrapper.rs
@@ -112,7 +112,7 @@ where
#[inline]
fn accept_document(&self, doc_id: DocId) -> bool {
if let Some(column) = &self.column_opt {
- for val in column.values_for_doc(doc_id) {
+ if let Some(val) = column.first(doc_id) {
if (self.predicate)(val) {
return true;
}
diff --git a/crates/tantivy/src/collector/histogram_collector.rs b/crates/tantivy/src/collector/histogram_collector.rs
deleted file mode 100644
index 2d99cca4..00000000
--- a/crates/tantivy/src/collector/histogram_collector.rs
+++ /dev/null
@@ -1,305 +0,0 @@
-use std::sync::Arc;
-
-use crate::columnar::ColumnValues;
-use fastdivide::DividerU64;
-
-use crate::collector::{Collector, SegmentCollector};
-use crate::fastfield::{FastFieldNotAvailableError, FastValue};
-use crate::schema::Type;
-use crate::{DocId, Score};
-
-/// Histogram builds an histogram of the values of a fastfield for the
-/// collected DocSet.
-///
-/// At construction, it is given parameters that define a partition of an interval
-/// [min_val, max_val) into N buckets with the same width.
-/// The ith bucket is then defined by `[min_val + i * bucket_width, min_val + (i+1) * bucket_width)`
-///
-/// An histogram is then defined as a `Vec<u64>` of length `num_buckets`, that contains a count of
-/// documents for each value bucket.
-///
-/// See also [`HistogramCollector::new()`].
-///
-/// # Warning
-///
-/// f64 fields are not supported.
-#[derive(Clone)]
-pub struct HistogramCollector {
- min_value: u64,
- num_buckets: usize,
- divider: DividerU64,
- field: String,
-}
-
-impl HistogramCollector {
- /// Builds a new HistogramCollector.
- ///
- /// The scale/range of the histogram is not dynamic. It is required to
- /// define it by supplying following parameter:
- /// - `min_value`: the minimum value that can be recorded in the histogram.
- /// - `bucket_width`: the length of the interval that is associated with each buckets.
- /// - `num_buckets`: The overall number of buckets.
- ///
- /// Together, this parameters define a partition of `[min_value, min_value + num_buckets *
- /// bucket_width)` into `num_buckets` intervals of width bucket that we call `bucket`.
- ///
- /// # Disclaimer
- /// This function panics if the field given is of type f64.
- pub fn new<TFastValue: FastValue>(
- field: String,
- min_value: TFastValue,
- bucket_width: u64,
- num_buckets: usize,
- ) -> HistogramCollector {
- let fast_type = TFastValue::to_type();
- assert!(fast_type == Type::U64 || fast_type == Type::I64 || fast_type == Type::Date);
- HistogramCollector {
- min_value: min_value.to_u64(),
- num_buckets,
- field,
- divider: DividerU64::divide_by(bucket_width),
- }
- }
-}
-
-struct HistogramComputer {
- counts: Vec<u64>,
- min_value: u64,
- divider: DividerU64,
-}
-
-impl HistogramComputer {
- #[inline]
- pub(crate) fn add_value(&mut self, value: u64) {
- if value < self.min_value {
- return;
- }
- let delta = value - self.min_value;
- let bucket_id: usize = self.divider.divide(delta) as usize;
- if bucket_id < self.counts.len() {
- self.counts[bucket_id] += 1;
- }
- }
-
- fn harvest(self) -> Vec<u64> {
- self.counts
- }
-}
-pub struct SegmentHistogramCollector {
- histogram_computer: HistogramComputer,
- column_u64: Arc<dyn ColumnValues<u64>>,
-}
-
-impl SegmentCollector for SegmentHistogramCollector {
- type Fruit = Vec<u64>;
-
- fn collect(&mut self, doc: DocId, _score: Score) {
- let value = self.column_u64.get_val(doc);
- self.histogram_computer.add_value(value);
- }
-
- fn harvest(self) -> Self::Fruit {
- self.histogram_computer.harvest()
- }
-}
-
-impl Collector for HistogramCollector {
- type Fruit = Vec<u64>;
- type Child = SegmentHistogramCollector;
-
- fn for_segment(
- &self,
- _segment_local_id: crate::SegmentOrdinal,
- segment: &crate::SegmentReader,
- ) -> crate::Result<Self::Child> {
- let column_opt = segment.fast_fields().u64_lenient(&self.field)?;
- let (column, _column_type) = column_opt.ok_or_else(|| FastFieldNotAvailableError {
- field_name: self.field.clone(),
- })?;
- let column_u64 = column.first_or_default_col(0u64);
- Ok(SegmentHistogramCollector {
- histogram_computer: HistogramComputer {
- counts: vec![0; self.num_buckets],
- min_value: self.min_value,
- divider: self.divider,
- },
- column_u64,
- })
- }
-
- fn requires_scoring(&self) -> bool {
- false
- }
-
- fn merge_fruits(&self, child_histograms: Vec<Vec<u64>>) -> crate::Result<Vec<u64>> {
- Ok(add_vecs(child_histograms, self.num_buckets))
- }
-}
-
-pub fn add_arrays_into(acc: &mut [u64], add: &[u64]) {
- assert_eq!(acc.len(), add.len());
- for (dest_bucket, bucket_count) in acc.iter_mut().zip(add) {
- *dest_bucket += bucket_count;
- }
-}
-
-fn add_vecs(mut vals_list: Vec<Vec<u64>>, len: usize) -> Vec<u64> {
- let mut acc = vals_list.pop().unwrap_or_else(|| vec![0u64; len]);
- assert_eq!(acc.len(), len);
- for vals in vals_list {
- add_arrays_into(&mut acc, &vals);
- }
- acc
-}
-
-#[cfg(test)]
-mod tests {
- use fastdivide::DividerU64;
- use query::AllQuery;
-
- use super::{add_vecs, HistogramCollector, HistogramComputer};
- use crate::schema::{Schema, FAST};
- use crate::time::{Date, Month};
- use crate::{query, DateTime, Index};
-
- #[test]
- fn test_add_histograms_simple() {
- assert_eq!(
- add_vecs(vec![vec![1, 0, 3], vec![11, 2, 3], vec![0, 0, 1]], 3),
- vec![12, 2, 7]
- )
- }
-
- #[test]
- fn test_add_histograms_empty() {
- assert_eq!(add_vecs(vec![], 3), vec![0, 0, 0])
- }
-
- #[test]
- fn test_histogram_builder_simple() {
- // [1..3)
- // [3..5)
- // ..
- // [9..11)
- let mut histogram_computer = HistogramComputer {
- counts: vec![0; 5],
- min_value: 1,
- divider: DividerU64::divide_by(2),
- };
- histogram_computer.add_value(1);
- histogram_computer.add_value(7);
- assert_eq!(histogram_computer.harvest(), vec![1, 0, 0, 1, 0]);
- }
-
- #[test]
- fn test_histogram_too_low_is_ignored() {
- let mut histogram_computer = HistogramComputer {
- counts: vec![0; 5],
- min_value: 2,
- divider: DividerU64::divide_by(2),
- };
- histogram_computer.add_value(0);
- assert_eq!(histogram_computer.harvest(), vec![0, 0, 0, 0, 0]);
- }
-
- #[test]
- fn test_histogram_too_high_is_ignored() {
- let mut histogram_computer = HistogramComputer {
- counts: vec![0u64; 5],
- min_value: 0,
- divider: DividerU64::divide_by(2),
- };
- histogram_computer.add_value(10);
- assert_eq!(histogram_computer.harvest(), vec![0, 0, 0, 0, 0]);
- }
- #[test]
- fn test_no_segments() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- schema_builder.add_u64_field("val_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let all_query = AllQuery;
- let histogram_collector = HistogramCollector::new("val_field".to_string(), 0u64, 2, 5);
- let histogram = searcher.search(&all_query, &histogram_collector)?;
- assert_eq!(histogram, vec![0; 5]);
- Ok(())
- }
-
- #[test]
- fn test_histogram_i64() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let val_field = schema_builder.add_i64_field("val_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer = index.writer_for_tests()?;
- writer.add_document(doc!(val_field=>12i64))?;
- writer.add_document(doc!(val_field=>-30i64))?;
- writer.add_document(doc!(val_field=>-12i64))?;
- writer.add_document(doc!(val_field=>-10i64))?;
- writer.commit()?;
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let all_query = AllQuery;
- let histogram_collector =
- HistogramCollector::new("val_field".to_string(), -20i64, 10u64, 4);
- let histogram = searcher.search(&all_query, &histogram_collector)?;
- assert_eq!(histogram, vec![1, 1, 0, 1]);
- Ok(())
- }
-
- #[test]
- fn test_histogram_merge() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let val_field = schema_builder.add_i64_field("val_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer = index.writer_for_tests()?;
- writer.add_document(doc!(val_field=>12i64))?;
- writer.commit()?;
- writer.add_document(doc!(val_field=>-30i64))?;
- writer.commit()?;
- writer.add_document(doc!(val_field=>-12i64))?;
- writer.commit()?;
- writer.add_document(doc!(val_field=>-10i64))?;
- writer.commit()?;
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let all_query = AllQuery;
- let histogram_collector =
- HistogramCollector::new("val_field".to_string(), -20i64, 10u64, 4);
- let histogram = searcher.search(&all_query, &histogram_collector)?;
- assert_eq!(histogram, vec![1, 1, 0, 1]);
- Ok(())
- }
-
- #[test]
- fn test_histogram_dates() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let date_field = schema_builder.add_date_field("date_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer = index.writer_for_tests()?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
- writer.add_document(
- doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1986, Month::March, 9)?.with_hms(0, 0, 0)?)),
- )?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;
- writer.commit()?;
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let all_query = AllQuery;
- let week_histogram_collector = HistogramCollector::new(
- "date_field".to_string(),
- DateTime::from_primitive(
- Date::from_calendar_date(1980, Month::January, 1)?.with_hms(0, 0, 0)?,
- ),
- 3_600_000_000_000 * 24 * 365, // it is just for a unit test... sorry leap years.
- 10,
- );
- let week_histogram = searcher.search(&all_query, &week_histogram_collector)?;
- assert_eq!(week_histogram, vec![0, 0, 1, 1, 0, 0, 1, 0, 0, 0]);
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/collector/mod.rs b/crates/tantivy/src/collector/mod.rs
index 1325f43e..ca5b1962 100644
--- a/crates/tantivy/src/collector/mod.rs
+++ b/crates/tantivy/src/collector/mod.rs
@@ -59,9 +59,6 @@ use crate::{DocId, Score, SegmentOrdinal, SegmentReader};
mod count_collector;
pub use self::count_collector::Count;
-mod histogram_collector;
-pub use histogram_collector::HistogramCollector;
-
mod multi_collector;
pub use self::multi_collector::{FruitHandle, MultiCollector, MultiFruit};
@@ -76,8 +73,6 @@ pub use self::custom_score_top_collector::{CustomScorer, CustomSegmentScorer};
mod tweak_score_top_collector;
pub use self::tweak_score_top_collector::{ScoreSegmentTweaker, ScoreTweaker};
-mod facet_collector;
-pub use self::facet_collector::{FacetCollector, FacetCounts};
use crate::query::Weight;
mod docset_collector;
diff --git a/crates/tantivy/src/collector/tests.rs b/crates/tantivy/src/collector/tests.rs
index 4b2c9c0b..ee3aa8bd 100644
--- a/crates/tantivy/src/collector/tests.rs
+++ b/crates/tantivy/src/collector/tests.rs
@@ -202,7 +202,7 @@ impl SegmentCollector for FastFieldSegmentCollector {
type Fruit = Vec<u64>;
fn collect(&mut self, doc: DocId, _score: Score) {
- self.vals.extend(self.reader.values_for_doc(doc));
+ self.vals.extend(self.reader.first(doc));
}
fn harvest(self) -> Vec<u64> {
diff --git a/crates/tantivy/src/collector/top_score_collector.rs b/crates/tantivy/src/collector/top_score_collector.rs
index a3eb05c1..632f4194 100644
--- a/crates/tantivy/src/collector/top_score_collector.rs
+++ b/crates/tantivy/src/collector/top_score_collector.rs
@@ -143,12 +143,8 @@ impl CustomScorer<u64> for ScorerByField {
sort_column_opt.ok_or_else(|| FastFieldNotAvailableError {
field_name: self.field.clone(),
})?;
- let mut default_value = 0u64;
- if self.order.is_asc() {
- default_value = u64::MAX;
- }
Ok(ScorerByFastFieldReader {
- sort_column: sort_column.first_or_default_col(default_value),
+ sort_column: sort_column.values,
order: self.order.clone(),
})
}
@@ -1083,6 +1079,7 @@ mod tests {
index_writer
.add_document(doc!(
title => "empty beer",
+ size => 18446744073709551615u64,
))
.unwrap();
});
diff --git a/crates/tantivy/src/columnar/block_accessor.rs b/crates/tantivy/src/columnar/block_accessor.rs
index f5687cc5..0807b2df 100644
--- a/crates/tantivy/src/columnar/block_accessor.rs
+++ b/crates/tantivy/src/columnar/block_accessor.rs
@@ -1,13 +1,8 @@
-use std::cmp::Ordering;
-
-use super::{Column, DocId, RowId};
+use super::{Column, DocId};
#[derive(Debug, Default, Clone)]
pub struct ColumnBlockAccessor<T> {
val_cache: Vec<T>,
- docid_cache: Vec<DocId>,
- missing_docids_cache: Vec<DocId>,
- row_id_cache: Vec<RowId>,
}
impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>
@@ -15,39 +10,12 @@ impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>
{
#[inline]
pub fn fetch_block<'a>(&'a mut self, docs: &'a [u32], accessor: &Column<T>) {
- if accessor.index.get_cardinality().is_full() {
- self.val_cache.resize(docs.len(), T::default());
- accessor.values.get_vals(docs, &mut self.val_cache);
- } else {
- self.docid_cache.clear();
- self.row_id_cache.clear();
- accessor.row_ids_for_docs(docs, &mut self.docid_cache, &mut self.row_id_cache);
- self.val_cache.resize(self.row_id_cache.len(), T::default());
- accessor
- .values
- .get_vals(&self.row_id_cache, &mut self.val_cache);
- }
+ self.val_cache.resize(docs.len(), T::default());
+ accessor.values.get_vals(docs, &mut self.val_cache);
}
#[inline]
- pub fn fetch_block_with_missing(&mut self, docs: &[u32], accessor: &Column<T>, missing: T) {
+ pub fn fetch_block_with_missing(&mut self, docs: &[u32], accessor: &Column<T>) {
self.fetch_block(docs, accessor);
- // no missing values
- if accessor.index.get_cardinality().is_full() {
- return;
- }
-
- // We can compare docid_cache length with docs to find missing docs
- // For multi value columns we can't rely on the length and always need to scan
- if accessor.index.get_cardinality().is_multivalue() || docs.len() != self.docid_cache.len()
- {
- self.missing_docids_cache.clear();
- find_missing_docs(docs, &self.docid_cache, |doc| {
- self.missing_docids_cache.push(doc);
- self.val_cache.push(missing);
- });
- self.docid_cache
- .extend_from_slice(&self.missing_docids_cache);
- }
}
#[inline]
@@ -62,99 +30,7 @@ impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>
///
/// The docs is used if the column is full (each docs has exactly one value), otherwise the
/// internal docid vec is used for the iterator, which e.g. may contain duplicate docs.
- pub fn iter_docid_vals<'a>(
- &'a self,
- docs: &'a [u32],
- accessor: &Column<T>,
- ) -> impl Iterator<Item = (DocId, T)> + '_ {
- if accessor.index.get_cardinality().is_full() {
- docs.iter().cloned().zip(self.val_cache.iter().cloned())
- } else {
- self.docid_cache
- .iter()
- .cloned()
- .zip(self.val_cache.iter().cloned())
- }
- }
-}
-
-/// Given two sorted lists of docids `docs` and `hits`, hits is a subset of `docs`.
-/// Return all docs that are not in `hits`.
-fn find_missing_docs<F>(docs: &[u32], hits: &[u32], mut callback: F)
-where
- F: FnMut(u32),
-{
- let mut docs_iter = docs.iter();
- let mut hits_iter = hits.iter();
-
- let mut doc = docs_iter.next();
- let mut hit = hits_iter.next();
-
- while let (Some(&current_doc), Some(&current_hit)) = (doc, hit) {
- match current_doc.cmp(&current_hit) {
- Ordering::Less => {
- callback(current_doc);
- doc = docs_iter.next();
- }
- Ordering::Equal => {
- doc = docs_iter.next();
- hit = hits_iter.next();
- }
- Ordering::Greater => {
- hit = hits_iter.next();
- }
- }
- }
-
- while let Some(&current_doc) = doc {
- callback(current_doc);
- doc = docs_iter.next();
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_find_missing_docs() {
- let docs: Vec<u32> = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
- let hits: Vec<u32> = vec![2, 4, 6, 8, 10];
-
- let mut missing_docs: Vec<u32> = Vec::new();
-
- find_missing_docs(&docs, &hits, |missing_doc| {
- missing_docs.push(missing_doc);
- });
-
- assert_eq!(missing_docs, vec![1, 3, 5, 7, 9]);
- }
-
- #[test]
- fn test_find_missing_docs_empty() {
- let docs: Vec<u32> = Vec::new();
- let hits: Vec<u32> = vec![2, 4, 6, 8, 10];
-
- let mut missing_docs: Vec<u32> = Vec::new();
-
- find_missing_docs(&docs, &hits, |missing_doc| {
- missing_docs.push(missing_doc);
- });
-
- assert_eq!(missing_docs, Vec::<u32>::new());
- }
-
- #[test]
- fn test_find_missing_docs_all_missing() {
- let docs: Vec<u32> = vec![1, 2, 3, 4, 5];
- let hits: Vec<u32> = Vec::new();
-
- let mut missing_docs: Vec<u32> = Vec::new();
-
- find_missing_docs(&docs, &hits, |missing_doc| {
- missing_docs.push(missing_doc);
- });
-
- assert_eq!(missing_docs, vec![1, 2, 3, 4, 5]);
+ pub fn iter_docid_vals<'a>(&'a self, docs: &'a [u32]) -> impl Iterator<Item = (DocId, T)> + '_ {
+ docs.iter().cloned().zip(self.val_cache.iter().cloned())
}
}
diff --git a/crates/tantivy/src/columnar/column/dictionary_encoded.rs b/crates/tantivy/src/columnar/column/dictionary_encoded.rs
index 28847329..bbe4c1d1 100644
--- a/crates/tantivy/src/columnar/column/dictionary_encoded.rs
+++ b/crates/tantivy/src/columnar/column/dictionary_encoded.rs
@@ -1,4 +1,3 @@
-use std::ops::Deref;
use std::sync::Arc;
use std::{fmt, io};
@@ -30,13 +29,6 @@ impl fmt::Debug for BytesColumn {
}
impl BytesColumn {
- pub fn empty(num_docs: u32) -> BytesColumn {
- BytesColumn {
- dictionary: Arc::new(Dictionary::empty()),
- term_ord_column: Column::build_empty_column(num_docs),
- }
- }
-
/// Fills the given `output` buffer with the term associated to the ordinal `ord`.
///
/// Returns `false` if the term does not exist (e.g. `term_ord` is greater or equal to the
@@ -51,7 +43,7 @@ impl BytesColumn {
}
pub fn term_ords(&self, row_id: RowId) -> impl Iterator<Item = u64> + '_ {
- self.term_ord_column.values_for_doc(row_id)
+ self.term_ord_column.first(row_id).into_iter()
}
/// Returns the column of ordinals
@@ -67,55 +59,3 @@ impl BytesColumn {
self.dictionary.as_ref()
}
}
-
-#[derive(Clone)]
-pub struct StrColumn(BytesColumn);
-
-impl fmt::Debug for StrColumn {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self.term_ord_column)
- }
-}
-
-impl From<StrColumn> for BytesColumn {
- fn from(str_column: StrColumn) -> BytesColumn {
- str_column.0
- }
-}
-
-impl StrColumn {
- pub fn wrap(bytes_column: BytesColumn) -> StrColumn {
- StrColumn(bytes_column)
- }
-
- pub fn dictionary(&self) -> &Dictionary<VoidSSTable> {
- self.0.dictionary.as_ref()
- }
-
- /// Fills the buffer
- pub fn ord_to_str(&self, term_ord: u64, output: &mut String) -> io::Result<bool> {
- unsafe {
- let buf = output.as_mut_vec();
- if !self.0.dictionary.ord_to_term(term_ord, buf)? {
- return Ok(false);
- }
- // TODO consider remove checks if it hurts performance.
- if std::str::from_utf8(buf.as_slice()).is_err() {
- buf.clear();
- return Err(io::Error::new(
- io::ErrorKind::InvalidData,
- "Not valid utf-8",
- ));
- }
- }
- Ok(true)
- }
-}
-
-impl Deref for StrColumn {
- type Target = BytesColumn;
-
- fn deref(&self) -> &Self::Target {
- &self.0
- }
-}
diff --git a/crates/tantivy/src/columnar/column/mod.rs b/crates/tantivy/src/columnar/column/mod.rs
index d4158dd4..1a6a32d2 100644
--- a/crates/tantivy/src/columnar/column/mod.rs
+++ b/crates/tantivy/src/columnar/column/mod.rs
@@ -7,16 +7,13 @@ use std::ops::{Range, RangeInclusive};
use std::sync::Arc;
use crate::common::BinarySerializable;
-pub use dictionary_encoded::{BytesColumn, StrColumn};
-pub use serialize::{
- open_column_bytes, open_column_str, open_column_u128, open_column_u128_as_compact_u64,
- open_column_u64, serialize_column_mappable_to_u128, serialize_column_mappable_to_u64,
-};
+pub use dictionary_encoded::BytesColumn;
+pub use serialize::{open_column_bytes, open_column_u64, serialize_column_mappable_to_u64};
-use super::column_index::{ColumnIndex, Set};
+use super::column_index::ColumnIndex;
use super::column_values::monotonic_mapping::StrictlyMonotonicMappingToInternal;
use super::column_values::{monotonic_map_column, ColumnValues};
-use super::{Cardinality, DocId, EmptyColumnValues, MonotonicallyMappableToU64, RowId};
+use super::{Cardinality, DocId, MonotonicallyMappableToU64, RowId};
#[derive(Clone)]
pub struct Column<T = u64> {
@@ -28,21 +25,12 @@ impl<T: Debug + PartialOrd + Send + Sync + Copy + 'static> Debug for Column<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let num_docs = self.num_docs();
let entries = (0..num_docs)
- .map(|i| (i, self.values_for_doc(i).collect::<Vec<_>>()))
- .filter(|(_, vals)| !vals.is_empty());
+ .map(|i| (i, self.first(i)))
+ .collect::<Vec<_>>();
f.debug_map().entries(entries).finish()
}
}
-impl<T: PartialOrd + Default> Column<T> {
- pub fn build_empty_column(num_docs: u32) -> Column<T> {
- Column {
- index: ColumnIndex::Empty { num_docs },
- values: Arc::new(EmptyColumnValues),
- }
- }
-}
-
impl<T: MonotonicallyMappableToU64> Column<T> {
pub fn to_u64_monotonic(self) -> Column<u64> {
let values = Arc::new(monotonic_map_column(
@@ -57,21 +45,9 @@ impl<T: MonotonicallyMappableToU64> Column<T> {
}
impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
- #[inline]
- pub fn get_cardinality(&self) -> Cardinality {
- self.index.get_cardinality()
- }
-
pub fn num_docs(&self) -> RowId {
match &self.index {
- ColumnIndex::Empty { num_docs } => *num_docs,
ColumnIndex::Full => self.values.num_vals(),
- ColumnIndex::Optional(optional_index) => optional_index.num_docs(),
- ColumnIndex::Multivalued(col_index) => {
- // The multivalued index contains all value start row_id,
- // and one extra value at the end with the overall number of rows.
- col_index.num_docs()
- }
}
}
@@ -85,31 +61,14 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
#[inline]
pub fn first(&self, row_id: RowId) -> Option<T> {
- self.values_for_doc(row_id).next()
+ Some(self.values.get_val(row_id))
}
/// Load the first value for each docid in the provided slice.
#[inline]
pub fn first_vals(&self, docids: &[DocId], output: &mut [Option<T>]) {
match &self.index {
- ColumnIndex::Empty { .. } => {}
ColumnIndex::Full => self.values.get_vals_opt(docids, output),
- ColumnIndex::Optional(optional_index) => {
- for (i, docid) in docids.iter().enumerate() {
- output[i] = optional_index
- .rank_if_exists(*docid)
- .map(|rowid| self.values.get_val(rowid));
- }
- }
- ColumnIndex::Multivalued(multivalued_index) => {
- for (i, docid) in docids.iter().enumerate() {
- let range = multivalued_index.range(*docid);
- let is_empty = range.start == range.end;
- if !is_empty {
- output[i] = Some(self.values.get_val(range.start));
- }
- }
- }
}
}
@@ -130,12 +89,6 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
self.index.docids_to_rowids(doc_ids, doc_ids_out, row_ids)
}
- pub fn values_for_doc(&self, doc_id: DocId) -> impl Iterator<Item = T> + '_ {
- self.index
- .value_row_ids(doc_id)
- .map(|value_row_id: RowId| self.values.get_val(value_row_id))
- }
-
/// Get the docids of values which are in the provided value range.
#[inline]
pub fn get_docids_for_value_range(
@@ -152,25 +105,6 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
// Load rows
self.values
.get_row_ids_for_value_range(value_range, rowid_range, doc_ids);
- // Convert rows to docids
- self.index
- .select_batch_in_place(selected_docid_range.start, doc_ids);
- }
-
- /// Fills the output vector with the (possibly multiple values that are associated_with
- /// `row_id`.
- ///
- /// This method clears the `output` vector.
- pub fn fill_vals(&self, row_id: RowId, output: &mut Vec<T>) {
- output.clear();
- output.extend(self.values_for_doc(row_id));
- }
-
- pub fn first_or_default_col(self, default_value: T) -> Arc<dyn ColumnValues<T>> {
- Arc::new(FirstValueWithDefault {
- column: self,
- default_value,
- })
}
}
@@ -185,35 +119,3 @@ impl BinarySerializable for Cardinality {
Ok(cardinality)
}
}
-
-// TODO simplify or optimize
-struct FirstValueWithDefault<T: Copy> {
- column: Column<T>,
- default_value: T,
-}
-
-impl<T: PartialOrd + Debug + Send + Sync + Copy + 'static> ColumnValues<T>
- for FirstValueWithDefault<T>
-{
- #[inline(always)]
- fn get_val(&self, idx: u32) -> T {
- self.column.first(idx).unwrap_or(self.default_value)
- }
-
- fn min_value(&self) -> T {
- self.column.values.min_value()
- }
-
- fn max_value(&self) -> T {
- self.column.values.max_value()
- }
-
- fn num_vals(&self) -> u32 {
- match &self.column.index {
- ColumnIndex::Empty { .. } => 0u32,
- ColumnIndex::Full => self.column.values.num_vals(),
- ColumnIndex::Optional(optional_idx) => optional_idx.num_docs(),
- ColumnIndex::Multivalued(multivalue_idx) => multivalue_idx.num_docs(),
- }
- }
-}
diff --git a/crates/tantivy/src/columnar/column/serialize.rs b/crates/tantivy/src/columnar/column/serialize.rs
index eca3fb08..211432d7 100644
--- a/crates/tantivy/src/columnar/column/serialize.rs
+++ b/crates/tantivy/src/columnar/column/serialize.rs
@@ -5,28 +5,16 @@ use std::sync::Arc;
use crate::common::OwnedBytes;
use crate::sstable::Dictionary;
-use super::StrColumn;
use crate::columnar::column::{BytesColumn, Column};
use crate::columnar::column_index::{serialize_column_index, SerializableColumnIndex};
use crate::columnar::column_values::{
- load_u64_based_column_values, serialize_column_values_u128, serialize_u64_based_column_values,
- CodecType, MonotonicallyMappableToU128, MonotonicallyMappableToU64,
+ load_u64_based_column_values, serialize_u64_based_column_values, CodecType,
+ MonotonicallyMappableToU64,
};
use crate::columnar::iterable::Iterable;
-pub fn serialize_column_mappable_to_u128<T: MonotonicallyMappableToU128>(
- column_index: SerializableColumnIndex<'_>,
- iterable: &dyn Iterable<T>,
- output: &mut impl Write,
-) -> io::Result<()> {
- let column_index_num_bytes = serialize_column_index(column_index, output)?;
- serialize_column_values_u128(iterable, output)?;
- output.write_all(&column_index_num_bytes.to_le_bytes())?;
- Ok(())
-}
-
pub fn serialize_column_mappable_to_u64<T: MonotonicallyMappableToU64>(
- column_index: SerializableColumnIndex<'_>,
+ column_index: SerializableColumnIndex,
column_values: &impl Iterable<T>,
output: &mut impl Write,
) -> io::Result<()> {
@@ -57,46 +45,6 @@ pub fn open_column_u64<T: MonotonicallyMappableToU64>(bytes: OwnedBytes) -> io::
})
}
-pub fn open_column_u128<T: MonotonicallyMappableToU128>(
- bytes: OwnedBytes,
-) -> io::Result<Column<T>> {
- let (body, column_index_num_bytes_payload) = bytes.rsplit(4);
- let column_index_num_bytes = u32::from_le_bytes(
- column_index_num_bytes_payload
- .as_slice()
- .try_into()
- .unwrap(),
- );
- let (column_index_data, column_values_data) = body.split(column_index_num_bytes as usize);
- let column_index = crate::columnar::column_index::open_column_index(column_index_data)?;
- let column_values = crate::columnar::column_values::open_u128_mapped(column_values_data)?;
- Ok(Column {
- index: column_index,
- values: column_values,
- })
-}
-
-/// Open the column as u64.
-///
-/// See [`open_u128_as_compact_u64`] for more details.
-pub fn open_column_u128_as_compact_u64(bytes: OwnedBytes) -> io::Result<Column<u64>> {
- let (body, column_index_num_bytes_payload) = bytes.rsplit(4);
- let column_index_num_bytes = u32::from_le_bytes(
- column_index_num_bytes_payload
- .as_slice()
- .try_into()
- .unwrap(),
- );
- let (column_index_data, column_values_data) = body.split(column_index_num_bytes as usize);
- let column_index = crate::columnar::column_index::open_column_index(column_index_data)?;
- let column_values =
- crate::columnar::column_values::open_u128_as_compact_u64(column_values_data)?;
- Ok(Column {
- index: column_index,
- values: column_values,
- })
-}
-
pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {
let (body, dictionary_len_bytes) = data.rsplit(8);
let dictionary_len = u64::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());
@@ -108,8 +56,3 @@ pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {
term_ord_column,
})
}
-
-pub fn open_column_str(data: OwnedBytes) -> io::Result<StrColumn> {
- let bytes_column = open_column_bytes(data)?;
- Ok(StrColumn::wrap(bytes_column))
-}
diff --git a/crates/tantivy/src/columnar/column_index/merge/mod.rs b/crates/tantivy/src/columnar/column_index/merge/mod.rs
index 22149951..199f3107 100644
--- a/crates/tantivy/src/columnar/column_index/merge/mod.rs
+++ b/crates/tantivy/src/columnar/column_index/merge/mod.rs
@@ -1,89 +1,17 @@
mod shuffled;
mod stacked;
-use crate::common::ReadOnlyBitSet;
use shuffled::merge_column_index_shuffled;
use stacked::merge_column_index_stacked;
use crate::columnar::column_index::SerializableColumnIndex;
-use crate::columnar::{Cardinality, ColumnIndex, MergeRowOrder};
+use crate::columnar::{Cardinality, MergeRowOrder};
-fn detect_cardinality_single_column_index(
- column_index: &ColumnIndex,
- alive_bitset_opt: &Option<ReadOnlyBitSet>,
-) -> Cardinality {
- let Some(alive_bitset) = alive_bitset_opt else {
- return column_index.get_cardinality();
- };
- let cardinality_before_deletes = column_index.get_cardinality();
- if cardinality_before_deletes == Cardinality::Full {
- // The columnar cardinality can only become more restrictive in the presence of deletes
- // (where cardinality sorted from the more restrictive to the least restrictive are Full,
- // Optional, Multivalued)
- //
- // If we are already "Full", we are guaranteed to stay "Full" after deletes.
- return Cardinality::Full;
- }
- let mut cardinality_so_far = Cardinality::Full;
- for doc_id in alive_bitset.iter() {
- let num_values = column_index.value_row_ids(doc_id).len();
- let row_cardinality = match num_values {
- 0 => Cardinality::Optional,
- 1 => Cardinality::Full,
- _ => Cardinality::Multivalued,
- };
- cardinality_so_far = cardinality_so_far.max(row_cardinality);
- if cardinality_so_far >= cardinality_before_deletes {
- // There won't be any improvement in the cardinality.
- // We can early exit.
- return cardinality_before_deletes;
- }
- }
- cardinality_so_far
-}
-
-fn detect_cardinality(
- column_indexes: &[ColumnIndex],
- merge_row_order: &MergeRowOrder,
-) -> Cardinality {
- match merge_row_order {
- MergeRowOrder::Stack(_) => column_indexes
- .iter()
- .map(ColumnIndex::get_cardinality)
- .max()
- .unwrap_or(Cardinality::Full),
- MergeRowOrder::Shuffled(shuffle_merge_order) => {
- let mut merged_cardinality = Cardinality::Full;
- for (column_index, alive_bitset_opt) in column_indexes
- .iter()
- .zip(shuffle_merge_order.alive_bitsets.iter())
- {
- let cardinality: Cardinality =
- detect_cardinality_single_column_index(column_index, alive_bitset_opt);
- if cardinality == Cardinality::Multivalued {
- return cardinality;
- }
- merged_cardinality = merged_cardinality.max(cardinality);
- }
- merged_cardinality
- }
- }
-}
-
-pub fn merge_column_index<'a>(
- columns: &'a [ColumnIndex],
- merge_row_order: &'a MergeRowOrder,
-) -> SerializableColumnIndex<'a> {
- // For simplification, we do not try to detect whether the cardinality could be
- // downgraded thanks to deletes.
- let cardinality_after_merge = detect_cardinality(columns, merge_row_order);
+pub fn merge_column_index(merge_row_order: &MergeRowOrder) -> SerializableColumnIndex {
+ let cardinality_after_merge = Cardinality::Full;
match merge_row_order {
- MergeRowOrder::Stack(stack_merge_order) => {
- merge_column_index_stacked(columns, cardinality_after_merge, stack_merge_order)
- }
- MergeRowOrder::Shuffled(complex_merge_order) => {
- merge_column_index_shuffled(columns, cardinality_after_merge, complex_merge_order)
- }
+ MergeRowOrder::Stack(_) => merge_column_index_stacked(cardinality_after_merge),
+ MergeRowOrder::Shuffled(_) => merge_column_index_shuffled(cardinality_after_merge),
}
}
@@ -92,121 +20,3 @@ pub fn merge_column_index<'a>(
// The merge order restricted to a specific column keeps the original row order.
//
// This may offer some optimization that we have not explored yet.
-
-#[cfg(test)]
-mod tests {
- use crate::columnar::column_index::merge::detect_cardinality;
- use crate::columnar::column_index::multivalued_index::MultiValueIndex;
- use crate::columnar::column_index::{
- merge_column_index, OptionalIndex, SerializableColumnIndex,
- };
- use crate::columnar::{
- Cardinality, ColumnIndex, MergeRowOrder, RowAddr, RowId, ShuffleMergeOrder, StackMergeOrder,
- };
-
- #[test]
- fn test_detect_cardinality() {
- assert_eq!(
- detect_cardinality(&[], &StackMergeOrder::stack_for_test(&[]).into()),
- Cardinality::Full
- );
- let optional_index: ColumnIndex = OptionalIndex::for_test(1, &[]).into();
- let multivalued_index: ColumnIndex = MultiValueIndex::for_test(&[0, 1]).into();
- assert_eq!(
- detect_cardinality(
- &[optional_index.clone(), ColumnIndex::Empty { num_docs: 0 }],
- &StackMergeOrder::stack_for_test(&[1, 0]).into()
- ),
- Cardinality::Optional
- );
- assert_eq!(
- detect_cardinality(
- &[optional_index.clone(), ColumnIndex::Full],
- &StackMergeOrder::stack_for_test(&[1, 1]).into()
- ),
- Cardinality::Optional
- );
- assert_eq!(
- detect_cardinality(
- &[
- multivalued_index.clone(),
- ColumnIndex::Empty { num_docs: 0 }
- ],
- &StackMergeOrder::stack_for_test(&[1, 0]).into()
- ),
- Cardinality::Multivalued
- );
- assert_eq!(
- detect_cardinality(
- &[multivalued_index.clone(), optional_index.clone()],
- &StackMergeOrder::stack_for_test(&[1, 1]).into()
- ),
- Cardinality::Multivalued
- );
- assert_eq!(
- detect_cardinality(
- &[optional_index, multivalued_index],
- &StackMergeOrder::stack_for_test(&[1, 1]).into()
- ),
- Cardinality::Multivalued
- );
- }
-
- #[test]
- fn test_merge_index_multivalued_sorted() {
- let column_indexes: Vec<ColumnIndex> = vec![MultiValueIndex::for_test(&[0, 2, 5]).into()];
- let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(
- &[2],
- vec![
- RowAddr {
- segment_ord: 0u32,
- row_id: 1u32,
- },
- RowAddr {
- segment_ord: 0u32,
- row_id: 0u32,
- },
- ],
- )
- .into();
- let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);
- let SerializableColumnIndex::Multivalued(start_index_iterable) = merged_column_index else {
- panic!("Excpected a multivalued index")
- };
- let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();
- assert_eq!(&start_indexes, &[0, 3, 5]);
- }
-
- #[test]
- fn test_merge_index_multivalued_sorted_several_segment() {
- let column_indexes: Vec<ColumnIndex> = vec![
- MultiValueIndex::for_test(&[0, 2, 5]).into(),
- ColumnIndex::Empty { num_docs: 0 },
- MultiValueIndex::for_test(&[0, 1, 4]).into(),
- ];
- let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(
- &[2, 0, 2],
- vec![
- RowAddr {
- segment_ord: 2u32,
- row_id: 1u32,
- },
- RowAddr {
- segment_ord: 0u32,
- row_id: 0u32,
- },
- RowAddr {
- segment_ord: 2u32,
- row_id: 0u32,
- },
- ],
- )
- .into();
- let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);
- let SerializableColumnIndex::Multivalued(start_index_iterable) = merged_column_index else {
- panic!("Excpected a multivalued index")
- };
- let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();
- assert_eq!(&start_indexes, &[0, 3, 5, 6]);
- }
-}
diff --git a/crates/tantivy/src/columnar/column_index/merge/shuffled.rs b/crates/tantivy/src/columnar/column_index/merge/shuffled.rs
index 4d9c96c0..f84f0a9d 100644
--- a/crates/tantivy/src/columnar/column_index/merge/shuffled.rs
+++ b/crates/tantivy/src/columnar/column_index/merge/shuffled.rs
@@ -1,171 +1,10 @@
-use std::iter;
+use crate::columnar::column_index::SerializableColumnIndex;
+use crate::columnar::Cardinality;
-use crate::columnar::column_index::{SerializableColumnIndex, Set};
-use crate::columnar::iterable::Iterable;
-use crate::columnar::{Cardinality, ColumnIndex, RowId, ShuffleMergeOrder};
-
-pub fn merge_column_index_shuffled<'a>(
- column_indexes: &'a [ColumnIndex],
+pub fn merge_column_index_shuffled(
cardinality_after_merge: Cardinality,
- shuffle_merge_order: &'a ShuffleMergeOrder,
-) -> SerializableColumnIndex<'a> {
+) -> SerializableColumnIndex {
match cardinality_after_merge {
Cardinality::Full => SerializableColumnIndex::Full,
- Cardinality::Optional => {
- let non_null_row_ids =
- merge_column_index_shuffled_optional(column_indexes, shuffle_merge_order);
- SerializableColumnIndex::Optional {
- non_null_row_ids,
- num_rows: shuffle_merge_order.num_rows(),
- }
- }
- Cardinality::Multivalued => {
- let multivalue_start_index =
- merge_column_index_shuffled_multivalued(column_indexes, shuffle_merge_order);
- SerializableColumnIndex::Multivalued(multivalue_start_index)
- }
- }
-}
-
-/// Merge several column indexes into one, ordering rows according to the merge_order passed as
-/// argument. While it is true that the `merge_order` may imply deletes and hence could in theory a
-/// multivalued index into an optional one, this is not supported today for simplification.
-///
-/// In other words the column_indexes passed as argument may NOT be multivalued.
-fn merge_column_index_shuffled_optional<'a>(
- column_indexes: &'a [ColumnIndex],
- merge_order: &'a ShuffleMergeOrder,
-) -> Box<dyn Iterable<RowId> + 'a> {
- Box::new(ShuffledIndex {
- column_indexes,
- merge_order,
- })
-}
-
-struct ShuffledIndex<'a> {
- column_indexes: &'a [ColumnIndex],
- merge_order: &'a ShuffleMergeOrder,
-}
-
-impl<'a> Iterable<u32> for ShuffledIndex<'a> {
- fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
- Box::new(
- self.merge_order
- .iter_new_to_old_row_addrs()
- .enumerate()
- .filter_map(|(new_row_id, old_row_addr)| {
- let column_index = &self.column_indexes[old_row_addr.segment_ord as usize];
- let row_id = new_row_id as u32;
- if column_index.has_value(old_row_addr.row_id) {
- Some(row_id)
- } else {
- None
- }
- }),
- )
- }
-}
-
-fn merge_column_index_shuffled_multivalued<'a>(
- column_indexes: &'a [ColumnIndex],
- merge_order: &'a ShuffleMergeOrder,
-) -> Box<dyn Iterable<RowId> + 'a> {
- Box::new(ShuffledMultivaluedIndex {
- column_indexes,
- merge_order,
- })
-}
-
-struct ShuffledMultivaluedIndex<'a> {
- column_indexes: &'a [ColumnIndex],
- merge_order: &'a ShuffleMergeOrder,
-}
-
-fn iter_num_values<'a>(
- column_indexes: &'a [ColumnIndex],
- merge_order: &'a ShuffleMergeOrder,
-) -> impl Iterator<Item = u32> + 'a {
- merge_order.iter_new_to_old_row_addrs().map(|row_addr| {
- let column_index = &column_indexes[row_addr.segment_ord as usize];
- match column_index {
- ColumnIndex::Empty { .. } => 0u32,
- ColumnIndex::Full => 1,
- ColumnIndex::Optional(optional_index) => {
- u32::from(optional_index.contains(row_addr.row_id))
- }
- ColumnIndex::Multivalued(multivalued_index) => {
- multivalued_index.range(row_addr.row_id).len() as u32
- }
- }
- })
-}
-
-/// Transforms an iterator containing the number of vals per row (with `num_rows` elements)
-/// into a `start_offset` iterator starting at 0 and (with `num_rows + 1` element)
-fn integrate_num_vals(num_vals: impl Iterator<Item = u32>) -> impl Iterator<Item = RowId> {
- iter::once(0u32).chain(num_vals.scan(0, |state, num_vals| {
- *state += num_vals;
- Some(*state)
- }))
-}
-
-impl<'a> Iterable<u32> for ShuffledMultivaluedIndex<'a> {
- fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
- let num_vals_per_row = iter_num_values(self.column_indexes, self.merge_order);
- Box::new(integrate_num_vals(num_vals_per_row))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::columnar::column_index::OptionalIndex;
- use crate::columnar::RowAddr;
-
- #[test]
- fn test_integrate_num_vals_empty() {
- assert!(integrate_num_vals(iter::empty()).eq(iter::once(0)));
- }
-
- #[test]
- fn test_integrate_num_vals_one_el() {
- assert!(integrate_num_vals(iter::once(10)).eq([0, 10].into_iter()));
- }
-
- #[test]
- fn test_integrate_num_vals_several() {
- assert!(integrate_num_vals([3, 0, 10, 20].into_iter()).eq([0, 3, 3, 13, 33].into_iter()));
- }
-
- #[test]
- fn test_merge_column_index_optional_shuffle() {
- let optional_index: ColumnIndex = OptionalIndex::for_test(2, &[0]).into();
- let column_indexes = [optional_index, ColumnIndex::Full];
- let row_addrs = vec![
- RowAddr {
- segment_ord: 0u32,
- row_id: 1u32,
- },
- RowAddr {
- segment_ord: 1u32,
- row_id: 0u32,
- },
- ];
- let shuffle_merge_order = ShuffleMergeOrder::for_test(&[2, 1], row_addrs);
- let serializable_index = merge_column_index_shuffled(
- &column_indexes[..],
- Cardinality::Optional,
- &shuffle_merge_order,
- );
- let SerializableColumnIndex::Optional {
- non_null_row_ids,
- num_rows,
- } = serializable_index
- else {
- panic!()
- };
- assert_eq!(num_rows, 2);
- let non_null_rows: Vec<RowId> = non_null_row_ids.boxed_iter().collect();
- assert_eq!(&non_null_rows, &[1]);
}
}
diff --git a/crates/tantivy/src/columnar/column_index/merge/stacked.rs b/crates/tantivy/src/columnar/column_index/merge/stacked.rs
index c727d066..cd1c3131 100644
--- a/crates/tantivy/src/columnar/column_index/merge/stacked.rs
+++ b/crates/tantivy/src/columnar/column_index/merge/stacked.rs
@@ -1,148 +1,12 @@
-use std::iter;
-
-use crate::columnar::column_index::{SerializableColumnIndex, Set};
-use crate::columnar::iterable::Iterable;
-use crate::columnar::{Cardinality, ColumnIndex, RowId, StackMergeOrder};
+use crate::columnar::column_index::SerializableColumnIndex;
+use crate::columnar::Cardinality;
/// Simple case:
/// The new mapping just consists in stacking the different column indexes.
///
/// There are no sort nor deletes involved.
-pub fn merge_column_index_stacked<'a>(
- columns: &'a [ColumnIndex],
- cardinality_after_merge: Cardinality,
- stack_merge_order: &'a StackMergeOrder,
-) -> SerializableColumnIndex<'a> {
+pub fn merge_column_index_stacked(cardinality_after_merge: Cardinality) -> SerializableColumnIndex {
match cardinality_after_merge {
Cardinality::Full => SerializableColumnIndex::Full,
- Cardinality::Optional => SerializableColumnIndex::Optional {
- non_null_row_ids: Box::new(StackedOptionalIndex {
- columns,
- stack_merge_order,
- }),
- num_rows: stack_merge_order.num_rows(),
- },
- Cardinality::Multivalued => {
- let stacked_multivalued_index = StackedMultivaluedIndex {
- columns,
- stack_merge_order,
- };
- SerializableColumnIndex::Multivalued(Box::new(stacked_multivalued_index))
- }
- }
-}
-
-struct StackedOptionalIndex<'a> {
- columns: &'a [ColumnIndex],
- stack_merge_order: &'a StackMergeOrder,
-}
-
-impl<'a> Iterable<RowId> for StackedOptionalIndex<'a> {
- fn boxed_iter(&self) -> Box<dyn Iterator<Item = RowId> + 'a> {
- Box::new(
- self.columns
- .iter()
- .enumerate()
- .flat_map(|(columnar_id, column_index_opt)| {
- let columnar_row_range = self.stack_merge_order.columnar_range(columnar_id);
- let rows_it: Box<dyn Iterator<Item = RowId>> = match column_index_opt {
- ColumnIndex::Full => Box::new(columnar_row_range),
- ColumnIndex::Optional(optional_index) => Box::new(
- optional_index
- .iter_rows()
- .map(move |row_id: RowId| columnar_row_range.start + row_id),
- ),
- ColumnIndex::Multivalued(_) => {
- panic!("No multivalued index is allowed when stacking column index");
- }
- ColumnIndex::Empty { .. } => Box::new(std::iter::empty()),
- };
- rows_it
- }),
- )
- }
-}
-
-#[derive(Clone, Copy)]
-struct StackedMultivaluedIndex<'a> {
- columns: &'a [ColumnIndex],
- stack_merge_order: &'a StackMergeOrder,
-}
-
-fn convert_column_opt_to_multivalued_index<'a>(
- column_index_opt: &'a ColumnIndex,
- num_rows: RowId,
-) -> Box<dyn Iterator<Item = RowId> + 'a> {
- match column_index_opt {
- ColumnIndex::Empty { .. } => Box::new(iter::repeat(0u32).take(num_rows as usize + 1)),
- ColumnIndex::Full => Box::new(0..num_rows + 1),
- ColumnIndex::Optional(optional_index) => {
- Box::new(
- (0..num_rows)
- // TODO optimize
- .map(|row_id| optional_index.rank(row_id))
- .chain(std::iter::once(optional_index.num_non_nulls())),
- )
- }
- ColumnIndex::Multivalued(multivalued_index) => multivalued_index.start_index_column.iter(),
- }
-}
-
-impl<'a> Iterable<RowId> for StackedMultivaluedIndex<'a> {
- fn boxed_iter(&self) -> Box<dyn Iterator<Item = RowId> + '_> {
- let multivalued_indexes =
- self.columns
- .iter()
- .enumerate()
- .map(|(columnar_id, column_opt)| {
- let num_rows =
- self.stack_merge_order.columnar_range(columnar_id).len() as RowId;
- convert_column_opt_to_multivalued_index(column_opt, num_rows)
- });
- stack_multivalued_indexes(multivalued_indexes)
- }
-}
-
-// Refactor me
-fn stack_multivalued_indexes<'a>(
- mut multivalued_indexes: impl Iterator<Item = Box<dyn Iterator<Item = RowId> + 'a>> + 'a,
-) -> Box<dyn Iterator<Item = RowId> + 'a> {
- let mut offset = 0;
- let mut last_row_id = 0;
- let mut current_it = multivalued_indexes.next();
- Box::new(std::iter::from_fn(move || loop {
- if let Some(row_id) = current_it.as_mut()?.next() {
- last_row_id = offset + row_id;
- return Some(last_row_id);
- }
- offset = last_row_id;
- loop {
- current_it = multivalued_indexes.next();
- if current_it.as_mut()?.next().is_some() {
- break;
- }
- }
- }))
-}
-
-#[cfg(test)]
-mod tests {
- use super::RowId;
-
- fn it<'a>(row_ids: &'a [RowId]) -> Box<dyn Iterator<Item = RowId> + 'a> {
- Box::new(row_ids.iter().copied())
- }
-
- #[test]
- fn test_stack() {
- let columns = [
- it(&[0u32, 0u32]),
- it(&[0u32, 1u32, 1u32, 4u32]),
- it(&[0u32, 3u32, 5u32]),
- it(&[0u32, 4u32]),
- ]
- .into_iter();
- let start_offsets: Vec<RowId> = super::stack_multivalued_indexes(columns).collect();
- assert_eq!(start_offsets, &[0, 0, 1, 1, 4, 7, 9, 13]);
}
}
diff --git a/crates/tantivy/src/columnar/column_index/mod.rs b/crates/tantivy/src/columnar/column_index/mod.rs
index 24883bcd..7a279735 100644
--- a/crates/tantivy/src/columnar/column_index/mod.rs
+++ b/crates/tantivy/src/columnar/column_index/mod.rs
@@ -14,72 +14,17 @@ pub use merge::merge_column_index;
pub use optional_index::{OptionalIndex, Set};
pub use serialize::{open_column_index, serialize_column_index, SerializableColumnIndex};
-use super::column_index::multivalued_index::MultiValueIndex;
use super::{Cardinality, DocId, RowId};
#[derive(Clone, Debug)]
pub enum ColumnIndex {
- Empty {
- num_docs: u32,
- },
Full,
- Optional(OptionalIndex),
- /// In addition, at index num_rows, an extra value is added
- /// containing the overal number of values.
- Multivalued(MultiValueIndex),
-}
-
-impl From<OptionalIndex> for ColumnIndex {
- fn from(optional_index: OptionalIndex) -> ColumnIndex {
- ColumnIndex::Optional(optional_index)
- }
-}
-
-impl From<MultiValueIndex> for ColumnIndex {
- fn from(multi_value_index: MultiValueIndex) -> ColumnIndex {
- ColumnIndex::Multivalued(multi_value_index)
- }
}
impl ColumnIndex {
- /// Returns the cardinality of the column index.
- ///
- /// By convention, if the column contains no docs, we consider that it is
- /// full.
- #[inline]
- pub fn get_cardinality(&self) -> Cardinality {
- match self {
- ColumnIndex::Empty { num_docs: 0 } | ColumnIndex::Full => Cardinality::Full,
- ColumnIndex::Empty { .. } => Cardinality::Optional,
- ColumnIndex::Optional(_) => Cardinality::Optional,
- ColumnIndex::Multivalued(_) => Cardinality::Multivalued,
- }
- }
-
- /// Returns true if and only if there are at least one value associated to the row.
- pub fn has_value(&self, doc_id: DocId) -> bool {
- match self {
- ColumnIndex::Empty { .. } => false,
- ColumnIndex::Full => true,
- ColumnIndex::Optional(optional_index) => optional_index.contains(doc_id),
- ColumnIndex::Multivalued(multivalued_index) => {
- !multivalued_index.range(doc_id).is_empty()
- }
- }
- }
-
pub fn value_row_ids(&self, doc_id: DocId) -> Range<RowId> {
match self {
- ColumnIndex::Empty { .. } => 0..0,
ColumnIndex::Full => doc_id..doc_id + 1,
- ColumnIndex::Optional(optional_index) => {
- if let Some(val) = optional_index.rank_if_exists(doc_id) {
- val..val + 1
- } else {
- 0..0
- }
- }
- ColumnIndex::Multivalued(multivalued_index) => multivalued_index.range(doc_id),
}
}
@@ -98,84 +43,16 @@ impl ColumnIndex {
row_ids: &mut Vec<RowId>,
) {
match self {
- ColumnIndex::Empty { .. } => {}
ColumnIndex::Full => {
doc_ids_out.extend_from_slice(doc_ids);
row_ids.extend_from_slice(doc_ids);
}
- ColumnIndex::Optional(optional_index) => {
- for doc_id in doc_ids {
- if let Some(row_id) = optional_index.rank_if_exists(*doc_id) {
- doc_ids_out.push(*doc_id);
- row_ids.push(row_id);
- }
- }
- }
- ColumnIndex::Multivalued(multivalued_index) => {
- for doc_id in doc_ids {
- for row_id in multivalued_index.range(*doc_id) {
- doc_ids_out.push(*doc_id);
- row_ids.push(row_id);
- }
- }
- }
}
}
pub fn docid_range_to_rowids(&self, doc_id_range: Range<DocId>) -> Range<RowId> {
match self {
- ColumnIndex::Empty { .. } => 0..0,
ColumnIndex::Full => doc_id_range,
- ColumnIndex::Optional(optional_index) => {
- let row_start = optional_index.rank(doc_id_range.start);
- let row_end = optional_index.rank(doc_id_range.end);
- row_start..row_end
- }
- ColumnIndex::Multivalued(multivalued_index) => {
- let end_docid = doc_id_range.end.min(multivalued_index.num_docs() - 1) + 1;
- let start_docid = doc_id_range.start.min(end_docid);
-
- let row_start = multivalued_index.start_index_column.get_val(start_docid);
- let row_end = multivalued_index.start_index_column.get_val(end_docid);
-
- row_start..row_end
- }
- }
- }
-
- pub fn select_batch_in_place(&self, doc_id_start: DocId, rank_ids: &mut Vec<RowId>) {
- match self {
- ColumnIndex::Empty { .. } => {
- rank_ids.clear();
- }
- ColumnIndex::Full => {
- // No need to do anything:
- // value_idx and row_idx are the same.
- }
- ColumnIndex::Optional(optional_index) => {
- optional_index.select_batch(&mut rank_ids[..]);
- }
- ColumnIndex::Multivalued(multivalued_index) => {
- multivalued_index.select_batch_in_place(doc_id_start, rank_ids)
- }
}
}
}
-
-#[cfg(test)]
-mod tests {
- use super::{Cardinality, ColumnIndex};
-
- #[test]
- fn test_column_index_get_cardinality() {
- assert_eq!(
- ColumnIndex::Empty { num_docs: 0 }.get_cardinality(),
- Cardinality::Full
- );
- assert_eq!(ColumnIndex::Full.get_cardinality(), Cardinality::Full);
- assert_eq!(
- ColumnIndex::Empty { num_docs: 1 }.get_cardinality(),
- Cardinality::Optional
- );
- }
-}
diff --git a/crates/tantivy/src/columnar/column_index/multivalued_index.rs b/crates/tantivy/src/columnar/column_index/multivalued_index.rs
index e346d3e1..e8cd8cf5 100644
--- a/crates/tantivy/src/columnar/column_index/multivalued_index.rs
+++ b/crates/tantivy/src/columnar/column_index/multivalued_index.rs
@@ -1,11 +1,10 @@
use std::io;
use std::io::Write;
-use std::ops::Range;
use std::sync::Arc;
use crate::common::OwnedBytes;
-use super::{DocId, RowId};
+use super::RowId;
use crate::columnar::column_values::{
load_u64_based_column_values, serialize_u64_based_column_values, CodecType, ColumnValues,
};
@@ -57,88 +56,9 @@ impl MultiValueIndex {
open_multivalued_index(bytes).unwrap()
}
- /// Returns `[start, end)`, such that the values associated with
- /// the given document are `start..end`.
- #[inline]
- pub(crate) fn range(&self, doc_id: DocId) -> Range<RowId> {
- let start = self.start_index_column.get_val(doc_id);
- let end = self.start_index_column.get_val(doc_id + 1);
- start..end
- }
-
/// Returns the number of documents in the index.
#[inline]
pub fn num_docs(&self) -> u32 {
self.start_index_column.num_vals() - 1
}
-
- /// Converts a list of ranks (row ids of values) in a 1:n index to the corresponding list of
- /// docids. Positions are converted inplace to docids.
- ///
- /// Since there is no index for value pos -> docid, but docid -> value pos range, we scan the
- /// index.
- ///
- /// Correctness: positions needs to be sorted. idx_reader needs to contain monotonically
- /// increasing positions.
- ///
- /// TODO: Instead of a linear scan we can employ a exponential search into binary search to
- /// match a docid to its value position.
- #[allow(clippy::bool_to_int_with_if)]
- pub(crate) fn select_batch_in_place(&self, docid_start: DocId, ranks: &mut Vec<u32>) {
- if ranks.is_empty() {
- return;
- }
- let mut cur_doc = docid_start;
- let mut last_doc = None;
-
- assert!(self.start_index_column.get_val(docid_start) <= ranks[0]);
-
- let mut write_doc_pos = 0;
- for i in 0..ranks.len() {
- let pos = ranks[i];
- loop {
- let end = self.start_index_column.get_val(cur_doc + 1);
- if end > pos {
- ranks[write_doc_pos] = cur_doc;
- write_doc_pos += if last_doc == Some(cur_doc) { 0 } else { 1 };
- last_doc = Some(cur_doc);
- break;
- }
- cur_doc += 1;
- }
- }
- ranks.truncate(write_doc_pos);
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::ops::Range;
-
- use super::MultiValueIndex;
-
- fn index_to_pos_helper(
- index: &MultiValueIndex,
- doc_id_range: Range<u32>,
- positions: &[u32],
- ) -> Vec<u32> {
- let mut positions = positions.to_vec();
- index.select_batch_in_place(doc_id_range.start, &mut positions);
- positions
- }
-
- #[test]
- fn test_positions_to_docid() {
- let index = MultiValueIndex::for_test(&[0, 10, 12, 15, 22, 23]);
- assert_eq!(index.num_docs(), 5);
- let positions = &[10u32, 11, 15, 20, 21, 22];
- assert_eq!(index_to_pos_helper(&index, 0..5, positions), vec![1, 3, 4]);
- assert_eq!(index_to_pos_helper(&index, 1..5, positions), vec![1, 3, 4]);
- assert_eq!(index_to_pos_helper(&index, 0..5, &[9]), vec![0]);
- assert_eq!(index_to_pos_helper(&index, 1..5, &[10]), vec![1]);
- assert_eq!(index_to_pos_helper(&index, 1..5, &[11]), vec![1]);
- assert_eq!(index_to_pos_helper(&index, 2..5, &[12]), vec![2]);
- assert_eq!(index_to_pos_helper(&index, 2..5, &[12, 14]), vec![2]);
- assert_eq!(index_to_pos_helper(&index, 2..5, &[12, 14, 15]), vec![2, 3]);
- }
}
diff --git a/crates/tantivy/src/columnar/column_index/serialize.rs b/crates/tantivy/src/columnar/column_index/serialize.rs
index 88e2d6a8..9a4501e7 100644
--- a/crates/tantivy/src/columnar/column_index/serialize.rs
+++ b/crates/tantivy/src/columnar/column_index/serialize.rs
@@ -3,29 +3,17 @@ use std::io::Write;
use crate::common::{CountingWriter, OwnedBytes};
-use super::{Cardinality, RowId};
-use crate::columnar::column_index::multivalued_index::serialize_multivalued_index;
-use crate::columnar::column_index::optional_index::serialize_optional_index;
+use super::Cardinality;
use crate::columnar::column_index::ColumnIndex;
-use crate::columnar::iterable::Iterable;
-pub enum SerializableColumnIndex<'a> {
+pub enum SerializableColumnIndex {
Full,
- Optional {
- non_null_row_ids: Box<dyn Iterable<RowId> + 'a>,
- num_rows: RowId,
- },
- // TODO remove the Arc<dyn> apart from serialization this is not
- // dynamic at all.
- Multivalued(Box<dyn Iterable<RowId> + 'a>),
}
-impl<'a> SerializableColumnIndex<'a> {
+impl SerializableColumnIndex {
pub fn get_cardinality(&self) -> Cardinality {
match self {
SerializableColumnIndex::Full => Cardinality::Full,
- SerializableColumnIndex::Optional { .. } => Cardinality::Optional,
- SerializableColumnIndex::Multivalued(_) => Cardinality::Multivalued,
}
}
}
@@ -40,13 +28,6 @@ pub fn serialize_column_index(
output.write_all(&[cardinality])?;
match column_index {
SerializableColumnIndex::Full => {}
- SerializableColumnIndex::Optional {
- non_null_row_ids,
- num_rows,
- } => serialize_optional_index(non_null_row_ids.as_ref(), num_rows, &mut output)?,
- SerializableColumnIndex::Multivalued(multivalued_index) => {
- serialize_multivalued_index(&*multivalued_index, &mut output)?
- }
}
let column_index_num_bytes = output.written_bytes() as u32;
Ok(column_index_num_bytes)
@@ -65,14 +46,6 @@ pub fn open_column_index(mut bytes: OwnedBytes) -> io::Result<ColumnIndex> {
bytes.advance(1);
match cardinality {
Cardinality::Full => Ok(ColumnIndex::Full),
- Cardinality::Optional => {
- let optional_index = super::optional_index::open_optional_index(bytes)?;
- Ok(ColumnIndex::Optional(optional_index))
- }
- Cardinality::Multivalued => {
- let multivalue_index = super::multivalued_index::open_multivalued_index(bytes)?;
- Ok(ColumnIndex::Multivalued(multivalue_index))
- }
}
}
diff --git a/crates/tantivy/src/columnar/column_values/mod.rs b/crates/tantivy/src/columnar/column_values/mod.rs
index ea214924..39261688 100644
--- a/crates/tantivy/src/columnar/column_values/mod.rs
+++ b/crates/tantivy/src/columnar/column_values/mod.rs
@@ -18,7 +18,6 @@ mod merge;
pub(crate) mod monotonic_mapping;
pub(crate) mod monotonic_mapping_u128;
mod stats;
-mod u128_based;
mod u64_based;
mod vec_column;
@@ -26,10 +25,6 @@ mod monotonic_column;
pub(crate) use merge::MergedColumnValues;
pub use stats::ColumnStats;
-pub use u128_based::{
- open_u128_as_compact_u64, open_u128_mapped, serialize_column_values_u128,
- CompactSpaceU64Accessor,
-};
pub use u64_based::{
load_u64_based_column_values, serialize_and_load_u64_based_column_values,
serialize_u64_based_column_values, CodecType, ALL_U64_CODEC_TYPES,
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/blank_range.rs b/crates/tantivy/src/columnar/column_values/u128_based/compact_space/blank_range.rs
deleted file mode 100644
index 4d1ceefd..00000000
--- a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/blank_range.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-use std::ops::RangeInclusive;
-
-/// The range of a blank in value space.
-///
-/// A blank is an unoccupied space in the data.
-/// Use try_into() to construct.
-/// A range has to have at least length of 3. Invalid ranges will be rejected.
-///
-/// Ordered by range length.
-#[derive(Debug, Eq, PartialEq, Clone)]
-pub(crate) struct BlankRange {
- blank_range: RangeInclusive<u128>,
-}
-impl TryFrom<RangeInclusive<u128>> for BlankRange {
- type Error = &'static str;
- fn try_from(range: RangeInclusive<u128>) -> Result<Self, Self::Error> {
- let blank_size = range.end().saturating_sub(*range.start());
- if blank_size < 2 {
- Err("invalid range")
- } else {
- Ok(BlankRange { blank_range: range })
- }
- }
-}
-impl BlankRange {
- pub(crate) fn blank_size(&self) -> u128 {
- self.blank_range.end() - self.blank_range.start() + 1
- }
- pub(crate) fn blank_range(&self) -> RangeInclusive<u128> {
- self.blank_range.clone()
- }
-}
-
-impl Ord for BlankRange {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.blank_size().cmp(&other.blank_size())
- }
-}
-impl PartialOrd for BlankRange {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/build_compact_space.rs b/crates/tantivy/src/columnar/column_values/u128_based/compact_space/build_compact_space.rs
deleted file mode 100644
index a2a13645..00000000
--- a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/build_compact_space.rs
+++ /dev/null
@@ -1,252 +0,0 @@
-use std::collections::{BTreeSet, BinaryHeap};
-use std::iter;
-use std::ops::RangeInclusive;
-
-use itertools::Itertools;
-
-use super::blank_range::BlankRange;
-use super::{CompactSpace, RangeMapping};
-
-/// Put the blanks for the sorted values into a binary heap
-fn get_blanks(values_sorted: &BTreeSet<u128>) -> BinaryHeap<BlankRange> {
- let mut blanks: BinaryHeap<BlankRange> = BinaryHeap::new();
- for (first, second) in values_sorted.iter().copied().tuple_windows() {
- // Correctness Overflow: the values are deduped and sorted (BTreeSet property), that means
- // there's always space between two values.
- let blank_range = first + 1..=second - 1;
- let blank_range: Result<BlankRange, _> = blank_range.try_into();
- if let Ok(blank_range) = blank_range {
- blanks.push(blank_range);
- }
- }
-
- blanks
-}
-
-struct BlankCollector {
- blanks: Vec<BlankRange>,
- staged_blanks_sum: u128,
-}
-impl BlankCollector {
- fn new() -> Self {
- Self {
- blanks: vec![],
- staged_blanks_sum: 0,
- }
- }
- fn stage_blank(&mut self, blank: BlankRange) {
- self.staged_blanks_sum += blank.blank_size();
- self.blanks.push(blank);
- }
- fn drain(&mut self) -> impl Iterator<Item = BlankRange> + '_ {
- self.staged_blanks_sum = 0;
- self.blanks.drain(..)
- }
- fn staged_blanks_sum(&self) -> u128 {
- self.staged_blanks_sum
- }
- fn num_staged_blanks(&self) -> usize {
- self.blanks.len()
- }
-}
-fn num_bits(val: u128) -> u8 {
- (128u32 - val.leading_zeros()) as u8
-}
-
-/// Will collect blanks and add them to compact space if more bits are saved than cost from
-/// metadata.
-pub fn get_compact_space(
- values_deduped_sorted: &BTreeSet<u128>,
- total_num_values: u32,
- cost_per_blank: usize,
-) -> CompactSpace {
- let mut compact_space_builder = CompactSpaceBuilder::new();
- if values_deduped_sorted.is_empty() {
- return compact_space_builder.finish();
- }
-
- // We start by space that's limited to min_value..=max_value
- // Replace after stabilization of https://github.com/rust-lang/rust/issues/62924
- let min_value = values_deduped_sorted.iter().next().copied().unwrap_or(0);
- let max_value = values_deduped_sorted.iter().last().copied().unwrap_or(0);
-
- let mut blanks: BinaryHeap<BlankRange> = get_blanks(values_deduped_sorted);
-
- // +1 for null, in case min and max covers the whole space, we are off by one.
- let mut amplitude_compact_space = (max_value - min_value).saturating_add(1);
- if min_value != 0 {
- compact_space_builder.add_blanks(iter::once(0..=min_value - 1));
- }
- if max_value != u128::MAX {
- compact_space_builder.add_blanks(iter::once(max_value + 1..=u128::MAX));
- }
-
- let mut amplitude_bits: u8 = num_bits(amplitude_compact_space);
-
- let mut blank_collector = BlankCollector::new();
-
- // We will stage blanks until they reduce the compact space by at least 1 bit and then flush
- // them if the metadata cost is lower than the total number of saved bits.
- // Binary heap to process the gaps by their size
- while let Some(blank_range) = blanks.pop() {
- blank_collector.stage_blank(blank_range);
-
- let staged_spaces_sum: u128 = blank_collector.staged_blanks_sum();
- let amplitude_new_compact_space = amplitude_compact_space - staged_spaces_sum;
- let amplitude_new_bits = num_bits(amplitude_new_compact_space);
-
- if amplitude_bits == amplitude_new_bits {
- continue;
- }
- let saved_bits = (amplitude_bits - amplitude_new_bits) as usize * total_num_values as usize;
- // TODO: Maybe calculate exact cost of blanks and run this more expensive computation only,
- // when amplitude_new_bits changes
- let cost = blank_collector.num_staged_blanks() * cost_per_blank;
-
- // We want to end up with a compact space that fits into 32 bits.
- // In order to deal with pathological cases, we force the algorithm to keep
- // refining the compact space the amplitude bits is lower than 32.
- //
- // The worst case scenario happens for a large number of u128s regularly
- // spread over the full u128 space.
- //
- // This change will force the algorithm to degenerate into dictionary encoding.
- if amplitude_bits <= 32 && cost >= saved_bits {
- // Continue here, since although we walk over the blanks by size,
- // we can potentially save a lot at the last bits, which are smaller blanks
- //
- // E.g. if the first range reduces the compact space by 1000 from 2000 to 1000, which
- // saves 11-10=1 bit and the next range reduces the compact space by 950 to
- // 50, which saves 10-6=4 bit
- continue;
- }
-
- amplitude_compact_space = amplitude_new_compact_space;
- amplitude_bits = amplitude_new_bits;
- compact_space_builder.add_blanks(blank_collector.drain().map(|blank| blank.blank_range()));
- }
-
- assert!(amplitude_bits <= 32);
-
- // special case, when we don't collected any blanks because:
- // * the data is empty (early exit)
- // * the algorithm did decide it's not worth the cost, which can be the case for single values
- //
- // We drain one collected blank unconditionally, so the empty case is reserved for empty
- // data, and therefore empty compact_space means the data is empty and no data is covered
- // (conversely to all data) and we can assign null to it.
- if compact_space_builder.is_empty() {
- compact_space_builder.add_blanks(
- blank_collector
- .drain()
- .map(|blank| blank.blank_range())
- .take(1),
- );
- }
-
- let compact_space = compact_space_builder.finish();
- if max_value - min_value != u128::MAX {
- debug_assert_eq!(
- compact_space.amplitude_compact_space(),
- amplitude_compact_space
- );
- }
- compact_space
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct CompactSpaceBuilder {
- blanks: Vec<RangeInclusive<u128>>,
-}
-
-impl CompactSpaceBuilder {
- /// Creates a new compact space builder which will initially cover the whole space.
- fn new() -> Self {
- Self { blanks: Vec::new() }
- }
-
- /// Assumes that repeated add_blank calls don't overlap and are not adjacent,
- /// e.g. [3..=5, 5..=10] is not allowed
- ///
- /// Both of those assumptions are true when blanks are produced from sorted values.
- fn add_blanks(&mut self, blank: impl Iterator<Item = RangeInclusive<u128>>) {
- self.blanks.extend(blank);
- }
-
- fn is_empty(&self) -> bool {
- self.blanks.is_empty()
- }
-
- /// Convert blanks to covered space and assign null value
- fn finish(mut self) -> CompactSpace {
- // sort by start. ranges are not allowed to overlap
- self.blanks.sort_unstable_by_key(|blank| *blank.start());
-
- let mut covered_space = Vec::with_capacity(self.blanks.len());
-
- // begining of the blanks
- if let Some(first_blank_start) = self.blanks.first().map(RangeInclusive::start) {
- if *first_blank_start != 0 {
- covered_space.push(0..=first_blank_start - 1);
- }
- }
-
- // Between the blanks
- let between_blanks = self.blanks.iter().tuple_windows().map(|(left, right)| {
- assert!(
- left.end() < right.start(),
- "overlapping or adjacent ranges detected"
- );
- *left.end() + 1..=*right.start() - 1
- });
- covered_space.extend(between_blanks);
-
- // end of the blanks
- if let Some(last_blank_end) = self.blanks.last().map(RangeInclusive::end) {
- if *last_blank_end != u128::MAX {
- covered_space.push(last_blank_end + 1..=u128::MAX);
- }
- }
-
- if covered_space.is_empty() {
- covered_space.push(0..=0); // empty data case
- };
-
- let mut compact_start: u32 = 1; // 0 is reserved for `null`
- let mut ranges_mapping: Vec<RangeMapping> = Vec::with_capacity(covered_space.len());
- for cov in covered_space {
- let range_mapping = super::RangeMapping {
- value_range: cov,
- compact_start,
- };
- let covered_range_len = range_mapping.range_length();
- ranges_mapping.push(range_mapping);
- compact_start += covered_range_len;
- }
- // println!("num ranges {}", ranges_mapping.len());
- CompactSpace { ranges_mapping }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::columnar::column_values::u128_based::compact_space::COST_PER_BLANK_IN_BITS;
-
- #[test]
- fn test_binary_heap_pop_order() {
- let mut blanks: BinaryHeap<BlankRange> = BinaryHeap::new();
- blanks.push((0..=10).try_into().unwrap());
- blanks.push((100..=200).try_into().unwrap());
- blanks.push((100..=110).try_into().unwrap());
- assert_eq!(blanks.pop().unwrap().blank_size(), 101);
- assert_eq!(blanks.pop().unwrap().blank_size(), 11);
- }
-
- #[test]
- fn test_worst_case_scenario() {
- let vals: BTreeSet<u128> = (0..8).map(|i| i * ((1u128 << 34) / 8)).collect();
- let compact_space = get_compact_space(&vals, vals.len() as u32, COST_PER_BLANK_IN_BITS);
- assert!(compact_space.amplitude_compact_space() < u32::MAX as u128);
- }
-}
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/mod.rs b/crates/tantivy/src/columnar/column_values/u128_based/compact_space/mod.rs
deleted file mode 100644
index bf7f0138..00000000
--- a/crates/tantivy/src/columnar/column_values/u128_based/compact_space/mod.rs
+++ /dev/null
@@ -1,844 +0,0 @@
-/// This codec takes a large number space (u128) and reduces it to a compact number space.
-///
-/// It will find spaces in the number range. For example:
-///
-/// 100, 101, 102, 103, 104, 50000, 50001
-/// could be mapped to
-/// 100..104 -> 0..4
-/// 50000..50001 -> 5..6
-///
-/// Compact space 0..=6 requires much less bits than 100..=50001
-///
-/// The codec is created to compress ip addresses, but may be employed in other use cases.
-use std::{
- cmp::Ordering,
- collections::BTreeSet,
- io::{self, Write},
- ops::{Range, RangeInclusive},
-};
-
-mod blank_range;
-mod build_compact_space;
-
-use crate::bitpacker::{BitPacker, BitUnpacker};
-use crate::common::{BinarySerializable, CountingWriter, OwnedBytes, VInt, VIntU128};
-use build_compact_space::get_compact_space;
-
-use crate::columnar::column_values::ColumnValues;
-use crate::columnar::RowId;
-
-/// The cost per blank is quite hard actually, since blanks are delta encoded, the actual cost of
-/// blanks depends on the number of blanks.
-///
-/// The number is taken by looking at a real dataset. It is optimized for larger datasets.
-const COST_PER_BLANK_IN_BITS: usize = 36;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct CompactSpace {
- ranges_mapping: Vec<RangeMapping>,
-}
-
-/// Maps the range from the original space to compact_start + range.len()
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct RangeMapping {
- value_range: RangeInclusive<u128>,
- compact_start: u32,
-}
-impl RangeMapping {
- fn range_length(&self) -> u32 {
- (self.value_range.end() - self.value_range.start()) as u32 + 1
- }
-
- // The last value of the compact space in this range
- fn compact_end(&self) -> u32 {
- self.compact_start + self.range_length() - 1
- }
-}
-
-impl BinarySerializable for CompactSpace {
- fn serialize<W: io::Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
- VInt(self.ranges_mapping.len() as u64).serialize(writer)?;
-
- let mut prev_value = 0;
- for value_range in self
- .ranges_mapping
- .iter()
- .map(|range_mapping| &range_mapping.value_range)
- {
- let blank_delta_start = value_range.start() - prev_value;
- VIntU128(blank_delta_start).serialize(writer)?;
- prev_value = *value_range.start();
-
- let blank_delta_end = value_range.end() - prev_value;
- VIntU128(blank_delta_end).serialize(writer)?;
- prev_value = *value_range.end();
- }
-
- Ok(())
- }
-
- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
- let num_ranges = VInt::deserialize(reader)?.0;
- let mut ranges_mapping: Vec<RangeMapping> = vec![];
- let mut value = 0u128;
- let mut compact_start = 1u32; // 0 is reserved for `null`
- for _ in 0..num_ranges {
- let blank_delta_start = VIntU128::deserialize(reader)?.0;
- value += blank_delta_start;
- let blank_start = value;
-
- let blank_delta_end = VIntU128::deserialize(reader)?.0;
- value += blank_delta_end;
- let blank_end = value;
-
- let range_mapping = RangeMapping {
- value_range: blank_start..=blank_end,
- compact_start,
- };
- let range_length = range_mapping.range_length();
- ranges_mapping.push(range_mapping);
- compact_start += range_length;
- }
-
- Ok(Self { ranges_mapping })
- }
-}
-
-impl CompactSpace {
- /// Amplitude is the value range of the compact space including the sentinel value used to
- /// identify null values. The compact space is 0..=amplitude .
- ///
- /// It's only used to verify we don't exceed u64 number space, which would indicate a bug.
- fn amplitude_compact_space(&self) -> u128 {
- self.ranges_mapping
- .last()
- .map(|last_range| last_range.compact_end() as u128)
- .unwrap_or(1) // compact space starts at 1, 0 == null
- }
-
- fn get_range_mapping(&self, pos: usize) -> &RangeMapping {
- &self.ranges_mapping[pos]
- }
-
- /// Returns either Ok(the value in the compact space) or if it is outside the compact space the
- /// Err(position where it would be inserted)
- fn u128_to_compact(&self, value: u128) -> Result<u32, usize> {
- self.ranges_mapping
- .binary_search_by(|probe| {
- let value_range: &RangeInclusive<u128> = &probe.value_range;
- if value < *value_range.start() {
- Ordering::Greater
- } else if value > *value_range.end() {
- Ordering::Less
- } else {
- Ordering::Equal
- }
- })
- .map(|pos| {
- let range_mapping = &self.ranges_mapping[pos];
- let pos_in_range: u32 = (value - range_mapping.value_range.start()) as u32;
- range_mapping.compact_start + pos_in_range
- })
- }
-
- /// Unpacks a value from compact space u32 to u128 space
- fn compact_to_u128(&self, compact: u32) -> u128 {
- let pos = self
- .ranges_mapping
- .binary_search_by_key(&compact, |range_mapping| range_mapping.compact_start)
- // Correctness: Overflow. The first range starts at compact space 0, the error from
- // binary search can never be 0
- .unwrap_or_else(|e| e - 1);
-
- let range_mapping = &self.ranges_mapping[pos];
- let diff = compact - range_mapping.compact_start;
- range_mapping.value_range.start() + diff as u128
- }
-}
-
-pub struct CompactSpaceCompressor {
- params: IPCodecParams,
-}
-
-#[derive(Debug, Clone)]
-pub struct IPCodecParams {
- compact_space: CompactSpace,
- bit_unpacker: BitUnpacker,
- min_value: u128,
- max_value: u128,
- num_vals: RowId,
- num_bits: u8,
-}
-
-impl CompactSpaceCompressor {
- pub fn num_vals(&self) -> RowId {
- self.params.num_vals
- }
-
- /// Taking the vals as Vec may cost a lot of memory. It is used to sort the vals.
- pub fn train_from(iter: impl Iterator<Item = u128>) -> Self {
- let mut values_sorted = BTreeSet::new();
- // Total number of values, with their redundancy.
- let mut total_num_values = 0u32;
- for val in iter {
- total_num_values += 1u32;
- values_sorted.insert(val);
- }
- let min_value = *values_sorted.iter().next().unwrap_or(&0);
- let max_value = *values_sorted.iter().last().unwrap_or(&0);
-
- let compact_space =
- get_compact_space(&values_sorted, total_num_values, COST_PER_BLANK_IN_BITS);
- let amplitude_compact_space = compact_space.amplitude_compact_space();
-
- assert!(
- amplitude_compact_space <= u64::MAX as u128,
- "case unsupported."
- );
-
- let num_bits = crate::bitpacker::compute_num_bits(amplitude_compact_space as u64);
-
- assert_eq!(
- compact_space
- .u128_to_compact(max_value)
- .expect("could not convert max value to compact space"),
- amplitude_compact_space as u32
- );
- CompactSpaceCompressor {
- params: IPCodecParams {
- compact_space,
- bit_unpacker: BitUnpacker::new(num_bits),
- min_value,
- max_value,
- num_vals: total_num_values,
- num_bits,
- },
- }
- }
-
- fn write_footer(self, writer: &mut impl Write) -> io::Result<()> {
- let writer = &mut CountingWriter::wrap(writer);
- self.params.serialize(writer)?;
-
- let footer_len = writer.written_bytes() as u32;
- footer_len.serialize(writer)?;
-
- Ok(())
- }
-
- pub fn compress_into(
- self,
- vals: impl Iterator<Item = u128>,
- write: &mut impl Write,
- ) -> io::Result<()> {
- let mut bitpacker = BitPacker::default();
- for val in vals {
- let compact = self
- .params
- .compact_space
- .u128_to_compact(val)
- .map_err(|_| {
- io::Error::new(
- io::ErrorKind::InvalidData,
- "Could not convert value to compact_space. This is a bug.",
- )
- })?;
- bitpacker.write(compact as u64, self.params.num_bits, write)?;
- }
- bitpacker.close(write)?;
- self.write_footer(write)?;
- Ok(())
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct CompactSpaceDecompressor {
- data: OwnedBytes,
- params: IPCodecParams,
-}
-
-impl BinarySerializable for IPCodecParams {
- fn serialize<W: io::Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
- // header flags for future optional dictionary encoding
- let footer_flags = 0u64;
- footer_flags.serialize(writer)?;
-
- VIntU128(self.min_value).serialize(writer)?;
- VIntU128(self.max_value).serialize(writer)?;
- VIntU128(self.num_vals as u128).serialize(writer)?;
- self.num_bits.serialize(writer)?;
-
- self.compact_space.serialize(writer)?;
-
- Ok(())
- }
-
- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
- let _header_flags = u64::deserialize(reader)?;
- let min_value = VIntU128::deserialize(reader)?.0;
- let max_value = VIntU128::deserialize(reader)?.0;
- let num_vals = VIntU128::deserialize(reader)?.0 as u32;
- let num_bits = u8::deserialize(reader)?;
- let compact_space = CompactSpace::deserialize(reader)?;
-
- Ok(Self {
- compact_space,
- bit_unpacker: BitUnpacker::new(num_bits),
- min_value,
- max_value,
- num_vals,
- num_bits,
- })
- }
-}
-
-/// Exposes the compact space compressed values as u64.
-///
-/// This allows faster access to the values, as u64 is faster to work with than u128.
-/// It also allows to handle u128 values like u64, via the `open_u64_lenient` as a uniform
-/// access interface.
-///
-/// When converting from the internal u64 to u128 `compact_to_u128` can be used.
-pub struct CompactSpaceU64Accessor(CompactSpaceDecompressor);
-impl CompactSpaceU64Accessor {
- pub(crate) fn open(data: OwnedBytes) -> io::Result<CompactSpaceU64Accessor> {
- let decompressor = CompactSpaceU64Accessor(CompactSpaceDecompressor::open(data)?);
- Ok(decompressor)
- }
- /// Convert a compact space value to u128
- pub fn compact_to_u128(&self, compact: u32) -> u128 {
- self.0.compact_to_u128(compact)
- }
-}
-
-impl ColumnValues<u64> for CompactSpaceU64Accessor {
- #[inline]
- fn get_val(&self, doc: u32) -> u64 {
- let compact = self.0.get_compact(doc);
- compact as u64
- }
-
- fn min_value(&self) -> u64 {
- self.0.u128_to_compact(self.0.min_value()).unwrap() as u64
- }
-
- fn max_value(&self) -> u64 {
- self.0.u128_to_compact(self.0.max_value()).unwrap() as u64
- }
-
- fn num_vals(&self) -> u32 {
- self.0.params.num_vals
- }
-
- #[inline]
- fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
- Box::new(self.0.iter_compact().map(|el| el as u64))
- }
-
- #[inline]
- fn get_row_ids_for_value_range(
- &self,
- value_range: RangeInclusive<u64>,
- position_range: Range<u32>,
- positions: &mut Vec<u32>,
- ) {
- let value_range = self.0.compact_to_u128(*value_range.start() as u32)
- ..=self.0.compact_to_u128(*value_range.end() as u32);
- self.0
- .get_row_ids_for_value_range(value_range, position_range, positions)
- }
-}
-
-impl ColumnValues<u128> for CompactSpaceDecompressor {
- #[inline]
- fn get_val(&self, doc: u32) -> u128 {
- self.get(doc)
- }
-
- fn min_value(&self) -> u128 {
- self.min_value()
- }
-
- fn max_value(&self) -> u128 {
- self.max_value()
- }
-
- fn num_vals(&self) -> u32 {
- self.params.num_vals
- }
-
- #[inline]
- fn iter(&self) -> Box<dyn Iterator<Item = u128> + '_> {
- Box::new(self.iter())
- }
-
- #[inline]
- fn get_row_ids_for_value_range(
- &self,
- value_range: RangeInclusive<u128>,
- position_range: Range<u32>,
- positions: &mut Vec<u32>,
- ) {
- if value_range.start() > value_range.end() {
- return;
- }
- let position_range = position_range.start..position_range.end.min(self.num_vals());
- let from_value = *value_range.start();
- let to_value = *value_range.end();
- assert!(to_value >= from_value);
- let compact_from = self.u128_to_compact(from_value);
- let compact_to = self.u128_to_compact(to_value);
-
- // Quick return, if both ranges fall into the same non-mapped space, the range can't cover
- // any values, so we can early exit
- match (compact_to, compact_from) {
- (Err(pos1), Err(pos2)) if pos1 == pos2 => return,
- _ => {}
- }
-
- let compact_from = compact_from.unwrap_or_else(|pos| {
- // Correctness: Out of bounds, if this value is Err(last_index + 1), we early exit,
- // since the to_value also mapps into the same non-mapped space
- let range_mapping = self.params.compact_space.get_range_mapping(pos);
- range_mapping.compact_start
- });
- // If there is no compact space, we go to the closest upperbound compact space
- let compact_to = compact_to.unwrap_or_else(|pos| {
- // Correctness: Overflow, if this value is Err(0), we early exit,
- // since the from_value also mapps into the same non-mapped space
-
- // Get end of previous range
- let pos = pos - 1;
- let range_mapping = self.params.compact_space.get_range_mapping(pos);
- range_mapping.compact_end()
- });
-
- let value_range = compact_from..=compact_to;
- self.get_positions_for_compact_value_range(value_range, position_range, positions);
- }
-}
-
-impl CompactSpaceDecompressor {
- pub fn open(data: OwnedBytes) -> io::Result<CompactSpaceDecompressor> {
- let (data_slice, footer_len_bytes) = data.split_at(data.len() - 4);
- let footer_len = u32::deserialize(&mut &footer_len_bytes[..])?;
-
- let data_footer = &data_slice[data_slice.len() - footer_len as usize..];
- let params = IPCodecParams::deserialize(&mut &data_footer[..])?;
- let decompressor = CompactSpaceDecompressor { data, params };
-
- Ok(decompressor)
- }
-
- /// Converting to compact space for the decompressor is more complex, since we may get values
- /// which are outside the compact space. e.g. if we map
- /// 1000 => 5
- /// 2000 => 6
- ///
- /// and we want a mapping for 1005, there is no equivalent compact space. We instead return an
- /// error with the index of the next range.
- fn u128_to_compact(&self, value: u128) -> Result<u32, usize> {
- self.params.compact_space.u128_to_compact(value)
- }
-
- fn compact_to_u128(&self, compact: u32) -> u128 {
- self.params.compact_space.compact_to_u128(compact)
- }
-
- #[inline]
- fn iter_compact(&self) -> impl Iterator<Item = u32> + '_ {
- (0..self.params.num_vals)
- .map(move |idx| self.params.bit_unpacker.get(idx, &self.data) as u32)
- }
-
- #[inline]
- fn iter(&self) -> impl Iterator<Item = u128> + '_ {
- // TODO: Performance. It would be better to iterate on the ranges and check existence via
- // the bit_unpacker.
- self.iter_compact()
- .map(|compact| self.compact_to_u128(compact))
- }
-
- #[inline]
- pub fn get_compact(&self, idx: u32) -> u32 {
- self.params.bit_unpacker.get(idx, &self.data) as u32
- }
-
- #[inline]
- pub fn get(&self, idx: u32) -> u128 {
- let compact = self.get_compact(idx);
- self.compact_to_u128(compact)
- }
-
- pub fn min_value(&self) -> u128 {
- self.params.min_value
- }
-
- pub fn max_value(&self) -> u128 {
- self.params.max_value
- }
-
- fn get_positions_for_compact_value_range(
- &self,
- value_range: RangeInclusive<u32>,
- position_range: Range<u32>,
- positions: &mut Vec<u32>,
- ) {
- self.params.bit_unpacker.get_ids_for_value_range(
- *value_range.start() as u64..=*value_range.end() as u64,
- position_range,
- &self.data,
- positions,
- );
- }
-}
-
-#[cfg(test)]
-mod tests {
-
- use itertools::Itertools;
-
- use super::*;
- use crate::columnar::column_values::u128_based::U128Header;
- use crate::columnar::column_values::{open_u128_mapped, serialize_column_values_u128};
-
- #[test]
- fn compact_space_test() {
- let ips: BTreeSet<u128> = [
- 2u128, 4u128, 1000, 1001, 1002, 1003, 1004, 1005, 1008, 1010, 1012, 1260,
- ]
- .into_iter()
- .collect();
- let compact_space = get_compact_space(&ips, ips.len() as u32, 11);
- let amplitude = compact_space.amplitude_compact_space();
- assert_eq!(amplitude, 17);
- assert_eq!(1, compact_space.u128_to_compact(2).unwrap());
- assert_eq!(2, compact_space.u128_to_compact(3).unwrap());
- assert_eq!(compact_space.u128_to_compact(100).unwrap_err(), 1);
-
- for (num1, num2) in (0..3).tuple_windows() {
- assert_eq!(
- compact_space.get_range_mapping(num1).compact_end() + 1,
- compact_space.get_range_mapping(num2).compact_start
- );
- }
-
- let mut output: Vec<u8> = Vec::new();
- compact_space.serialize(&mut output).unwrap();
-
- assert_eq!(
- compact_space,
- CompactSpace::deserialize(&mut &output[..]).unwrap()
- );
-
- for ip in ips {
- let compact = compact_space.u128_to_compact(ip).unwrap();
- assert_eq!(compact_space.compact_to_u128(compact), ip);
- }
- }
-
- #[test]
- fn compact_space_amplitude_test() {
- let ips = &[100000u128, 1000000].into_iter().collect();
- let compact_space = get_compact_space(ips, ips.len() as u32, 1);
- let amplitude = compact_space.amplitude_compact_space();
- assert_eq!(amplitude, 2);
- }
-
- fn test_all(mut data: OwnedBytes, expected: &[u128]) {
- let _header = U128Header::deserialize(&mut data);
- let decompressor = CompactSpaceDecompressor::open(data).unwrap();
- for (idx, expected_val) in expected.iter().cloned().enumerate() {
- let val = decompressor.get(idx as u32);
- assert_eq!(val, expected_val);
-
- let test_range = |range: RangeInclusive<u128>| {
- let expected_positions = expected
- .iter()
- .positions(|val| range.contains(val))
- .map(|pos| pos as u32)
- .collect::<Vec<_>>();
- let mut positions = Vec::new();
- decompressor.get_row_ids_for_value_range(
- range,
- 0..decompressor.num_vals(),
- &mut positions,
- );
- assert_eq!(positions, expected_positions);
- };
-
- test_range(expected_val.saturating_sub(1)..=expected_val);
- test_range(expected_val..=expected_val);
- test_range(expected_val..=expected_val.saturating_add(1));
- test_range(expected_val.saturating_sub(1)..=expected_val.saturating_add(1));
- }
- }
-
- fn test_aux_vals(u128_vals: &[u128]) -> OwnedBytes {
- let mut out = Vec::new();
- serialize_column_values_u128(&u128_vals, &mut out).unwrap();
- let data = OwnedBytes::new(out);
- test_all(data.clone(), u128_vals);
- data
- }
-
- #[test]
- fn test_range_1() {
- let vals = &[
- 1u128,
- 100u128,
- 3u128,
- 99999u128,
- 100000u128,
- 100001u128,
- 4_000_211_221u128,
- 4_000_211_222u128,
- 333u128,
- ];
- let mut data = test_aux_vals(vals);
-
- let _header = U128Header::deserialize(&mut data);
- let decomp = CompactSpaceDecompressor::open(data).unwrap();
- let complete_range = 0..vals.len() as u32;
- for (pos, val) in vals.iter().enumerate() {
- let val = *val;
- let pos = pos as u32;
- let mut positions = Vec::new();
- decomp.get_row_ids_for_value_range(val..=val, pos..pos + 1, &mut positions);
- assert_eq!(positions, vec![pos]);
- }
-
- // handle docid range out of bounds
- let positions: Vec<u32> = get_positions_for_value_range_helper(&decomp, 0..=1, 1..u32::MAX);
- assert!(positions.is_empty());
-
- let positions =
- get_positions_for_value_range_helper(&decomp, 0..=1, complete_range.clone());
- assert_eq!(positions, vec![0]);
- let positions =
- get_positions_for_value_range_helper(&decomp, 0..=2, complete_range.clone());
- assert_eq!(positions, vec![0]);
- let positions =
- get_positions_for_value_range_helper(&decomp, 0..=3, complete_range.clone());
- assert_eq!(positions, vec![0, 2]);
- assert_eq!(
- get_positions_for_value_range_helper(
- &decomp,
- 99999u128..=99999u128,
- complete_range.clone()
- ),
- vec![3]
- );
- assert_eq!(
- get_positions_for_value_range_helper(
- &decomp,
- 99999u128..=100000u128,
- complete_range.clone()
- ),
- vec![3, 4]
- );
- assert_eq!(
- get_positions_for_value_range_helper(
- &decomp,
- 99998u128..=100000u128,
- complete_range.clone()
- ),
- vec![3, 4]
- );
- assert_eq!(
- &get_positions_for_value_range_helper(
- &decomp,
- 99998u128..=99999u128,
- complete_range.clone()
- ),
- &[3]
- );
- assert!(get_positions_for_value_range_helper(
- &decomp,
- 99998u128..=99998u128,
- complete_range.clone()
- )
- .is_empty());
- assert_eq!(
- &get_positions_for_value_range_helper(
- &decomp,
- 333u128..=333u128,
- complete_range.clone()
- ),
- &[8]
- );
- assert_eq!(
- &get_positions_for_value_range_helper(
- &decomp,
- 332u128..=333u128,
- complete_range.clone()
- ),
- &[8]
- );
- assert_eq!(
- &get_positions_for_value_range_helper(
- &decomp,
- 332u128..=334u128,
- complete_range.clone()
- ),
- &[8]
- );
- assert_eq!(
- &get_positions_for_value_range_helper(
- &decomp,
- 333u128..=334u128,
- complete_range.clone()
- ),
- &[8]
- );
-
- assert_eq!(
- &get_positions_for_value_range_helper(
- &decomp,
- 4_000_211_221u128..=5_000_000_000u128,
- complete_range
- ),
- &[6, 7]
- );
- }
-
- #[test]
- fn test_empty() {
- let vals = &[];
- let data = test_aux_vals(vals);
- let _decomp = CompactSpaceDecompressor::open(data).unwrap();
- }
-
- #[test]
- fn test_range_2() {
- let vals = &[
- 100u128,
- 99999u128,
- 100000u128,
- 100001u128,
- 4_000_211_221u128,
- 4_000_211_222u128,
- 333u128,
- ];
- let mut data = test_aux_vals(vals);
- let _header = U128Header::deserialize(&mut data);
- let decomp = CompactSpaceDecompressor::open(data).unwrap();
- let complete_range = 0..vals.len() as u32;
- assert!(
- &get_positions_for_value_range_helper(&decomp, 0..=5, complete_range.clone())
- .is_empty(),
- );
- assert_eq!(
- &get_positions_for_value_range_helper(&decomp, 0..=100, complete_range.clone()),
- &[0]
- );
- assert_eq!(
- &get_positions_for_value_range_helper(&decomp, 0..=105, complete_range),
- &[0]
- );
- }
-
- fn get_positions_for_value_range_helper<C: ColumnValues<T> + ?Sized, T: PartialOrd>(
- column: &C,
- value_range: RangeInclusive<T>,
- doc_id_range: Range<u32>,
- ) -> Vec<u32> {
- let mut positions = Vec::new();
- column.get_row_ids_for_value_range(value_range, doc_id_range, &mut positions);
- positions
- }
-
- #[test]
- fn test_range_3() {
- let vals = &[
- 200u128,
- 201,
- 202,
- 203,
- 204,
- 204,
- 206,
- 207,
- 208,
- 209,
- 210,
- 1_000_000,
- 5_000_000_000,
- ];
- let mut out = Vec::new();
- serialize_column_values_u128(&&vals[..], &mut out).unwrap();
- let decomp = open_u128_mapped(OwnedBytes::new(out)).unwrap();
- let complete_range = 0..vals.len() as u32;
-
- assert_eq!(
- get_positions_for_value_range_helper(&*decomp, 199..=200, complete_range.clone()),
- vec![0]
- );
-
- assert_eq!(
- get_positions_for_value_range_helper(&*decomp, 199..=201, complete_range.clone()),
- vec![0, 1]
- );
-
- assert_eq!(
- get_positions_for_value_range_helper(&*decomp, 200..=200, complete_range.clone()),
- vec![0]
- );
-
- assert_eq!(
- get_positions_for_value_range_helper(&*decomp, 1_000_000..=1_000_000, complete_range),
- vec![11]
- );
- }
-
- #[test]
- fn test_bug1() {
- let vals = &[9223372036854775806];
- let _data = test_aux_vals(vals);
- }
-
- #[test]
- fn test_bug2() {
- let vals = &[340282366920938463463374607431768211455u128];
- let _data = test_aux_vals(vals);
- }
-
- #[test]
- fn test_bug3() {
- let vals = &[340282366920938463463374607431768211454];
- let _data = test_aux_vals(vals);
- }
-
- #[test]
- fn test_bug4() {
- let vals = &[340282366920938463463374607431768211455, 0];
- let _data = test_aux_vals(vals);
- }
-
- #[test]
- fn test_first_large_gaps() {
- let vals = &[1_000_000_000u128; 100];
- let _data = test_aux_vals(vals);
- }
-
- use proptest::prelude::*;
-
- fn num_strategy() -> impl Strategy<Value = u128> {
- prop_oneof![
- 1 => prop::num::u128::ANY.prop_map(|num| u128::MAX - (num % 10) ),
- 1 => prop::num::u128::ANY.prop_map(|num| i64::MAX as u128 + 5 - (num % 10) ),
- 1 => prop::num::u128::ANY.prop_map(|num| i128::MAX as u128 + 5 - (num % 10) ),
- 1 => prop::num::u128::ANY.prop_map(|num| num % 10 ),
- 20 => prop::num::u128::ANY,
- ]
- }
-
- proptest! {
- #![proptest_config(ProptestConfig::with_cases(10))]
-
- #[test]
- fn compress_decompress_random(vals in proptest::collection::vec(num_strategy() , 1..1000)) {
- let _data = test_aux_vals(&vals);
- }
- }
-}
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/mod.rs b/crates/tantivy/src/columnar/column_values/u128_based/mod.rs
deleted file mode 100644
index 9e678753..00000000
--- a/crates/tantivy/src/columnar/column_values/u128_based/mod.rs
+++ /dev/null
@@ -1,197 +0,0 @@
-use std::fmt::Debug;
-use std::io;
-use std::io::Write;
-use std::sync::Arc;
-
-mod compact_space;
-
-use crate::common::{BinarySerializable, OwnedBytes, VInt};
-pub use compact_space::{
- CompactSpaceCompressor, CompactSpaceDecompressor, CompactSpaceU64Accessor,
-};
-
-use super::{ColumnValues, MonotonicallyMappableToU128};
-use crate::columnar::column_values::monotonic_map_column;
-use crate::columnar::column_values::monotonic_mapping::{
- StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal,
-};
-use crate::columnar::iterable::Iterable;
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub(crate) struct U128Header {
- pub num_vals: u32,
- pub codec_type: U128FastFieldCodecType,
-}
-
-impl BinarySerializable for U128Header {
- fn serialize<W: io::Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
- VInt(self.num_vals as u64).serialize(writer)?;
- self.codec_type.serialize(writer)?;
- Ok(())
- }
-
- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
- let num_vals = VInt::deserialize(reader)?.0 as u32;
- let codec_type = U128FastFieldCodecType::deserialize(reader)?;
- Ok(U128Header {
- num_vals,
- codec_type,
- })
- }
-}
-
-/// Serializes u128 values with the compact space codec.
-pub fn serialize_column_values_u128<T: MonotonicallyMappableToU128>(
- iterable: &dyn Iterable<T>,
- output: &mut impl io::Write,
-) -> io::Result<()> {
- let compressor = CompactSpaceCompressor::train_from(
- iterable
- .boxed_iter()
- .map(MonotonicallyMappableToU128::to_u128),
- );
- let header = U128Header {
- num_vals: compressor.num_vals(),
- codec_type: U128FastFieldCodecType::CompactSpace,
- };
- header.serialize(output)?;
- compressor.compress_into(
- iterable
- .boxed_iter()
- .map(MonotonicallyMappableToU128::to_u128),
- output,
- )?;
- Ok(())
-}
-
-#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
-#[repr(u8)]
-/// Available codecs to use to encode the u128 (via [`MonotonicallyMappableToU128`]) converted data.
-pub(crate) enum U128FastFieldCodecType {
- /// This codec takes a large number space (u128) and reduces it to a compact number space, by
- /// removing the holes.
- CompactSpace = 1,
-}
-
-impl BinarySerializable for U128FastFieldCodecType {
- fn serialize<W: Write + ?Sized>(&self, wrt: &mut W) -> io::Result<()> {
- self.to_code().serialize(wrt)
- }
-
- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
- let code = u8::deserialize(reader)?;
- let codec_type: Self = Self::from_code(code)
- .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Unknown code `{code}.`"))?;
- Ok(codec_type)
- }
-}
-
-impl U128FastFieldCodecType {
- pub(crate) fn to_code(self) -> u8 {
- self as u8
- }
-
- pub(crate) fn from_code(code: u8) -> Option<Self> {
- match code {
- 1 => Some(Self::CompactSpace),
- _ => None,
- }
- }
-}
-
-/// Returns the correct codec reader wrapped in the `Arc` for the data.
-pub fn open_u128_mapped<T: MonotonicallyMappableToU128 + Debug>(
- mut bytes: OwnedBytes,
-) -> io::Result<Arc<dyn ColumnValues<T>>> {
- let header = U128Header::deserialize(&mut bytes)?;
- assert_eq!(header.codec_type, U128FastFieldCodecType::CompactSpace);
- let reader = CompactSpaceDecompressor::open(bytes)?;
- let inverted: StrictlyMonotonicMappingInverter<StrictlyMonotonicMappingToInternal<T>> =
- StrictlyMonotonicMappingToInternal::<T>::new().into();
- Ok(Arc::new(monotonic_map_column(reader, inverted)))
-}
-
-/// Returns the u64 representation of the u128 data.
-/// The internal representation of the data as u64 is useful for faster processing.
-///
-/// In order to convert to u128 back cast to `CompactSpaceU64Accessor` and call
-/// `compact_to_u128`.
-///
-/// # Notice
-/// In case there are new codecs added, check for usages of `CompactSpaceDecompressorU64` and
-/// also handle the new codecs.
-pub fn open_u128_as_compact_u64(mut bytes: OwnedBytes) -> io::Result<Arc<dyn ColumnValues<u64>>> {
- let header = U128Header::deserialize(&mut bytes)?;
- assert_eq!(header.codec_type, U128FastFieldCodecType::CompactSpace);
- let reader = CompactSpaceU64Accessor::open(bytes)?;
- Ok(Arc::new(reader))
-}
-
-#[cfg(test)]
-pub mod tests {
- use super::*;
- use crate::columnar::column_values::u64_based::{
- serialize_and_load_u64_based_column_values, serialize_u64_based_column_values,
- ALL_U64_CODEC_TYPES,
- };
- use crate::columnar::column_values::CodecType;
-
- #[test]
- fn test_serialize_deserialize_u128_header() {
- let original = U128Header {
- num_vals: 11,
- codec_type: U128FastFieldCodecType::CompactSpace,
- };
- let mut out = Vec::new();
- original.serialize(&mut out).unwrap();
- let restored = U128Header::deserialize(&mut &out[..]).unwrap();
- assert_eq!(restored, original);
- }
-
- #[test]
- fn test_serialize_deserialize() {
- let original = [1u64, 5u64, 10u64];
- let restored: Vec<u64> =
- serialize_and_load_u64_based_column_values(&&original[..], &ALL_U64_CODEC_TYPES)
- .iter()
- .collect();
- assert_eq!(&restored, &original[..]);
- }
-
- #[test]
- fn test_fastfield_bool_size_bitwidth_1() {
- let mut buffer = Vec::new();
- serialize_u64_based_column_values::<bool>(
- &&[false, true][..],
- &ALL_U64_CODEC_TYPES,
- &mut buffer,
- )
- .unwrap();
- // TODO put the header as a footer so that it serves as a padding.
- // 5 bytes of header, 1 byte of value, 7 bytes of padding.
- assert_eq!(buffer.len(), 5 + 1);
- }
-
- #[test]
- fn test_fastfield_bool_bit_size_bitwidth_0() {
- let mut buffer = Vec::new();
- serialize_u64_based_column_values::<bool>(
- &&[false, true][..],
- &ALL_U64_CODEC_TYPES,
- &mut buffer,
- )
- .unwrap();
- // 6 bytes of header, 0 bytes of value, 7 bytes of padding.
- assert_eq!(buffer.len(), 6);
- }
-
- #[test]
- fn test_fastfield_gcd() {
- let mut buffer = Vec::new();
- let vals: Vec<u64> = (0..80).map(|val| (val % 7) * 1_000u64).collect();
- serialize_u64_based_column_values(&&vals[..], &[CodecType::Bitpacked], &mut buffer)
- .unwrap();
- // Values are stored over 3 bits.
- assert_eq!(buffer.len(), 6 + (3 * 80 / 8));
- }
-}
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs
index 7d89941d..bd070533 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs
@@ -79,7 +79,7 @@ impl ColumnValues for BitpackedReader {
fn get_row_ids_for_value_range(
&self,
range: RangeInclusive<u64>,
- doc_id_range: Range<u32>,
+ mut doc_id_range: Range<u32>,
positions: &mut Vec<u32>,
) {
let Some(transformed_range) =
@@ -88,6 +88,8 @@ impl ColumnValues for BitpackedReader {
positions.clear();
return;
};
+ doc_id_range.end = doc_id_range.end.min(self.num_vals());
+
self.bit_unpacker.get_ids_for_value_range(
transformed_range,
doc_id_range,
diff --git a/crates/tantivy/src/columnar/columnar/column_type.rs b/crates/tantivy/src/columnar/columnar/column_type.rs
index 24dc9e66..2016c8f8 100644
--- a/crates/tantivy/src/columnar/columnar/column_type.rs
+++ b/crates/tantivy/src/columnar/columnar/column_type.rs
@@ -1,6 +1,5 @@
use std::fmt;
use std::fmt::Debug;
-use std::net::Ipv6Addr;
use serde::{Deserialize, Serialize};
@@ -16,10 +15,8 @@ pub enum ColumnType {
U64 = 1u8,
F64 = 2u8,
Bytes = 3u8,
- Str = 4u8,
- Bool = 5u8,
- IpAddr = 6u8,
- DateTime = 7u8,
+ Bool = 4u8,
+ DateTime = 5u8,
}
impl fmt::Display for ColumnType {
@@ -29,9 +26,7 @@ impl fmt::Display for ColumnType {
ColumnType::U64 => "u64",
ColumnType::F64 => "f64",
ColumnType::Bytes => "bytes",
- ColumnType::Str => "str",
ColumnType::Bool => "bool",
- ColumnType::IpAddr => "ip",
ColumnType::DateTime => "datetime",
};
write!(f, "{short_str}")
@@ -39,14 +34,12 @@ impl fmt::Display for ColumnType {
}
// The order needs to match _exactly_ the order in the enum
-const COLUMN_TYPES: [ColumnType; 8] = [
+const COLUMN_TYPES: [ColumnType; 6] = [
ColumnType::I64,
ColumnType::U64,
ColumnType::F64,
ColumnType::Bytes,
- ColumnType::Str,
ColumnType::Bool,
- ColumnType::IpAddr,
ColumnType::DateTime,
];
@@ -79,11 +72,7 @@ impl ColumnType {
ColumnType::I64 => Some(NumericalType::I64),
ColumnType::U64 => Some(NumericalType::U64),
ColumnType::F64 => Some(NumericalType::F64),
- ColumnType::Bytes
- | ColumnType::Str
- | ColumnType::Bool
- | ColumnType::IpAddr
- | ColumnType::DateTime => None,
+ ColumnType::Bytes | ColumnType::Bool | ColumnType::DateTime => None,
}
}
}
@@ -142,16 +131,6 @@ impl HasAssociatedColumnType for crate::common::DateTime {
}
}
-impl HasAssociatedColumnType for Ipv6Addr {
- fn column_type() -> ColumnType {
- ColumnType::IpAddr
- }
-
- fn default_value() -> Self {
- Ipv6Addr::from([0u8; 16])
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -178,6 +157,6 @@ mod tests {
num_cardinality += 1;
}
}
- assert_eq!(num_cardinality, 3);
+ assert_eq!(num_cardinality, 1);
}
}
diff --git a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs
index 27a7c307..a30f13a6 100644
--- a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs
+++ b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs
@@ -12,7 +12,7 @@ use crate::columnar::{BytesColumn, MergeRowOrder, ShuffleMergeOrder};
// Serialize [Dictionary, Column, dictionary num bytes U64::LE]
// Column: [Column Index, Column Values, column index num bytes U32::LE]
pub fn merge_bytes_or_str_column(
- column_index: SerializableColumnIndex<'_>,
+ column_index: SerializableColumnIndex,
bytes_columns: &[Option<BytesColumn>],
merge_row_order: &MergeRowOrder,
output: &mut impl Write,
@@ -79,8 +79,15 @@ impl<'a> RemappedTermOrdinalsValues<'a> {
Box::new(
shuffle_merge_order
.iter_new_to_old_row_addrs()
- .flat_map(move |old_addr| {
+ .filter_map(move |old_addr| {
let segment_ord = self.term_ord_mapping.get_segment(old_addr.segment_ord);
+ if !segment_ord.is_empty() {
+ Some((old_addr, segment_ord))
+ } else {
+ None
+ }
+ })
+ .flat_map(|(old_addr, segment_ord)| {
self.bytes_columns[old_addr.segment_ord as usize]
.as_ref()
.into_iter()
@@ -98,7 +105,7 @@ fn compute_term_bitset(column: &BytesColumn, row_bitset: &ReadOnlyBitSet) -> Bit
let num_terms = column.dictionary().num_terms();
let mut term_bitset = BitSet::with_max_value(num_terms as u32);
for row_id in row_bitset.iter() {
- for term_ord in column.term_ord_column.values_for_doc(row_id) {
+ if let Some(term_ord) = column.term_ord_column.first(row_id) {
term_bitset.insert(term_ord as u32);
}
}
diff --git a/crates/tantivy/src/columnar/columnar/merge/mod.rs b/crates/tantivy/src/columnar/columnar/merge/mod.rs
index 6acc8f14..1ee7e4a3 100644
--- a/crates/tantivy/src/columnar/columnar/merge/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/merge/mod.rs
@@ -4,16 +4,12 @@ mod term_merger;
use std::collections::{BTreeMap, HashSet};
use std::io;
-use std::net::Ipv6Addr;
use std::sync::Arc;
-use itertools::Itertools;
pub use merge_mapping::{MergeRowOrder, ShuffleMergeOrder, StackMergeOrder};
use super::writer::ColumnarSerializer;
-use crate::columnar::column::{
- serialize_column_mappable_to_u128, serialize_column_mappable_to_u64,
-};
+use crate::columnar::column::serialize_column_mappable_to_u64;
use crate::columnar::column_values::MergedColumnValues;
use crate::columnar::columnar::merge::merge_dict_column::merge_bytes_or_str_column;
use crate::columnar::columnar::writer::CompatibleNumericalTypes;
@@ -37,9 +33,7 @@ use crate::columnar::{
pub(crate) enum ColumnTypeCategory {
Numerical,
Bytes,
- Str,
Bool,
- IpAddr,
DateTime,
}
@@ -50,9 +44,7 @@ impl From<ColumnType> for ColumnTypeCategory {
ColumnType::U64 => ColumnTypeCategory::Numerical,
ColumnType::F64 => ColumnTypeCategory::Numerical,
ColumnType::Bytes => ColumnTypeCategory::Bytes,
- ColumnType::Str => ColumnTypeCategory::Str,
ColumnType::Bool => ColumnTypeCategory::Bool,
- ColumnType::IpAddr => ColumnTypeCategory::IpAddr,
ColumnType::DateTime => ColumnTypeCategory::DateTime,
}
}
@@ -84,10 +76,6 @@ pub fn merge_columnar(
output: &mut impl io::Write,
) -> io::Result<()> {
let mut serializer = ColumnarSerializer::new(output);
- let num_rows_per_columnar = columnar_readers
- .iter()
- .map(|reader| reader.num_rows())
- .collect::<Vec<u32>>();
let columns_to_merge =
group_columns_for_merge(columnar_readers, required_columns, &merge_row_order)?;
@@ -106,7 +94,6 @@ pub fn merge_columnar(
serializer.start_serialize_column(column_name.as_bytes(), column_type);
merge_column(
column_type,
- &num_rows_per_columnar,
columns,
&merge_row_order,
&mut column_serializer,
@@ -125,13 +112,12 @@ fn dynamic_column_to_u64_monotonic(dynamic_column: DynamicColumn) -> Option<Colu
DynamicColumn::U64(column) => Some(column.to_u64_monotonic()),
DynamicColumn::F64(column) => Some(column.to_u64_monotonic()),
DynamicColumn::DateTime(column) => Some(column.to_u64_monotonic()),
- DynamicColumn::IpAddr(_) | DynamicColumn::Bytes(_) | DynamicColumn::Str(_) => None,
+ DynamicColumn::Bytes(_) => None,
}
}
fn merge_column(
column_type: ColumnType,
- num_docs_per_column: &[u32],
columns: Vec<Option<DynamicColumn>>,
merge_row_order: &MergeRowOrder,
wrt: &mut impl io::Write,
@@ -145,23 +131,16 @@ fn merge_column(
let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());
let mut column_values: Vec<Option<Arc<dyn ColumnValues>>> =
Vec::with_capacity(columns.len());
- for (i, dynamic_column_opt) in columns.into_iter().enumerate() {
+ for dynamic_column_opt in columns {
if let Some(Column { index: idx, values }) =
dynamic_column_opt.and_then(dynamic_column_to_u64_monotonic)
{
column_indexes.push(idx);
column_values.push(Some(values));
- } else {
- column_indexes.push(ColumnIndex::Empty {
- num_docs: num_docs_per_column[i],
- });
- column_values.push(None);
}
}
- let merged_column_index = crate::columnar::column_index::merge_column_index(
- &column_indexes[..],
- merge_row_order,
- );
+ let merged_column_index =
+ crate::columnar::column_index::merge_column_index(merge_row_order);
let merge_column_values = MergedColumnValues {
column_indexes: &column_indexes[..],
column_values: &column_values[..],
@@ -169,61 +148,17 @@ fn merge_column(
};
serialize_column_mappable_to_u64(merged_column_index, &merge_column_values, wrt)?;
}
- ColumnType::IpAddr => {
- let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());
- let mut column_values: Vec<Option<Arc<dyn ColumnValues<Ipv6Addr>>>> =
- Vec::with_capacity(columns.len());
- for (i, dynamic_column_opt) in columns.into_iter().enumerate() {
- if let Some(DynamicColumn::IpAddr(Column { index: idx, values })) =
- dynamic_column_opt
- {
- column_indexes.push(idx);
- column_values.push(Some(values));
- } else {
- column_indexes.push(ColumnIndex::Empty {
- num_docs: num_docs_per_column[i],
- });
- column_values.push(None);
- }
- }
-
- let merged_column_index = crate::columnar::column_index::merge_column_index(
- &column_indexes[..],
- merge_row_order,
- );
- let merge_column_values = MergedColumnValues {
- column_indexes: &column_indexes[..],
- column_values: &column_values,
- merge_row_order,
- };
-
- serialize_column_mappable_to_u128(merged_column_index, &merge_column_values, wrt)?;
- }
- ColumnType::Bytes | ColumnType::Str => {
+ ColumnType::Bytes => {
let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());
let mut bytes_columns: Vec<Option<BytesColumn>> = Vec::with_capacity(columns.len());
- for (i, dynamic_column_opt) in columns.into_iter().enumerate() {
- match dynamic_column_opt {
- Some(DynamicColumn::Str(str_column)) => {
- column_indexes.push(str_column.term_ord_column.index.clone());
- bytes_columns.push(Some(str_column.into()));
- }
- Some(DynamicColumn::Bytes(bytes_column)) => {
- column_indexes.push(bytes_column.term_ord_column.index.clone());
- bytes_columns.push(Some(bytes_column));
- }
- _ => {
- column_indexes.push(ColumnIndex::Empty {
- num_docs: num_docs_per_column[i],
- });
- bytes_columns.push(None);
- }
+ for dynamic_column in columns.into_iter().flatten() {
+ if let DynamicColumn::Bytes(bytes_column) = dynamic_column {
+ column_indexes.push(bytes_column.term_ord_column.index.clone());
+ bytes_columns.push(Some(bytes_column));
}
}
- let merged_column_index = crate::columnar::column_index::merge_column_index(
- &column_indexes[..],
- merge_row_order,
- );
+ let merged_column_index =
+ crate::columnar::column_index::merge_column_index(merge_row_order);
merge_bytes_or_str_column(merged_column_index, &bytes_columns, merge_row_order, wrt)?;
}
}
@@ -286,10 +221,10 @@ impl GroupedColumnsHandle {
for (columnar_id, column) in self.columns.iter().enumerate() {
if let Some(column) = column {
let column = column.open()?;
+
// We skip columns that end up with 0 documents.
// That way, we make sure they don't end up influencing the merge type or
// creating empty columns.
-
if is_empty_after_merge(merge_row_order, &column, columnar_id) {
columns.push(None);
} else {
@@ -368,36 +303,7 @@ fn is_empty_after_merge(
if let Some(alive_bitset) = &shuffled.alive_bitsets[columnar_ord] {
let column_index = column.column_index();
match column_index {
- ColumnIndex::Empty { .. } => true,
ColumnIndex::Full => alive_bitset.len() == 0,
- ColumnIndex::Optional(optional_index) => {
- for doc in optional_index.iter_rows() {
- if alive_bitset.contains(doc) {
- return false;
- }
- }
- true
- }
- ColumnIndex::Multivalued(multivalued_index) => {
- for (doc_id, (start_index, end_index)) in multivalued_index
- .start_index_column
- .iter()
- .tuple_windows()
- .enumerate()
- {
- let doc_id = doc_id as u32;
- if start_index == end_index {
- // There are no values in this document
- continue;
- }
- // The document contains values and is present in the alive bitset.
- // The column is therefore not empty.
- if alive_bitset.contains(doc_id) {
- return false;
- }
- }
- true
- }
}
} else {
// No document is being deleted.
@@ -482,11 +388,7 @@ fn min_max_if_numerical(column: &DynamicColumn) -> Option<(NumericalValue, Numer
DynamicColumn::I64(column) => Some((column.min_value().into(), column.max_value().into())),
DynamicColumn::U64(column) => Some((column.min_value().into(), column.max_value().into())),
DynamicColumn::F64(column) => Some((column.min_value().into(), column.max_value().into())),
- DynamicColumn::Bool(_)
- | DynamicColumn::IpAddr(_)
- | DynamicColumn::DateTime(_)
- | DynamicColumn::Bytes(_)
- | DynamicColumn::Str(_) => None,
+ DynamicColumn::Bool(_) | DynamicColumn::DateTime(_) | DynamicColumn::Bytes(_) => None,
}
}
diff --git a/crates/tantivy/src/columnar/columnar/merge/tests.rs b/crates/tantivy/src/columnar/columnar/merge/tests.rs
index 0f77129d..da1bdde4 100644
--- a/crates/tantivy/src/columnar/columnar/merge/tests.rs
+++ b/crates/tantivy/src/columnar/columnar/merge/tests.rs
@@ -1,5 +1,5 @@
use super::*;
-use crate::columnar::{Cardinality, ColumnarWriter, HasAssociatedColumnType, RowId};
+use crate::columnar::{ColumnarWriter, HasAssociatedColumnType, RowId};
fn make_columnar<T: Into<NumericalValue> + HasAssociatedColumnType + Copy>(
column_name: &str,
@@ -73,28 +73,6 @@ fn test_group_columns_with_required_column() {
assert!(column_map.contains_key(&("numbers".to_string(), ColumnTypeCategory::Numerical)));
}
-#[test]
-fn test_group_columns_required_column_with_no_existing_columns() {
- let columnar1 = make_columnar("numbers", &[2u64]);
- let columnar2 = make_columnar("numbers", &[2u64]);
- let columnars = &[&columnar1, &columnar2];
- let merge_order = StackMergeOrder::stack(columnars).into();
- let column_map: BTreeMap<_, _> = group_columns_for_merge(
- columnars,
- &[("required_col".to_string(), ColumnType::Str)],
- &merge_order,
- )
- .unwrap();
- assert_eq!(column_map.len(), 2);
- let columns = &column_map
- .get(&("required_col".to_string(), ColumnTypeCategory::Str))
- .unwrap()
- .columns;
- assert_eq!(columns.len(), 2);
- assert!(columns[0].is_none());
- assert!(columns[1].is_none());
-}
-
#[test]
fn test_group_columns_required_column_is_above_all_columns_have_the_same_type_rule() {
let columnar1 = make_columnar("numbers", &[2i64]);
@@ -139,354 +117,3 @@ fn test_missing_column() {
assert!(columns[1].is_some());
}
}
-
-fn make_numerical_columnar_multiple_columns(
- columns: &[(&str, &[&[NumericalValue]])],
-) -> ColumnarReader {
- let mut dataframe_writer = ColumnarWriter::default();
- for (column_name, column_values) in columns {
- for (row_id, vals) in column_values.iter().enumerate() {
- for val in vals.iter() {
- dataframe_writer.record_numerical(row_id as u32, column_name, *val);
- }
- }
- }
- let num_rows = columns
- .iter()
- .map(|(_, val_rows)| val_rows.len() as RowId)
- .max()
- .unwrap_or(0u32);
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer
- .serialize(num_rows, None, &mut buffer)
- .unwrap();
- ColumnarReader::open(buffer).unwrap()
-}
-
-#[track_caller]
-fn make_byte_columnar_multiple_columns(
- columns: &[(&str, &[&[&[u8]]])],
- num_rows: u32,
-) -> ColumnarReader {
- let mut dataframe_writer = ColumnarWriter::default();
- for (column_name, column_values) in columns {
- assert_eq!(
- column_values.len(),
- num_rows as usize,
- "All columns must have `{num_rows}` rows"
- );
- for (row_id, vals) in column_values.iter().enumerate() {
- for val in vals.iter() {
- dataframe_writer.record_bytes(row_id as u32, column_name, val);
- }
- }
- }
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer
- .serialize(num_rows, None, &mut buffer)
- .unwrap();
- ColumnarReader::open(buffer).unwrap()
-}
-
-fn make_text_columnar_multiple_columns(columns: &[(&str, &[&[&str]])]) -> ColumnarReader {
- let mut dataframe_writer = ColumnarWriter::default();
- for (column_name, column_values) in columns {
- for (row_id, vals) in column_values.iter().enumerate() {
- for val in vals.iter() {
- dataframe_writer.record_str(row_id as u32, column_name, val);
- }
- }
- }
- let num_rows = columns
- .iter()
- .map(|(_, val_rows)| val_rows.len() as RowId)
- .max()
- .unwrap_or(0u32);
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer
- .serialize(num_rows, None, &mut buffer)
- .unwrap();
- ColumnarReader::open(buffer).unwrap()
-}
-
-#[test]
-fn test_merge_columnar_numbers() {
- let columnar1 =
- make_numerical_columnar_multiple_columns(&[("numbers", &[&[NumericalValue::from(-1f64)]])]);
- let columnar2 = make_numerical_columnar_multiple_columns(&[(
- "numbers",
- &[&[], &[NumericalValue::from(-3f64)]],
- )]);
- let mut buffer = Vec::new();
- let columnars = &[&columnar1, &columnar2];
- let stack_merge_order = StackMergeOrder::stack(columnars);
- crate::columnar::columnar::merge_columnar(
- columnars,
- &[],
- MergeRowOrder::Stack(stack_merge_order),
- &mut buffer,
- )
- .unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_rows(), 3);
- assert_eq!(columnar_reader.num_columns(), 1);
- let cols = columnar_reader.read_columns("numbers").unwrap();
- let dynamic_column = cols[0].open().unwrap();
- let DynamicColumn::F64(vals) = dynamic_column else {
- panic!()
- };
- assert_eq!(vals.get_cardinality(), Cardinality::Optional);
- assert_eq!(vals.first(0u32), Some(-1f64));
- assert_eq!(vals.first(1u32), None);
- assert_eq!(vals.first(2u32), Some(-3f64));
-}
-
-#[test]
-fn test_merge_columnar_texts() {
- let columnar1 = make_text_columnar_multiple_columns(&[("texts", &[&["a"]])]);
- let columnar2 = make_text_columnar_multiple_columns(&[("texts", &[&[], &["b"]])]);
- let mut buffer = Vec::new();
- let columnars = &[&columnar1, &columnar2];
- let stack_merge_order = StackMergeOrder::stack(columnars);
- crate::columnar::columnar::merge_columnar(
- columnars,
- &[],
- MergeRowOrder::Stack(stack_merge_order),
- &mut buffer,
- )
- .unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_rows(), 3);
- assert_eq!(columnar_reader.num_columns(), 1);
- let cols = columnar_reader.read_columns("texts").unwrap();
- let dynamic_column = cols[0].open().unwrap();
- let DynamicColumn::Str(vals) = dynamic_column else {
- panic!()
- };
- assert_eq!(vals.ords().get_cardinality(), Cardinality::Optional);
-
- let get_str_for_ord = |ord| {
- let mut out = String::new();
- vals.ord_to_str(ord, &mut out).unwrap();
- out
- };
-
- assert_eq!(vals.dictionary.num_terms(), 2);
- assert_eq!(get_str_for_ord(0), "a");
- assert_eq!(get_str_for_ord(1), "b");
-
- let get_str_for_row = |row_id| {
- let term_ords: Vec<u64> = vals.term_ords(row_id).collect();
- assert!(term_ords.len() <= 1);
- let mut out = String::new();
- if term_ords.len() == 1 {
- vals.ord_to_str(term_ords[0], &mut out).unwrap();
- }
- out
- };
-
- assert_eq!(get_str_for_row(0), "a");
- assert_eq!(get_str_for_row(1), "");
- assert_eq!(get_str_for_row(2), "b");
-}
-
-#[test]
-fn test_merge_columnar_byte() {
- let columnar1 = make_byte_columnar_multiple_columns(&[("bytes", &[&[b"bbbb"], &[b"baaa"]])], 2);
- let columnar2 = make_byte_columnar_multiple_columns(&[("bytes", &[&[], &[b"a"]])], 2);
- let mut buffer = Vec::new();
- let columnars = &[&columnar1, &columnar2];
- let stack_merge_order = StackMergeOrder::stack(columnars);
- crate::columnar::columnar::merge_columnar(
- columnars,
- &[],
- MergeRowOrder::Stack(stack_merge_order),
- &mut buffer,
- )
- .unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_rows(), 4);
- assert_eq!(columnar_reader.num_columns(), 1);
- let cols = columnar_reader.read_columns("bytes").unwrap();
- let dynamic_column = cols[0].open().unwrap();
- let DynamicColumn::Bytes(vals) = dynamic_column else {
- panic!()
- };
- let get_bytes_for_ord = |ord| {
- let mut out = Vec::new();
- vals.ord_to_bytes(ord, &mut out).unwrap();
- out
- };
-
- assert_eq!(vals.dictionary.num_terms(), 3);
- assert_eq!(get_bytes_for_ord(0), b"a");
- assert_eq!(get_bytes_for_ord(1), b"baaa");
- assert_eq!(get_bytes_for_ord(2), b"bbbb");
-
- let get_bytes_for_row = |row_id| {
- let term_ords: Vec<u64> = vals.term_ords(row_id).collect();
- assert!(term_ords.len() <= 1);
- let mut out = Vec::new();
- if term_ords.len() == 1 {
- vals.ord_to_bytes(term_ords[0], &mut out).unwrap();
- }
- out
- };
-
- assert_eq!(get_bytes_for_row(0), b"bbbb");
- assert_eq!(get_bytes_for_row(1), b"baaa");
- assert_eq!(get_bytes_for_row(2), b"");
- assert_eq!(get_bytes_for_row(3), b"a");
-}
-
-#[test]
-fn test_merge_columnar_byte_with_missing() {
- let columnar1 = make_byte_columnar_multiple_columns(&[], 3);
- let columnar2 = make_byte_columnar_multiple_columns(&[("col", &[&[b"b"], &[]])], 2);
- let columnar3 = make_byte_columnar_multiple_columns(
- &[
- ("col", &[&[], &[b"b"], &[b"a", b"b"]]),
- ("col2", &[&[b"hello"], &[], &[b"a", b"b"]]),
- ],
- 3,
- );
- let mut buffer = Vec::new();
- let columnars = &[&columnar1, &columnar2, &columnar3];
- let stack_merge_order = StackMergeOrder::stack(columnars);
- crate::columnar::columnar::merge_columnar(
- columnars,
- &[],
- MergeRowOrder::Stack(stack_merge_order),
- &mut buffer,
- )
- .unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_rows(), 3 + 2 + 3);
- assert_eq!(columnar_reader.num_columns(), 2);
- let cols = columnar_reader.read_columns("col").unwrap();
- let dynamic_column = cols[0].open().unwrap();
- let DynamicColumn::Bytes(vals) = dynamic_column else {
- panic!()
- };
- let get_bytes_for_ord = |ord| {
- let mut out = Vec::new();
- vals.ord_to_bytes(ord, &mut out).unwrap();
- out
- };
- assert_eq!(vals.dictionary.num_terms(), 2);
- assert_eq!(get_bytes_for_ord(0), b"a");
- assert_eq!(get_bytes_for_ord(1), b"b");
- let get_bytes_for_row = |row_id| {
- let terms: Vec<Vec<u8>> = vals
- .term_ords(row_id)
- .map(|term_ord| {
- let mut out = Vec::new();
- vals.ord_to_bytes(term_ord, &mut out).unwrap();
- out
- })
- .collect();
- terms
- };
- assert!(get_bytes_for_row(0).is_empty());
- assert!(get_bytes_for_row(1).is_empty());
- assert!(get_bytes_for_row(2).is_empty());
- assert_eq!(get_bytes_for_row(3), vec![b"b".to_vec()]);
- assert!(get_bytes_for_row(4).is_empty());
- assert!(get_bytes_for_row(5).is_empty());
- assert_eq!(get_bytes_for_row(6), vec![b"b".to_vec()]);
- assert_eq!(get_bytes_for_row(7), vec![b"a".to_vec(), b"b".to_vec()]);
-}
-
-#[test]
-fn test_merge_columnar_different_types() {
- let columnar1 = make_text_columnar_multiple_columns(&[("mixed", &[&["a"]])]);
- let columnar2 = make_text_columnar_multiple_columns(&[("mixed", &[&[], &["b"]])]);
- let columnar3 = make_columnar("mixed", &[1i64]);
- let mut buffer = Vec::new();
- let columnars = &[&columnar1, &columnar2, &columnar3];
- let stack_merge_order = StackMergeOrder::stack(columnars);
- crate::columnar::columnar::merge_columnar(
- columnars,
- &[],
- MergeRowOrder::Stack(stack_merge_order),
- &mut buffer,
- )
- .unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_rows(), 4);
- assert_eq!(columnar_reader.num_columns(), 2);
- let cols = columnar_reader.read_columns("mixed").unwrap();
-
- // numeric column
- let dynamic_column = cols[0].open().unwrap();
- let DynamicColumn::I64(vals) = dynamic_column else {
- panic!()
- };
- assert_eq!(vals.get_cardinality(), Cardinality::Optional);
- assert_eq!(vals.values_for_doc(0).collect_vec(), Vec::<i64>::new());
- assert_eq!(vals.values_for_doc(1).collect_vec(), Vec::<i64>::new());
- assert_eq!(vals.values_for_doc(2).collect_vec(), Vec::<i64>::new());
- assert_eq!(vals.values_for_doc(3).collect_vec(), vec![1]);
- assert_eq!(vals.values_for_doc(4).collect_vec(), Vec::<i64>::new());
-
- // text column
- let dynamic_column = cols[1].open().unwrap();
- let DynamicColumn::Str(vals) = dynamic_column else {
- panic!()
- };
- assert_eq!(vals.ords().get_cardinality(), Cardinality::Optional);
- let get_str_for_ord = |ord| {
- let mut out = String::new();
- vals.ord_to_str(ord, &mut out).unwrap();
- out
- };
-
- assert_eq!(vals.dictionary.num_terms(), 2);
- assert_eq!(get_str_for_ord(0), "a");
- assert_eq!(get_str_for_ord(1), "b");
-
- let get_str_for_row = |row_id| {
- let term_ords: Vec<String> = vals
- .term_ords(row_id)
- .map(|el| {
- let mut out = String::new();
- vals.ord_to_str(el, &mut out).unwrap();
- out
- })
- .collect();
- term_ords
- };
-
- assert_eq!(get_str_for_row(0), vec!["a".to_string()]);
- assert_eq!(get_str_for_row(1), Vec::<String>::new());
- assert_eq!(get_str_for_row(2), vec!["b".to_string()]);
- assert_eq!(get_str_for_row(3), Vec::<String>::new());
-}
-
-#[test]
-fn test_merge_columnar_different_empty_cardinality() {
- let columnar1 = make_text_columnar_multiple_columns(&[("mixed", &[&["a"]])]);
- let columnar2 = make_columnar("mixed", &[1i64]);
- let mut buffer = Vec::new();
- let columnars = &[&columnar1, &columnar2];
- let stack_merge_order = StackMergeOrder::stack(columnars);
- crate::columnar::columnar::merge_columnar(
- columnars,
- &[],
- MergeRowOrder::Stack(stack_merge_order),
- &mut buffer,
- )
- .unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_rows(), 2);
- assert_eq!(columnar_reader.num_columns(), 2);
- let cols = columnar_reader.read_columns("mixed").unwrap();
-
- // numeric column
- let dynamic_column = cols[0].open().unwrap();
- assert_eq!(dynamic_column.get_cardinality(), Cardinality::Optional);
-
- // text column
- let dynamic_column = cols[1].open().unwrap();
- assert_eq!(dynamic_column.get_cardinality(), Cardinality::Optional);
-}
diff --git a/crates/tantivy/src/columnar/columnar/mod.rs b/crates/tantivy/src/columnar/columnar/mod.rs
index bb9e485b..a33cce8f 100644
--- a/crates/tantivy/src/columnar/columnar/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/mod.rs
@@ -6,8 +6,6 @@ mod writer;
pub use column_type::{ColumnType, HasAssociatedColumnType};
pub use format_version::{Version, CURRENT_VERSION};
-#[cfg(test)]
-pub(crate) use merge::ColumnTypeCategory;
pub use merge::{merge_columnar, MergeRowOrder, ShuffleMergeOrder, StackMergeOrder};
pub use reader::ColumnarReader;
pub use writer::ColumnarWriter;
diff --git a/crates/tantivy/src/columnar/columnar/reader/mod.rs b/crates/tantivy/src/columnar/columnar/reader/mod.rs
index 662c2f32..95fe48a9 100644
--- a/crates/tantivy/src/columnar/columnar/reader/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/reader/mod.rs
@@ -197,7 +197,7 @@ mod tests {
#[test]
fn test_list_columns() {
let mut columnar_writer = ColumnarWriter::default();
- columnar_writer.record_column_type("col1", ColumnType::Str, false);
+ columnar_writer.record_column_type("col1", ColumnType::U64, false);
columnar_writer.record_column_type("col2", ColumnType::U64, false);
let mut buffer = Vec::new();
columnar_writer.serialize(1, None, &mut buffer).unwrap();
@@ -205,7 +205,7 @@ mod tests {
let columns = columnar.list_columns().unwrap();
assert_eq!(columns.len(), 2);
assert_eq!(&columns[0].0, "col1");
- assert_eq!(columns[0].1.column_type(), ColumnType::Str);
+ assert_eq!(columns[0].1.column_type(), ColumnType::U64);
assert_eq!(&columns[1].0, "col2");
assert_eq!(columns[1].1.column_type(), ColumnType::U64);
}
diff --git a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
index d66e3501..f9e35fb5 100644
--- a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
@@ -71,22 +71,15 @@ impl ColumnWriter {
}
/// Records a change of the document being recorded.
- ///
- /// This function will also update the cardinality of the column
- /// if necessary.
pub(super) fn record<S: SymbolValue>(&mut self, doc: RowId, value: S, arena: &mut MemoryArena) {
// Difference between `doc` and the last doc.
match delta_with_last_doc(self.last_doc_opt, doc) {
- DocumentStep::Same => {
- // This is the last encounterred document.
- self.cardinality = Cardinality::Multivalued;
- }
+ DocumentStep::Same => {}
DocumentStep::Next => {
self.last_doc_opt = Some(doc);
self.write_symbol::<S>(ColumnOperation::NewDoc(doc), arena);
}
DocumentStep::Skipped => {
- self.cardinality = self.cardinality.max(Cardinality::Optional);
self.last_doc_opt = Some(doc);
self.write_symbol::<S>(ColumnOperation::NewDoc(doc), arena);
}
@@ -94,15 +87,8 @@ impl ColumnWriter {
self.write_symbol(ColumnOperation::Value(value), arena);
}
- // Get the cardinality.
- // The overall number of docs in the column is necessary to
- // deal with the case where the all docs contain 1 value, except some documents
- // at the end of the column.
- pub(crate) fn get_cardinality(&self, num_docs: RowId) -> Cardinality {
- match delta_with_last_doc(self.last_doc_opt, num_docs) {
- DocumentStep::Same | DocumentStep::Next => self.cardinality,
- DocumentStep::Skipped => self.cardinality.max(Cardinality::Optional),
- }
+ pub(crate) fn get_cardinality(&self) -> Cardinality {
+ self.cardinality
}
/// Appends a new symbol to the `ColumnWriter`.
@@ -214,8 +200,8 @@ impl NumericalColumnWriter {
self.compatible_numerical_types.to_numerical_type()
}
- pub fn cardinality(&self, num_docs: RowId) -> Cardinality {
- self.column_writer.get_cardinality(num_docs)
+ pub fn cardinality(&self) -> Cardinality {
+ self.column_writer.get_cardinality()
}
pub fn record_numerical_value(
diff --git a/crates/tantivy/src/columnar/columnar/writer/mod.rs b/crates/tantivy/src/columnar/columnar/writer/mod.rs
index 4556047d..65da1134 100644
--- a/crates/tantivy/src/columnar/columnar/writer/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/mod.rs
@@ -4,7 +4,6 @@ mod serializer;
mod value_index;
use std::io;
-use std::net::Ipv6Addr;
use crate::common::CountingWriter;
use crate::stacker::{Addr, ArenaHashMap, MemoryArena};
@@ -13,7 +12,7 @@ pub(crate) use column_writers::CompatibleNumericalTypes;
pub(crate) use serializer::ColumnarSerializer;
use crate::columnar::column_index::SerializableColumnIndex;
-use crate::columnar::column_values::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};
+use crate::columnar::column_values::MonotonicallyMappableToU64;
use crate::columnar::columnar::column_type::ColumnType;
use crate::columnar::columnar::writer::column_writers::{
ColumnWriter, NumericalColumnWriter, StrOrBytesColumnWriter,
@@ -29,7 +28,6 @@ use crate::columnar::{Cardinality, RowId};
struct SpareBuffers {
value_index_builders: PreallocatedIndexBuilders,
u64_values: Vec<u64>,
- ip_addr_values: Vec<Ipv6Addr>,
}
/// Makes it possible to create a new columnar.
@@ -38,9 +36,7 @@ struct SpareBuffers {
/// use tantivy::columnar::ColumnarWriter;
///
/// let mut columnar_writer = ColumnarWriter::default();
-/// columnar_writer.record_str(0u32 /* doc id */, "product_name", "Red backpack");
/// columnar_writer.record_numerical(0u32 /* doc id */, "price", 10u64);
-/// columnar_writer.record_str(1u32 /* doc id */, "product_name", "Apple");
/// columnar_writer.record_numerical(0u32 /* doc id */, "price", 10.5f64); //< uh oh we ended up mixing integer and floats.
/// let mut wrt: Vec<u8> = Vec::new();
/// columnar_writer.serialize(2u32, None, &mut wrt).unwrap();
@@ -145,20 +141,14 @@ impl ColumnarWriter {
) {
if sort_values_within_row {
assert!(
- column_type == ColumnType::Bytes || column_type == ColumnType::Str,
- "sort_values_within_row is only allowed for Bytes and Str columns",
+ column_type == ColumnType::Bytes,
+ "sort_values_within_row is only allowed for Bytes columns",
);
}
match column_type {
- ColumnType::Str | ColumnType::Bytes => {
- let (hash_map, dictionaries) = (
- if column_type == ColumnType::Str {
- &mut self.str_field_hash_map
- } else {
- &mut self.bytes_field_hash_map
- },
- &mut self.dictionaries,
- );
+ ColumnType::Bytes => {
+ let (hash_map, dictionaries) =
+ (&mut self.bytes_field_hash_map, &mut self.dictionaries);
hash_map.mutate_or_create(
column_name.as_bytes(),
|column_opt: Option<StrOrBytesColumnWriter>| {
@@ -197,10 +187,6 @@ impl ColumnarWriter {
},
);
}
- ColumnType::IpAddr => self.ip_addr_field_hash_map.mutate_or_create(
- column_name.as_bytes(),
- |column_opt: Option<ColumnWriter>| column_opt.unwrap_or_default(),
- ),
}
}
@@ -221,18 +207,6 @@ impl ColumnarWriter {
);
}
- pub fn record_ip_addr(&mut self, doc: RowId, column_name: &str, ip_addr: Ipv6Addr) {
- let (hash_map, arena) = (&mut self.ip_addr_field_hash_map, &mut self.arena);
- hash_map.mutate_or_create(
- column_name.as_bytes(),
- |column_opt: Option<ColumnWriter>| {
- let mut column: ColumnWriter = column_opt.unwrap_or_default();
- column.record(doc, ip_addr, arena);
- column
- },
- );
- }
-
pub fn record_bool(&mut self, doc: RowId, column_name: &str, val: bool) {
let (hash_map, arena) = (&mut self.bool_field_hash_map, &mut self.arena);
hash_map.mutate_or_create(
@@ -266,27 +240,6 @@ impl ColumnarWriter {
);
}
- pub fn record_str(&mut self, doc: RowId, column_name: &str, value: &str) {
- let (hash_map, arena, dictionaries) = (
- &mut self.str_field_hash_map,
- &mut self.arena,
- &mut self.dictionaries,
- );
- hash_map.mutate_or_create(
- column_name.as_bytes(),
- |column_opt: Option<StrOrBytesColumnWriter>| {
- let mut column: StrOrBytesColumnWriter = column_opt.unwrap_or_else(|| {
- // Each column has its own dictionary
- let dictionary_id = dictionaries.len() as u32;
- dictionaries.push(DictionaryBuilder::default());
- StrOrBytesColumnWriter::with_dictionary_id(dictionary_id)
- });
- column.record_bytes(doc, value.as_bytes(), dictionaries, arena);
- column
- },
- );
- }
-
pub fn record_bytes(&mut self, doc: RowId, column_name: &str, value: &[u8]) {
let (hash_map, arena, dictionaries) = (
&mut self.bytes_field_hash_map,
@@ -329,21 +282,11 @@ impl ColumnarWriter {
.iter()
.map(|(term, addr)| (term, ColumnType::Bytes, addr)),
);
- columns.extend(
- self.str_field_hash_map
- .iter()
- .map(|(column_name, addr)| (column_name, ColumnType::Str, addr)),
- );
columns.extend(
self.bool_field_hash_map
.iter()
.map(|(column_name, addr)| (column_name, ColumnType::Bool, addr)),
);
- columns.extend(
- self.ip_addr_field_hash_map
- .iter()
- .map(|(column_name, addr)| (column_name, ColumnType::IpAddr, addr)),
- );
columns.extend(
self.datetime_field_hash_map
.iter()
@@ -357,12 +300,11 @@ impl ColumnarWriter {
match column_type {
ColumnType::Bool => {
let column_writer: ColumnWriter = self.bool_field_hash_map.read(addr);
- let cardinality = column_writer.get_cardinality(num_docs);
+ let cardinality = column_writer.get_cardinality();
let mut column_serializer =
serializer.start_serialize_column(column_name, column_type);
serialize_bool_column(
cardinality,
- num_docs,
column_writer.operation_iterator(
arena,
old_to_new_row_ids,
@@ -373,42 +315,16 @@ impl ColumnarWriter {
)?;
column_serializer.finalize()?;
}
- ColumnType::IpAddr => {
- let column_writer: ColumnWriter = self.ip_addr_field_hash_map.read(addr);
- let cardinality = column_writer.get_cardinality(num_docs);
- let mut column_serializer =
- serializer.start_serialize_column(column_name, ColumnType::IpAddr);
- serialize_ip_addr_column(
- cardinality,
- num_docs,
- column_writer.operation_iterator(
- arena,
- old_to_new_row_ids,
- &mut symbol_byte_buffer,
- ),
- buffers,
- &mut column_serializer,
- )?;
- column_serializer.finalize()?;
- }
- ColumnType::Bytes | ColumnType::Str => {
+ ColumnType::Bytes => {
let str_or_bytes_column_writer: StrOrBytesColumnWriter =
- if column_type == ColumnType::Bytes {
- self.bytes_field_hash_map.read(addr)
- } else {
- self.str_field_hash_map.read(addr)
- };
+ self.bytes_field_hash_map.read(addr);
let dictionary_builder =
&dictionaries[str_or_bytes_column_writer.dictionary_id as usize];
- let cardinality = str_or_bytes_column_writer
- .column_writer
- .get_cardinality(num_docs);
+ let cardinality = str_or_bytes_column_writer.column_writer.get_cardinality();
let mut column_serializer =
serializer.start_serialize_column(column_name, column_type);
serialize_bytes_or_str_column(
cardinality,
- num_docs,
- str_or_bytes_column_writer.sort_values_within_row,
dictionary_builder,
str_or_bytes_column_writer.operation_iterator(
arena,
@@ -424,13 +340,12 @@ impl ColumnarWriter {
ColumnType::F64 | ColumnType::I64 | ColumnType::U64 => {
let numerical_column_writer: NumericalColumnWriter =
self.numerical_field_hash_map.read(addr);
- let cardinality = numerical_column_writer.cardinality(num_docs);
+ let cardinality = numerical_column_writer.cardinality();
let mut column_serializer =
serializer.start_serialize_column(column_name, column_type);
let numerical_type = column_type.numerical_type().unwrap();
serialize_numerical_column(
cardinality,
- num_docs,
numerical_type,
numerical_column_writer.operation_iterator(
arena,
@@ -444,12 +359,11 @@ impl ColumnarWriter {
}
ColumnType::DateTime => {
let column_writer: ColumnWriter = self.datetime_field_hash_map.read(addr);
- let cardinality = column_writer.get_cardinality(num_docs);
+ let cardinality = column_writer.get_cardinality();
let mut column_serializer =
serializer.start_serialize_column(column_name, ColumnType::DateTime);
serialize_numerical_column(
cardinality,
- num_docs,
NumericalType::I64,
column_writer.operation_iterator(
arena,
@@ -473,8 +387,6 @@ impl ColumnarWriter {
#[allow(clippy::too_many_arguments)]
fn serialize_bytes_or_str_column(
cardinality: Cardinality,
- num_docs: RowId,
- sort_values_within_row: bool,
dictionary_builder: &DictionaryBuilder,
operation_it: impl Iterator<Item = ColumnOperation<UnorderedId>>,
buffers: &mut SpareBuffers,
@@ -504,8 +416,6 @@ fn serialize_bytes_or_str_column(
send_to_serialize_column_mappable_to_u64(
operation_iterator,
cardinality,
- num_docs,
- sort_values_within_row,
value_index_builders,
u64_values,
&mut wrt,
@@ -516,7 +426,6 @@ fn serialize_bytes_or_str_column(
fn serialize_numerical_column(
cardinality: Cardinality,
- num_docs: RowId,
numerical_type: NumericalType,
op_iterator: impl Iterator<Item = ColumnOperation<NumericalValue>>,
buffers: &mut SpareBuffers,
@@ -532,8 +441,6 @@ fn serialize_numerical_column(
send_to_serialize_column_mappable_to_u64(
coerce_numerical_symbol::<i64>(op_iterator),
cardinality,
- num_docs,
- false,
value_index_builders,
u64_values,
wrt,
@@ -543,8 +450,6 @@ fn serialize_numerical_column(
send_to_serialize_column_mappable_to_u64(
coerce_numerical_symbol::<u64>(op_iterator),
cardinality,
- num_docs,
- false,
value_index_builders,
u64_values,
wrt,
@@ -554,8 +459,6 @@ fn serialize_numerical_column(
send_to_serialize_column_mappable_to_u64(
coerce_numerical_symbol::<f64>(op_iterator),
cardinality,
- num_docs,
- false,
value_index_builders,
u64_values,
wrt,
@@ -567,7 +470,6 @@ fn serialize_numerical_column(
fn serialize_bool_column(
cardinality: Cardinality,
- num_docs: RowId,
column_operations_it: impl Iterator<Item = ColumnOperation<bool>>,
buffers: &mut SpareBuffers,
wrt: &mut impl io::Write,
@@ -583,8 +485,6 @@ fn serialize_bool_column(
ColumnOperation::Value(bool_val) => ColumnOperation::Value(bool_val.to_u64()),
}),
cardinality,
- num_docs,
- false,
value_index_builders,
u64_values,
wrt,
@@ -592,88 +492,9 @@ fn serialize_bool_column(
Ok(())
}
-fn serialize_ip_addr_column(
- cardinality: Cardinality,
- num_docs: RowId,
- column_operations_it: impl Iterator<Item = ColumnOperation<Ipv6Addr>>,
- buffers: &mut SpareBuffers,
- wrt: &mut impl io::Write,
-) -> io::Result<()> {
- let SpareBuffers {
- value_index_builders,
- ip_addr_values,
- ..
- } = buffers;
- send_to_serialize_column_mappable_to_u128(
- column_operations_it,
- cardinality,
- num_docs,
- value_index_builders,
- ip_addr_values,
- wrt,
- )?;
- Ok(())
-}
-
-fn send_to_serialize_column_mappable_to_u128<
- T: Copy + Ord + std::fmt::Debug + Send + Sync + MonotonicallyMappableToU128 + PartialOrd,
->(
- op_iterator: impl Iterator<Item = ColumnOperation<T>>,
- cardinality: Cardinality,
- num_rows: RowId,
- value_index_builders: &mut PreallocatedIndexBuilders,
- values: &mut Vec<T>,
- mut wrt: impl io::Write,
-) -> io::Result<()> {
- values.clear();
- // TODO: split index and values
- let serializable_column_index = match cardinality {
- Cardinality::Full => {
- consume_operation_iterator(
- op_iterator,
- value_index_builders.borrow_required_index_builder(),
- values,
- );
- SerializableColumnIndex::Full
- }
- Cardinality::Optional => {
- let optional_index_builder = value_index_builders.borrow_optional_index_builder();
- consume_operation_iterator(op_iterator, optional_index_builder, values);
- let optional_index = optional_index_builder.finish(num_rows);
- SerializableColumnIndex::Optional {
- num_rows,
- non_null_row_ids: Box::new(optional_index),
- }
- }
- Cardinality::Multivalued => {
- let multivalued_index_builder = value_index_builders.borrow_multivalued_index_builder();
- consume_operation_iterator(op_iterator, multivalued_index_builder, values);
- let multivalued_index = multivalued_index_builder.finish(num_rows);
- SerializableColumnIndex::Multivalued(Box::new(multivalued_index))
- }
- };
- crate::columnar::column::serialize_column_mappable_to_u128(
- serializable_column_index,
- &&values[..],
- &mut wrt,
- )?;
- Ok(())
-}
-
-fn sort_values_within_row_in_place(multivalued_index: &[RowId], values: &mut [u64]) {
- let mut start_index: usize = 0;
- for end_index in multivalued_index.iter().copied() {
- let end_index = end_index as usize;
- values[start_index..end_index].sort_unstable();
- start_index = end_index;
- }
-}
-
fn send_to_serialize_column_mappable_to_u64(
op_iterator: impl Iterator<Item = ColumnOperation<u64>>,
cardinality: Cardinality,
- num_rows: RowId,
- sort_values_within_row: bool,
value_index_builders: &mut PreallocatedIndexBuilders,
values: &mut Vec<u64>,
mut wrt: impl io::Write,
@@ -688,24 +509,6 @@ fn send_to_serialize_column_mappable_to_u64(
);
SerializableColumnIndex::Full
}
- Cardinality::Optional => {
- let optional_index_builder = value_index_builders.borrow_optional_index_builder();
- consume_operation_iterator(op_iterator, optional_index_builder, values);
- let optional_index = optional_index_builder.finish(num_rows);
- SerializableColumnIndex::Optional {
- non_null_row_ids: Box::new(optional_index),
- num_rows,
- }
- }
- Cardinality::Multivalued => {
- let multivalued_index_builder = value_index_builders.borrow_multivalued_index_builder();
- consume_operation_iterator(op_iterator, multivalued_index_builder, values);
- let multivalued_index = multivalued_index_builder.finish(num_rows);
- if sort_values_within_row {
- sort_values_within_row_in_place(multivalued_index, values);
- }
- SerializableColumnIndex::Multivalued(Box::new(multivalued_index))
- }
};
crate::columnar::column::serialize_column_mappable_to_u64(
serializable_column_index,
@@ -761,7 +564,7 @@ mod tests {
column_writer.record(0u32, NumericalValue::from(14i64), &mut arena);
column_writer.record(1u32, NumericalValue::from(15i64), &mut arena);
column_writer.record(2u32, NumericalValue::from(-16i64), &mut arena);
- assert_eq!(column_writer.get_cardinality(3), Cardinality::Full);
+ assert_eq!(column_writer.get_cardinality(), Cardinality::Full);
let mut buffer = Vec::new();
let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer
.operation_iterator(&arena, None, &mut buffer)
@@ -783,69 +586,4 @@ mod tests {
ColumnOperation::Value(NumericalValue::I64(-16i64))
));
}
-
- #[test]
- fn test_column_writer_optional_cardinality_missing_first() {
- let mut arena = MemoryArena::default();
- let mut column_writer = super::ColumnWriter::default();
- column_writer.record(1u32, NumericalValue::from(15i64), &mut arena);
- column_writer.record(2u32, NumericalValue::from(-16i64), &mut arena);
- assert_eq!(column_writer.get_cardinality(3), Cardinality::Optional);
- let mut buffer = Vec::new();
- let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer
- .operation_iterator(&arena, None, &mut buffer)
- .collect();
- assert_eq!(symbols.len(), 4);
- assert!(matches!(symbols[0], ColumnOperation::NewDoc(1u32)));
- assert!(matches!(
- symbols[1],
- ColumnOperation::Value(NumericalValue::I64(15i64))
- ));
- assert!(matches!(symbols[2], ColumnOperation::NewDoc(2u32)));
- assert!(matches!(
- symbols[3],
- ColumnOperation::Value(NumericalValue::I64(-16i64))
- ));
- }
-
- #[test]
- fn test_column_writer_optional_cardinality_missing_last() {
- let mut arena = MemoryArena::default();
- let mut column_writer = super::ColumnWriter::default();
- column_writer.record(0u32, NumericalValue::from(15i64), &mut arena);
- assert_eq!(column_writer.get_cardinality(2), Cardinality::Optional);
- let mut buffer = Vec::new();
- let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer
- .operation_iterator(&arena, None, &mut buffer)
- .collect();
- assert_eq!(symbols.len(), 2);
- assert!(matches!(symbols[0], ColumnOperation::NewDoc(0u32)));
- assert!(matches!(
- symbols[1],
- ColumnOperation::Value(NumericalValue::I64(15i64))
- ));
- }
-
- #[test]
- fn test_column_writer_multivalued() {
- let mut arena = MemoryArena::default();
- let mut column_writer = super::ColumnWriter::default();
- column_writer.record(0u32, NumericalValue::from(16i64), &mut arena);
- column_writer.record(0u32, NumericalValue::from(17i64), &mut arena);
- assert_eq!(column_writer.get_cardinality(1), Cardinality::Multivalued);
- let mut buffer = Vec::new();
- let symbols: Vec<ColumnOperation<NumericalValue>> = column_writer
- .operation_iterator(&arena, None, &mut buffer)
- .collect();
- assert_eq!(symbols.len(), 3);
- assert!(matches!(symbols[0], ColumnOperation::NewDoc(0u32)));
- assert!(matches!(
- symbols[1],
- ColumnOperation::Value(NumericalValue::I64(16i64))
- ));
- assert!(matches!(
- symbols[2],
- ColumnOperation::Value(NumericalValue::I64(17i64))
- ));
- }
}
diff --git a/crates/tantivy/src/columnar/columnar/writer/serializer.rs b/crates/tantivy/src/columnar/columnar/writer/serializer.rs
index 4aea5e6d..627fea20 100644
--- a/crates/tantivy/src/columnar/columnar/writer/serializer.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/serializer.rs
@@ -98,18 +98,3 @@ impl<'a, W: io::Write> io::Write for ColumnSerializer<'a, W> {
self.columnar_serializer.wrt.write_all(buf)
}
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_prepare_key_bytes() {
- let mut buffer: Vec<u8> = b"somegarbage".to_vec();
- prepare_key(b"root\0child", ColumnType::Str, &mut buffer);
- assert_eq!(buffer.len(), 12);
- assert_eq!(&buffer[..10], b"root0child");
- assert_eq!(buffer[10], 0u8);
- assert_eq!(buffer[11], ColumnType::Str.to_code());
- }
-}
diff --git a/crates/tantivy/src/columnar/dynamic_column.rs b/crates/tantivy/src/columnar/dynamic_column.rs
index 5d7118a6..f0674634 100644
--- a/crates/tantivy/src/columnar/dynamic_column.rs
+++ b/crates/tantivy/src/columnar/dynamic_column.rs
@@ -1,14 +1,13 @@
-use std::net::Ipv6Addr;
use std::sync::Arc;
use std::{fmt, io};
use crate::common::file_slice::FileSlice;
use crate::common::{ByteCount, DateTime, HasLen, OwnedBytes};
-use super::column::{BytesColumn, Column, StrColumn};
+use super::column::{BytesColumn, Column};
use super::column_values::{monotonic_map_column, StrictlyMonotonicFn};
use super::columnar::ColumnType;
-use super::{Cardinality, ColumnIndex, ColumnValues, NumericalType};
+use super::{ColumnIndex, ColumnValues, NumericalType};
#[derive(Clone)]
pub enum DynamicColumn {
@@ -16,24 +15,20 @@ pub enum DynamicColumn {
I64(Column<i64>),
U64(Column<u64>),
F64(Column<f64>),
- IpAddr(Column<Ipv6Addr>),
DateTime(Column<DateTime>),
Bytes(BytesColumn),
- Str(StrColumn),
}
impl fmt::Debug for DynamicColumn {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "[{} {} |", self.get_cardinality(), self.column_type())?;
+ write!(f, "[{} |", self.column_type())?;
match self {
DynamicColumn::Bool(col) => write!(f, " {col:?}")?,
DynamicColumn::I64(col) => write!(f, " {col:?}")?,
DynamicColumn::U64(col) => write!(f, " {col:?}")?,
DynamicColumn::F64(col) => write!(f, "{col:?}")?,
- DynamicColumn::IpAddr(col) => write!(f, "{col:?}")?,
DynamicColumn::DateTime(col) => write!(f, "{col:?}")?,
DynamicColumn::Bytes(col) => write!(f, "{col:?}")?,
- DynamicColumn::Str(col) => write!(f, "{col:?}")?,
}
write!(f, "]")
}
@@ -46,27 +41,19 @@ impl DynamicColumn {
DynamicColumn::I64(c) => &c.index,
DynamicColumn::U64(c) => &c.index,
DynamicColumn::F64(c) => &c.index,
- DynamicColumn::IpAddr(c) => &c.index,
DynamicColumn::DateTime(c) => &c.index,
DynamicColumn::Bytes(c) => &c.ords().index,
- DynamicColumn::Str(c) => &c.ords().index,
}
}
- pub fn get_cardinality(&self) -> Cardinality {
- self.column_index().get_cardinality()
- }
-
pub fn num_values(&self) -> u32 {
match self {
DynamicColumn::Bool(c) => c.values.num_vals(),
DynamicColumn::I64(c) => c.values.num_vals(),
DynamicColumn::U64(c) => c.values.num_vals(),
DynamicColumn::F64(c) => c.values.num_vals(),
- DynamicColumn::IpAddr(c) => c.values.num_vals(),
DynamicColumn::DateTime(c) => c.values.num_vals(),
DynamicColumn::Bytes(c) => c.ords().values.num_vals(),
- DynamicColumn::Str(c) => c.ords().values.num_vals(),
}
}
@@ -76,10 +63,8 @@ impl DynamicColumn {
DynamicColumn::I64(_) => ColumnType::I64,
DynamicColumn::U64(_) => ColumnType::U64,
DynamicColumn::F64(_) => ColumnType::F64,
- DynamicColumn::IpAddr(_) => ColumnType::IpAddr,
DynamicColumn::DateTime(_) => ColumnType::DateTime,
DynamicColumn::Bytes(_) => ColumnType::Bytes,
- DynamicColumn::Str(_) => ColumnType::Str,
}
}
@@ -224,9 +209,7 @@ static_dynamic_conversions!(Column<u64>, U64);
static_dynamic_conversions!(Column<i64>, I64);
static_dynamic_conversions!(Column<f64>, F64);
static_dynamic_conversions!(Column<DateTime>, DateTime);
-static_dynamic_conversions!(StrColumn, Str);
static_dynamic_conversions!(BytesColumn, Bytes);
-static_dynamic_conversions!(Column<Ipv6Addr>, IpAddr);
#[derive(Clone, Debug)]
pub struct DynamicColumnHandle {
@@ -259,14 +242,10 @@ impl DynamicColumnHandle {
pub fn open_u64_lenient(&self) -> io::Result<Option<Column<u64>>> {
let column_bytes = self.file_slice.read_bytes()?;
match self.column_type {
- ColumnType::Str | ColumnType::Bytes => {
+ ColumnType::Bytes => {
let column: BytesColumn = super::column::open_column_bytes(column_bytes)?;
Ok(Some(column.term_ord_column))
}
- ColumnType::IpAddr => {
- let column = super::column::open_column_u128_as_compact_u64(column_bytes)?;
- Ok(Some(column))
- }
ColumnType::Bool
| ColumnType::I64
| ColumnType::U64
@@ -281,12 +260,10 @@ impl DynamicColumnHandle {
fn open_internal(&self, column_bytes: OwnedBytes) -> io::Result<DynamicColumn> {
let dynamic_column: DynamicColumn = match self.column_type {
ColumnType::Bytes => super::column::open_column_bytes(column_bytes)?.into(),
- ColumnType::Str => super::column::open_column_str(column_bytes)?.into(),
ColumnType::I64 => super::column::open_column_u64::<i64>(column_bytes)?.into(),
ColumnType::U64 => super::column::open_column_u64::<u64>(column_bytes)?.into(),
ColumnType::F64 => super::column::open_column_u64::<f64>(column_bytes)?.into(),
ColumnType::Bool => super::column::open_column_u64::<bool>(column_bytes)?.into(),
- ColumnType::IpAddr => super::column::open_column_u128::<Ipv6Addr>(column_bytes)?.into(),
ColumnType::DateTime => {
super::column::open_column_u64::<DateTime>(column_bytes)?.into()
}
diff --git a/crates/tantivy/src/columnar/mod.rs b/crates/tantivy/src/columnar/mod.rs
index b8101b35..3438be2a 100644
--- a/crates/tantivy/src/columnar/mod.rs
+++ b/crates/tantivy/src/columnar/mod.rs
@@ -33,7 +33,7 @@ mod value;
use crate::sstable::VoidSSTable;
pub use block_accessor::ColumnBlockAccessor;
-pub use column::{BytesColumn, Column, StrColumn};
+pub use column::{BytesColumn, Column};
pub use column_index::ColumnIndex;
pub use column_values::{
ColumnValues, EmptyColumnValues, MonotonicallyMappableToU128, MonotonicallyMappableToU64,
@@ -80,30 +80,18 @@ pub enum Cardinality {
/// `Full` is the default for auto-detecting the Cardinality, since it is the most strict.
#[default]
Full = 0,
- /// All documents contain at most one value.
- Optional = 1,
- /// All documents may contain any number of values.
- Multivalued = 2,
}
impl Display for Cardinality {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let short_str = match self {
Cardinality::Full => "full",
- Cardinality::Optional => "opt",
- Cardinality::Multivalued => "mult",
};
write!(f, "{short_str}")
}
}
impl Cardinality {
- pub fn is_optional(&self) -> bool {
- matches!(self, Cardinality::Optional)
- }
- pub fn is_multivalue(&self) -> bool {
- matches!(self, Cardinality::Multivalued)
- }
pub fn is_full(&self) -> bool {
matches!(self, Cardinality::Full)
}
@@ -113,8 +101,6 @@ impl Cardinality {
pub(crate) fn try_from_code(code: u8) -> Result<Cardinality, InvalidData> {
match code {
0 => Ok(Cardinality::Full),
- 1 => Ok(Cardinality::Optional),
- 2 => Ok(Cardinality::Multivalued),
_ => Err(InvalidData),
}
}
diff --git a/crates/tantivy/src/columnar/tests.rs b/crates/tantivy/src/columnar/tests.rs
index bade724b..d800a34d 100644
--- a/crates/tantivy/src/columnar/tests.rs
+++ b/crates/tantivy/src/columnar/tests.rs
@@ -1,33 +1,6 @@
-use std::collections::HashMap;
-use std::fmt::Debug;
-use std::net::Ipv6Addr;
-
-use crate::common::DateTime;
-use proptest::prelude::*;
-use proptest::sample::subsequence;
-
-use super::column_values::MonotonicallyMappableToU128;
-use super::columnar::{ColumnType, ColumnTypeCategory};
use super::dynamic_column::{DynamicColumn, DynamicColumnHandle};
-use super::value::{Coerce, NumericalValue};
-use super::{
- BytesColumn, Cardinality, Column, ColumnarReader, ColumnarWriter, RowAddr, RowId,
- ShuffleMergeOrder, StackMergeOrder,
-};
-
-#[test]
-fn test_dataframe_writer_str() {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_str(1u32, "my_string", "hello");
- dataframe_writer.record_str(3u32, "my_string", "helloeee");
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer.serialize(5, None, &mut buffer).unwrap();
- let columnar = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar.num_columns(), 1);
- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
- assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 77);
-}
+use super::value::NumericalValue;
+use super::{ColumnarReader, ColumnarWriter};
#[test]
fn test_dataframe_writer_bytes() {
@@ -40,117 +13,7 @@ fn test_dataframe_writer_bytes() {
assert_eq!(columnar.num_columns(), 1);
let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 77);
-}
-
-#[test]
-fn test_dataframe_writer_bool() {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_bool(1u32, "bool.value", false);
- dataframe_writer.record_bool(3u32, "bool.value", true);
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer.serialize(5, None, &mut buffer).unwrap();
- let columnar = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar.num_columns(), 1);
- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("bool.value").unwrap();
- assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 22);
- assert_eq!(cols[0].column_type(), ColumnType::Bool);
- let dyn_bool_col = cols[0].open().unwrap();
- let DynamicColumn::Bool(bool_col) = dyn_bool_col else {
- panic!();
- };
- let vals: Vec<Option<bool>> = (0..5).map(|row_id| bool_col.first(row_id)).collect();
- assert_eq!(&vals, &[None, Some(false), None, Some(true), None,]);
-}
-
-#[test]
-fn test_dataframe_writer_u64_multivalued() {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_numerical(2u32, "divisor", 2u64);
- dataframe_writer.record_numerical(3u32, "divisor", 3u64);
- dataframe_writer.record_numerical(4u32, "divisor", 2u64);
- dataframe_writer.record_numerical(5u32, "divisor", 5u64);
- dataframe_writer.record_numerical(6u32, "divisor", 2u64);
- dataframe_writer.record_numerical(6u32, "divisor", 3u64);
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer.serialize(7, None, &mut buffer).unwrap();
- let columnar = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar.num_columns(), 1);
- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("divisor").unwrap();
- assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 29);
- let dyn_i64_col = cols[0].open().unwrap();
- let DynamicColumn::I64(divisor_col) = dyn_i64_col else {
- panic!();
- };
- assert_eq!(
- divisor_col.get_cardinality(),
- super::Cardinality::Multivalued
- );
- assert_eq!(divisor_col.num_docs(), 7);
-}
-
-#[test]
-fn test_dataframe_writer_ip_addr() {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_ip_addr(1, "ip_addr", Ipv6Addr::from_u128(1001));
- dataframe_writer.record_ip_addr(3, "ip_addr", Ipv6Addr::from_u128(1050));
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer.serialize(5, None, &mut buffer).unwrap();
- let columnar = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar.num_columns(), 1);
- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("ip_addr").unwrap();
- assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 42);
- assert_eq!(cols[0].column_type(), ColumnType::IpAddr);
- let dyn_bool_col = cols[0].open().unwrap();
- let DynamicColumn::IpAddr(ip_col) = dyn_bool_col else {
- panic!();
- };
- let vals: Vec<Option<Ipv6Addr>> = (0..5).map(|row_id| ip_col.first(row_id)).collect();
- assert_eq!(
- &vals,
- &[
- None,
- Some(Ipv6Addr::from_u128(1001)),
- None,
- Some(Ipv6Addr::from_u128(1050)),
- None,
- ]
- );
-}
-
-#[test]
-fn test_dataframe_writer_numerical() {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_numerical(1u32, "srical.value", NumericalValue::U64(12u64));
- dataframe_writer.record_numerical(2u32, "srical.value", NumericalValue::U64(13u64));
- dataframe_writer.record_numerical(4u32, "srical.value", NumericalValue::U64(15u64));
- let mut buffer: Vec<u8> = Vec::new();
- dataframe_writer.serialize(6, None, &mut buffer).unwrap();
- let columnar = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar.num_columns(), 1);
- let cols: Vec<DynamicColumnHandle> = columnar.read_columns("srical.value").unwrap();
- assert_eq!(cols.len(), 1);
- // Right now this 31 bytes are spent as follows
- //
- // - header 14 bytes
- // - vals 8 //< due to padding? could have been 1byte?.
- // - null footer 6 bytes
- assert_eq!(cols[0].num_bytes(), 33);
- let column = cols[0].open().unwrap();
- let DynamicColumn::I64(column_i64) = column else {
- panic!();
- };
- assert_eq!(column_i64.index.get_cardinality(), Cardinality::Optional);
- assert_eq!(column_i64.first(0), None);
- assert_eq!(column_i64.first(1), Some(12i64));
- assert_eq!(column_i64.first(2), Some(13i64));
- assert_eq!(column_i64.first(3), None);
- assert_eq!(column_i64.first(4), Some(15i64));
- assert_eq!(column_i64.first(5), None);
- assert_eq!(column_i64.first(6), None); //< we can change the spec for that one.
+ assert_eq!(cols[0].num_bytes(), 66);
}
#[test]
@@ -162,71 +25,6 @@ fn test_dataframe_sort_by_full() {
assert_eq!(data, vec![0, 1]);
}
-#[test]
-fn test_dataframe_sort_by_opt() {
- let mut dataframe_writer = ColumnarWriter::default();
- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(3));
- dataframe_writer.record_numerical(3u32, "value", NumericalValue::U64(2));
- let data = dataframe_writer.sort_order("value", 5, false);
- // 0, 2, 4 is 0.0
- assert_eq!(data, vec![0, 2, 4, 3, 1]);
- let data = dataframe_writer.sort_order("value", 5, true);
- assert_eq!(
- data,
- vec![4, 2, 0, 3, 1].into_iter().rev().collect::<Vec<_>>()
- );
-}
-
-#[test]
-fn test_dataframe_sort_by_multi() {
- let mut dataframe_writer = ColumnarWriter::default();
- // valid for sort
- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(2));
- // those are ignored for sort
- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(4));
- dataframe_writer.record_numerical(1u32, "value", NumericalValue::U64(4));
- // valid for sort
- dataframe_writer.record_numerical(3u32, "value", NumericalValue::U64(3));
- // ignored, would change sort order
- dataframe_writer.record_numerical(3u32, "value", NumericalValue::U64(1));
- let data = dataframe_writer.sort_order("value", 4, false);
- assert_eq!(data, vec![0, 2, 1, 3]);
-}
-
-#[test]
-fn test_dictionary_encoded_str() {
- let mut buffer = Vec::new();
- let mut columnar_writer = ColumnarWriter::default();
- columnar_writer.record_str(1, "my.column", "a");
- columnar_writer.record_str(3, "my.column", "c");
- columnar_writer.record_str(3, "my.column2", "different_column!");
- columnar_writer.record_str(4, "my.column", "b");
- columnar_writer.serialize(5, None, &mut buffer).unwrap();
- let columnar_reader = ColumnarReader::open(buffer).unwrap();
- assert_eq!(columnar_reader.num_columns(), 2);
- let col_handles = columnar_reader.read_columns("my.column").unwrap();
- assert_eq!(col_handles.len(), 1);
- let DynamicColumn::Str(str_col) = col_handles[0].open().unwrap() else {
- panic!();
- };
- let index: Vec<Option<u64>> = (0..5).map(|row_id| str_col.ords().first(row_id)).collect();
- assert_eq!(index, &[None, Some(0), None, Some(2), Some(1)]);
- assert_eq!(str_col.num_rows(), 5);
- let mut term_buffer = String::new();
- let term_ords = str_col.ords();
- assert_eq!(term_ords.first(0), None);
- assert_eq!(term_ords.first(1), Some(0));
- str_col.ord_to_str(0u64, &mut term_buffer).unwrap();
- assert_eq!(term_buffer, "a");
- assert_eq!(term_ords.first(2), None);
- assert_eq!(term_ords.first(3), Some(2));
- str_col.ord_to_str(2u64, &mut term_buffer).unwrap();
- assert_eq!(term_buffer, "c");
- assert_eq!(term_ords.first(4), Some(1));
- str_col.ord_to_str(1u64, &mut term_buffer).unwrap();
- assert_eq!(term_buffer, "b");
-}
-
#[test]
fn test_dictionary_encoded_bytes() {
let mut buffer = Vec::new();
@@ -243,703 +41,20 @@ fn test_dictionary_encoded_bytes() {
let DynamicColumn::Bytes(bytes_col) = col_handles[0].open().unwrap() else {
panic!();
};
- let index: Vec<Option<u64>> = (0..5)
- .map(|row_id| bytes_col.ords().first(row_id))
- .collect();
- assert_eq!(index, &[None, Some(0), None, Some(2), Some(1)]);
- assert_eq!(bytes_col.num_rows(), 5);
let mut term_buffer = Vec::new();
- let term_ords = bytes_col.ords();
- assert_eq!(term_ords.first(0), None);
- assert_eq!(term_ords.first(1), Some(0));
bytes_col
.dictionary
.ord_to_term(0u64, &mut term_buffer)
.unwrap();
assert_eq!(term_buffer, b"a");
- assert_eq!(term_ords.first(2), None);
- assert_eq!(term_ords.first(3), Some(2));
bytes_col
.dictionary
.ord_to_term(2u64, &mut term_buffer)
.unwrap();
assert_eq!(term_buffer, b"c");
- assert_eq!(term_ords.first(4), Some(1));
bytes_col
.dictionary
.ord_to_term(1u64, &mut term_buffer)
.unwrap();
assert_eq!(term_buffer, b"b");
}
-
-fn num_strategy() -> impl Strategy<Value = NumericalValue> {
- prop_oneof![
- 3 => Just(NumericalValue::U64(0u64)),
- 3 => Just(NumericalValue::U64(u64::MAX)),
- 3 => Just(NumericalValue::I64(0i64)),
- 3 => Just(NumericalValue::I64(i64::MIN)),
- 3 => Just(NumericalValue::I64(i64::MAX)),
- 3 => Just(NumericalValue::F64(1.2f64)),
- 1 => any::<f64>().prop_map(NumericalValue::from),
- 1 => any::<u64>().prop_map(NumericalValue::from),
- 1 => any::<i64>().prop_map(NumericalValue::from),
- ]
-}
-
-#[derive(Debug, Clone, Copy)]
-enum ColumnValue {
- Str(&'static str),
- Bytes(&'static [u8]),
- Numerical(NumericalValue),
- IpAddr(Ipv6Addr),
- Bool(bool),
- DateTime(DateTime),
-}
-
-impl<T: Into<NumericalValue>> From<T> for ColumnValue {
- fn from(val: T) -> ColumnValue {
- ColumnValue::Numerical(val.into())
- }
-}
-
-impl ColumnValue {
- pub(crate) fn column_type_category(&self) -> ColumnTypeCategory {
- match self {
- ColumnValue::Str(_) => ColumnTypeCategory::Str,
- ColumnValue::Bytes(_) => ColumnTypeCategory::Bytes,
- ColumnValue::Numerical(_) => ColumnTypeCategory::Numerical,
- ColumnValue::IpAddr(_) => ColumnTypeCategory::IpAddr,
- ColumnValue::Bool(_) => ColumnTypeCategory::Bool,
- ColumnValue::DateTime(_) => ColumnTypeCategory::DateTime,
- }
- }
-}
-
-fn column_name_strategy() -> impl Strategy<Value = &'static str> {
- prop_oneof![Just("c1"), Just("c2")]
-}
-
-fn string_strategy() -> impl Strategy<Value = &'static str> {
- prop_oneof![Just("a"), Just("b")]
-}
-
-fn bytes_strategy() -> impl Strategy<Value = &'static [u8]> {
- prop_oneof![Just(&[0u8][..]), Just(&[1u8][..])]
-}
-
-// A random column value
-fn column_value_strategy() -> impl Strategy<Value = ColumnValue> {
- prop_oneof![
- 10 => string_strategy().prop_map(ColumnValue::Str),
- 1 => bytes_strategy().prop_map(ColumnValue::Bytes),
- 40 => num_strategy().prop_map(ColumnValue::Numerical),
- 1 => (1u16..3u16).prop_map(|ip_addr_byte| ColumnValue::IpAddr(Ipv6Addr::new(
- 127,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- ip_addr_byte
- ))),
- 1 => any::<bool>().prop_map(ColumnValue::Bool),
- 1 => (679_723_993i64..1_679_723_995i64)
- .prop_map(|val| { ColumnValue::DateTime(DateTime::from_timestamp_secs(val)) })
- ]
-}
-
-// A document contains up to 4 values.
-fn doc_strategy() -> impl Strategy<Value = Vec<(&'static str, ColumnValue)>> {
- proptest::collection::vec((column_name_strategy(), column_value_strategy()), 0..=4)
-}
-
-fn num_docs_strategy() -> impl Strategy<Value = usize> {
- prop_oneof!(
- // We focus heavily on the 0..2 case as we assume it is sufficient to cover all edge cases.
- 0usize..=3usize,
- // We leave 50% of the effort exploring more defensively.
- 3usize..=12usize
- )
-}
-
-// A columnar contains up to 2 docs.
-fn columnar_docs_strategy() -> impl Strategy<Value = Vec<Vec<(&'static str, ColumnValue)>>> {
- num_docs_strategy()
- .prop_flat_map(|num_docs| proptest::collection::vec(doc_strategy(), num_docs))
-}
-
-fn columnar_docs_and_mapping_strategy(
-) -> impl Strategy<Value = (Vec<Vec<(&'static str, ColumnValue)>>, Vec<RowId>)> {
- columnar_docs_strategy().prop_flat_map(|docs| {
- permutation_strategy(docs.len()).prop_map(move |permutation| (docs.clone(), permutation))
- })
-}
-
-fn permutation_strategy(n: usize) -> impl Strategy<Value = Vec<RowId>> {
- Just((0u32..n as RowId).collect()).prop_shuffle()
-}
-
-fn permutation_and_subset_strategy(n: usize) -> impl Strategy<Value = Vec<usize>> {
- let vals: Vec<usize> = (0..n).collect();
- subsequence(vals, 0..=n).prop_shuffle()
-}
-
-fn build_columnar_with_mapping(
- docs: &[Vec<(&'static str, ColumnValue)>],
- old_to_new_row_ids_opt: Option<&[RowId]>,
-) -> ColumnarReader {
- let num_docs = docs.len() as u32;
- let mut buffer = Vec::new();
- let mut columnar_writer = ColumnarWriter::default();
- for (doc_id, vals) in docs.iter().enumerate() {
- for (column_name, col_val) in vals {
- match *col_val {
- ColumnValue::Str(str_val) => {
- columnar_writer.record_str(doc_id as u32, column_name, str_val);
- }
- ColumnValue::Bytes(bytes) => {
- columnar_writer.record_bytes(doc_id as u32, column_name, bytes)
- }
- ColumnValue::Numerical(num) => {
- columnar_writer.record_numerical(doc_id as u32, column_name, num);
- }
- ColumnValue::IpAddr(ip_addr) => {
- columnar_writer.record_ip_addr(doc_id as u32, column_name, ip_addr);
- }
- ColumnValue::Bool(bool_val) => {
- columnar_writer.record_bool(doc_id as u32, column_name, bool_val);
- }
- ColumnValue::DateTime(date_time) => {
- columnar_writer.record_datetime(doc_id as u32, column_name, date_time);
- }
- }
- }
- }
- columnar_writer
- .serialize(num_docs, old_to_new_row_ids_opt, &mut buffer)
- .unwrap();
-
- ColumnarReader::open(buffer).unwrap()
-}
-
-fn build_columnar(docs: &[Vec<(&'static str, ColumnValue)>]) -> ColumnarReader {
- build_columnar_with_mapping(docs, None)
-}
-
-fn assert_columnar_eq_strict(left: &ColumnarReader, right: &ColumnarReader) {
- assert_columnar_eq(left, right, false);
-}
-
-fn assert_columnar_eq(
- left: &ColumnarReader,
- right: &ColumnarReader,
- lenient_on_numerical_value: bool,
-) {
- assert_eq!(left.num_rows(), right.num_rows());
- let left_columns = left.list_columns().unwrap();
- let right_columns = right.list_columns().unwrap();
- assert_eq!(left_columns.len(), right_columns.len());
- for i in 0..left_columns.len() {
- assert_eq!(left_columns[i].0, right_columns[i].0);
- let left_column = left_columns[i].1.open().unwrap();
- let right_column = right_columns[i].1.open().unwrap();
- assert_dyn_column_eq(&left_column, &right_column, lenient_on_numerical_value);
- }
-}
-
-fn assert_column_eq<T: Copy + PartialOrd + Debug + Send + Sync + 'static>(
- left: &Column<T>,
- right: &Column<T>,
-) {
- assert_eq!(left.get_cardinality(), right.get_cardinality());
- assert_eq!(left.num_docs(), right.num_docs());
- let num_docs = left.num_docs();
- for doc in 0..num_docs {
- assert_eq!(
- left.index.value_row_ids(doc),
- right.index.value_row_ids(doc)
- );
- }
- assert_eq!(left.values.num_vals(), right.values.num_vals());
- let num_vals = left.values.num_vals();
- for i in 0..num_vals {
- assert_eq!(left.values.get_val(i), right.values.get_val(i));
- }
-}
-
-fn assert_bytes_column_eq(left: &BytesColumn, right: &BytesColumn) {
- assert_eq!(
- left.term_ord_column.get_cardinality(),
- right.term_ord_column.get_cardinality()
- );
- assert_eq!(left.num_rows(), right.num_rows());
- assert_column_eq(&left.term_ord_column, &right.term_ord_column);
- assert_eq!(left.dictionary.num_terms(), right.dictionary.num_terms());
- let num_terms = left.dictionary.num_terms();
- let mut left_terms = left.dictionary.stream().unwrap();
- let mut right_terms = right.dictionary.stream().unwrap();
- for _ in 0..num_terms {
- assert!(left_terms.advance());
- assert!(right_terms.advance());
- assert_eq!(left_terms.key(), right_terms.key());
- }
- assert!(!left_terms.advance());
- assert!(!right_terms.advance());
-}
-
-fn assert_dyn_column_eq(
- left_dyn_column: &DynamicColumn,
- right_dyn_column: &DynamicColumn,
- lenient_on_numerical_value: bool,
-) {
- assert_eq!(
- &left_dyn_column.get_cardinality(),
- &right_dyn_column.get_cardinality()
- );
- match &(left_dyn_column, right_dyn_column) {
- (DynamicColumn::Bool(left_col), DynamicColumn::Bool(right_col)) => {
- assert_column_eq(left_col, right_col);
- }
- (DynamicColumn::I64(left_col), DynamicColumn::I64(right_col)) => {
- assert_column_eq(left_col, right_col);
- }
- (DynamicColumn::U64(left_col), DynamicColumn::U64(right_col)) => {
- assert_column_eq(left_col, right_col);
- }
- (DynamicColumn::F64(left_col), DynamicColumn::F64(right_col)) => {
- assert_column_eq(left_col, right_col);
- }
- (DynamicColumn::DateTime(left_col), DynamicColumn::DateTime(right_col)) => {
- assert_column_eq(left_col, right_col);
- }
- (DynamicColumn::IpAddr(left_col), DynamicColumn::IpAddr(right_col)) => {
- assert_column_eq(left_col, right_col);
- }
- (DynamicColumn::Bytes(left_col), DynamicColumn::Bytes(right_col)) => {
- assert_bytes_column_eq(left_col, right_col);
- }
- (DynamicColumn::Str(left_col), DynamicColumn::Str(right_col)) => {
- assert_bytes_column_eq(left_col, right_col);
- }
- (left, right) => {
- if lenient_on_numerical_value {
- assert_eq!(
- ColumnTypeCategory::from(left.column_type()),
- ColumnTypeCategory::from(right.column_type())
- );
- } else {
- panic!(
- "Column type are not the same: {:?} vs {:?}",
- left.column_type(),
- right.column_type()
- );
- }
- }
- }
-}
-
-trait AssertEqualToColumnValue {
- fn assert_equal_to_column_value(&self, column_value: &ColumnValue);
-}
-
-impl AssertEqualToColumnValue for bool {
- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {
- let ColumnValue::Bool(val) = column_value else {
- panic!()
- };
- assert_eq!(self, val);
- }
-}
-
-impl AssertEqualToColumnValue for Ipv6Addr {
- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {
- let ColumnValue::IpAddr(val) = column_value else {
- panic!()
- };
- assert_eq!(self, val);
- }
-}
-
-impl<T: Coerce + PartialEq + Debug + Into<NumericalValue>> AssertEqualToColumnValue for T {
- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {
- let ColumnValue::Numerical(num) = column_value else {
- panic!()
- };
- assert_eq!(self, &T::coerce(*num));
- }
-}
-
-impl AssertEqualToColumnValue for DateTime {
- fn assert_equal_to_column_value(&self, column_value: &ColumnValue) {
- let ColumnValue::DateTime(dt) = column_value else {
- panic!()
- };
- assert_eq!(self, dt);
- }
-}
-
-fn assert_column_values<
- T: AssertEqualToColumnValue + PartialEq + Copy + PartialOrd + Debug + Send + Sync + 'static,
->(
- col: &Column<T>,
- expected: &HashMap<u32, Vec<&ColumnValue>>,
-) {
- let mut num_non_empty_rows = 0;
- for doc in 0..col.num_docs() {
- let doc_vals: Vec<T> = col.values_for_doc(doc).collect();
- if doc_vals.is_empty() {
- continue;
- }
- num_non_empty_rows += 1;
- let expected_vals = expected.get(&doc).unwrap();
- assert_eq!(doc_vals.len(), expected_vals.len());
- for (val, &expected) in doc_vals.iter().zip(expected_vals.iter()) {
- val.assert_equal_to_column_value(expected)
- }
- }
- assert_eq!(num_non_empty_rows, expected.len());
-}
-
-fn assert_bytes_column_values(
- col: &BytesColumn,
- expected: &HashMap<u32, Vec<&ColumnValue>>,
- is_str: bool,
-) {
- let mut num_non_empty_rows = 0;
- let mut buffer = Vec::new();
- for doc in 0..col.term_ord_column.num_docs() {
- let doc_vals: Vec<u64> = col.term_ords(doc).collect();
- if doc_vals.is_empty() {
- continue;
- }
- let expected_vals = expected.get(&doc).unwrap();
- assert_eq!(doc_vals.len(), expected_vals.len());
- for (&expected_col_val, &ord) in expected_vals.iter().zip(&doc_vals) {
- col.ord_to_bytes(ord, &mut buffer).unwrap();
- match expected_col_val {
- ColumnValue::Str(str_val) => {
- assert!(is_str);
- assert_eq!(str_val.as_bytes(), &buffer);
- }
- ColumnValue::Bytes(bytes_val) => {
- assert!(!is_str);
- assert_eq!(bytes_val, &buffer);
- }
- _ => {
- panic!();
- }
- }
- }
- num_non_empty_rows += 1;
- }
- assert_eq!(num_non_empty_rows, expected.len());
-}
-
-// This proptest attempts to create a tiny columnar based of up to 3 rows, and checks that the
-// resulting columnar matches the row data.
-proptest! {
- #![proptest_config(ProptestConfig::with_cases(500))]
- #[test]
- fn test_single_columnar_builder_proptest(docs in columnar_docs_strategy()) {
- let columnar = build_columnar(&docs[..]);
- assert_eq!(columnar.num_rows() as usize, docs.len());
- let mut expected_columns: HashMap<(&str, ColumnTypeCategory), HashMap<u32, Vec<&ColumnValue>> > = Default::default();
- for (doc_id, doc_vals) in docs.iter().enumerate() {
- for (col_name, col_val) in doc_vals {
- expected_columns
- .entry((col_name, col_val.column_type_category()))
- .or_default()
- .entry(doc_id as u32)
- .or_default()
- .push(col_val);
- }
- }
- let column_list = columnar.list_columns().unwrap();
- assert_eq!(expected_columns.len(), column_list.len());
- for (column_name, column) in column_list {
- let dynamic_column = column.open().unwrap();
- let col_category: ColumnTypeCategory = dynamic_column.column_type().into();
- let expected_col_values: &HashMap<u32, Vec<&ColumnValue>> = expected_columns.get(&(column_name.as_str(), col_category)).unwrap();
- match &dynamic_column {
- DynamicColumn::Bool(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::I64(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::U64(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::F64(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::IpAddr(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::DateTime(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::Bytes(col) =>
- assert_bytes_column_values(col, expected_col_values, false),
- DynamicColumn::Str(col) =>
- assert_bytes_column_values(col, expected_col_values, true),
- }
- }
- }
-}
-
-// Same as `test_single_columnar_builder_proptest` but with a shuffling mapping.
-proptest! {
- #![proptest_config(ProptestConfig::with_cases(500))]
- #[test]
- fn test_single_columnar_builder_with_shuffle_proptest((docs, mapping) in columnar_docs_and_mapping_strategy()) {
- let columnar = build_columnar_with_mapping(&docs[..], Some(&mapping));
- assert_eq!(columnar.num_rows() as usize, docs.len());
- let mut expected_columns: HashMap<(&str, ColumnTypeCategory), HashMap<u32, Vec<&ColumnValue>> > = Default::default();
- for (doc_id, doc_vals) in docs.iter().enumerate() {
- for (col_name, col_val) in doc_vals {
- expected_columns
- .entry((col_name, col_val.column_type_category()))
- .or_default()
- .entry(mapping[doc_id])
- .or_default()
- .push(col_val);
- }
- }
- let column_list = columnar.list_columns().unwrap();
- assert_eq!(expected_columns.len(), column_list.len());
- for (column_name, column) in column_list {
- let dynamic_column = column.open().unwrap();
- let col_category: ColumnTypeCategory = dynamic_column.column_type().into();
- let expected_col_values: &HashMap<u32, Vec<&ColumnValue>> = expected_columns.get(&(column_name.as_str(), col_category)).unwrap();
- for _doc_id in 0..columnar.num_rows() {
- match &dynamic_column {
- DynamicColumn::Bool(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::I64(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::U64(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::F64(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::IpAddr(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::DateTime(col) =>
- assert_column_values(col, expected_col_values),
- DynamicColumn::Bytes(col) =>
- assert_bytes_column_values(col, expected_col_values, false),
- DynamicColumn::Str(col) =>
- assert_bytes_column_values(col, expected_col_values, true),
- }
- }
- }
- }
-}
-
-// This tests create 2 or 3 random small columnar and attempts to merge them.
-// It compares the resulting merged dataframe with what would have been obtained by building the
-// dataframe from the concatenated rows to begin with.
-proptest! {
- #![proptest_config(ProptestConfig::with_cases(1000))]
- #[test]
- fn test_columnar_merge_proptest(columnar_docs in proptest::collection::vec(columnar_docs_strategy(), 2..=3)) {
- let columnar_readers: Vec<ColumnarReader> = columnar_docs.iter()
- .map(|docs| build_columnar(&docs[..]))
- .collect::<Vec<_>>();
- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();
- let mut output: Vec<u8> = Vec::new();
- let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]).into();
- super::merge_columnar(&columnar_readers_arr[..], &[], stack_merge_order, &mut output).unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> = columnar_docs.iter().flatten().cloned().collect();
- let expected_merged_columnar = build_columnar(&concat_rows[..]);
- assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
- }
-}
-
-#[test]
-fn test_columnar_merging_empty_columnar() {
- let columnar_docs: Vec<Vec<Vec<(&str, ColumnValue)>>> =
- vec![vec![], vec![vec![("c1", ColumnValue::Str("a"))]]];
- let columnar_readers: Vec<ColumnarReader> = columnar_docs
- .iter()
- .map(|docs| build_columnar(&docs[..]))
- .collect::<Vec<_>>();
- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();
- let mut output: Vec<u8> = Vec::new();
- let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]);
- super::merge_columnar(
- &columnar_readers_arr[..],
- &[],
- super::MergeRowOrder::Stack(stack_merge_order),
- &mut output,
- )
- .unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =
- columnar_docs.iter().flatten().cloned().collect();
- let expected_merged_columnar = build_columnar(&concat_rows[..]);
- assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
-}
-
-#[test]
-fn test_columnar_merging_number_columns() {
- let columnar_docs: Vec<Vec<Vec<(&str, ColumnValue)>>> = vec![
- // columnar 1
- vec![
- // doc 1.1
- vec![("c2", ColumnValue::Numerical(0i64.into()))],
- ],
- // columnar2
- vec![
- // doc 2.1
- vec![("c2", ColumnValue::Numerical(0u64.into()))],
- // doc 2.2
- vec![("c2", ColumnValue::Numerical(u64::MAX.into()))],
- ],
- ];
- let columnar_readers: Vec<ColumnarReader> = columnar_docs
- .iter()
- .map(|docs| build_columnar(&docs[..]))
- .collect::<Vec<_>>();
- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();
- let mut output: Vec<u8> = Vec::new();
- let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]);
- super::merge_columnar(
- &columnar_readers_arr[..],
- &[],
- super::MergeRowOrder::Stack(stack_merge_order),
- &mut output,
- )
- .unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =
- columnar_docs.iter().flatten().cloned().collect();
- let expected_merged_columnar = build_columnar(&concat_rows[..]);
- assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
-}
-
-// TODO add non trivial remap and merge
-// TODO test required_columns
-// TODO document edge case: required_columns incompatible with values.
-
-fn columnar_docs_and_remap(
-) -> impl Strategy<Value = (Vec<Vec<Vec<(&'static str, ColumnValue)>>>, Vec<RowAddr>)> {
- proptest::collection::vec(columnar_docs_strategy(), 2..=3).prop_flat_map(
- |columnars_docs: Vec<Vec<Vec<(&str, ColumnValue)>>>| {
- let row_addrs: Vec<RowAddr> = columnars_docs
- .iter()
- .enumerate()
- .flat_map(|(segment_ord, columnar_docs)| {
- (0u32..columnar_docs.len() as u32).map(move |row_id| RowAddr {
- segment_ord: segment_ord as u32,
- row_id,
- })
- })
- .collect();
- permutation_and_subset_strategy(row_addrs.len()).prop_map(move |shuffled_subset| {
- let shuffled_row_addr_subset: Vec<RowAddr> =
- shuffled_subset.iter().map(|ord| row_addrs[*ord]).collect();
- (columnars_docs.clone(), shuffled_row_addr_subset)
- })
- },
- )
-}
-
-proptest! {
- #![proptest_config(ProptestConfig::with_cases(1000))]
- #[test]
- fn test_columnar_merge_and_remap_proptest((columnar_docs, shuffle_merge_order) in columnar_docs_and_remap()) {
- let shuffled_rows: Vec<Vec<(&'static str, ColumnValue)>> = shuffle_merge_order.iter()
- .map(|row_addr| columnar_docs[row_addr.segment_ord as usize][row_addr.row_id as usize].clone())
- .collect();
- let expected_merged_columnar = build_columnar(&shuffled_rows[..]);
- let columnar_readers: Vec<ColumnarReader> = columnar_docs.iter()
- .map(|docs| build_columnar(&docs[..]))
- .collect::<Vec<_>>();
- let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();
- let mut output: Vec<u8> = Vec::new();
- let segment_num_rows: Vec<RowId> = columnar_docs.iter().map(|docs| docs.len() as RowId).collect();
- let shuffle_merge_order = ShuffleMergeOrder::for_test(&segment_num_rows, shuffle_merge_order);
- super::merge_columnar(&columnar_readers_arr[..], &[], shuffle_merge_order.into(), &mut output).unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- assert_columnar_eq(&merged_columnar, &expected_merged_columnar, true);
- }
-}
-
-#[test]
-fn test_columnar_merge_empty() {
- let columnar_reader_1 = build_columnar(&[]);
- let rows: &[Vec<_>] = &[vec![("c1", ColumnValue::Str("a"))]][..];
- let columnar_reader_2 = build_columnar(rows);
- let mut output: Vec<u8> = Vec::new();
- let segment_num_rows: Vec<RowId> = vec![0, 0];
- let shuffle_merge_order = ShuffleMergeOrder::for_test(&segment_num_rows, vec![]);
- super::merge_columnar(
- &[&columnar_reader_1, &columnar_reader_2],
- &[],
- shuffle_merge_order.into(),
- &mut output,
- )
- .unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- assert_eq!(merged_columnar.num_rows(), 0);
- assert_eq!(merged_columnar.num_columns(), 0);
-}
-
-#[test]
-fn test_columnar_merge_single_str_column() {
- let columnar_reader_1 = build_columnar(&[]);
- let rows: &[Vec<_>] = &[vec![("c1", ColumnValue::Str("a"))]][..];
- let columnar_reader_2 = build_columnar(rows);
- let mut output: Vec<u8> = Vec::new();
- let segment_num_rows: Vec<RowId> = vec![0, 1];
- let shuffle_merge_order = ShuffleMergeOrder::for_test(
- &segment_num_rows,
- vec![RowAddr {
- segment_ord: 1u32,
- row_id: 0u32,
- }],
- );
- super::merge_columnar(
- &[&columnar_reader_1, &columnar_reader_2],
- &[],
- shuffle_merge_order.into(),
- &mut output,
- )
- .unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- assert_eq!(merged_columnar.num_rows(), 1);
- assert_eq!(merged_columnar.num_columns(), 1);
-}
-
-#[test]
-fn test_delete_decrease_cardinality() {
- let columnar_reader_1 = build_columnar(&[]);
- let rows: &[Vec<_>] = &[
- vec![
- ("c", ColumnValue::from(0i64)),
- ("c", ColumnValue::from(0i64)),
- ],
- vec![("c", ColumnValue::from(0i64))],
- ][..];
- // c is multivalued here
- let columnar_reader_2 = build_columnar(rows);
- let mut output: Vec<u8> = Vec::new();
- let shuffle_merge_order = ShuffleMergeOrder::for_test(
- &[0, 2],
- vec![RowAddr {
- segment_ord: 1u32,
- row_id: 1u32,
- }],
- );
- super::merge_columnar(
- &[&columnar_reader_1, &columnar_reader_2],
- &[],
- shuffle_merge_order.into(),
- &mut output,
- )
- .unwrap();
- let merged_columnar = ColumnarReader::open(output).unwrap();
- assert_eq!(merged_columnar.num_rows(), 1);
- assert_eq!(merged_columnar.num_columns(), 1);
- let cols = merged_columnar.read_columns("c").unwrap();
- assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].column_type(), ColumnType::I64);
- assert_eq!(cols[0].open().unwrap().get_cardinality(), Cardinality::Full);
-}
diff --git a/crates/tantivy/src/core/json_utils.rs b/crates/tantivy/src/core/json_utils.rs
index ee480b03..72d8c9e8 100644
--- a/crates/tantivy/src/core/json_utils.rs
+++ b/crates/tantivy/src/core/json_utils.rs
@@ -197,9 +197,6 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
ReferenceValueLeaf::Bytes(_) => {
unimplemented!("Bytes support in dynamic fields is not yet implemented")
}
- ReferenceValueLeaf::Facet(_) => {
- unimplemented!("Facet support in dynamic fields is not yet implemented")
- }
ReferenceValueLeaf::IpAddr(_) => {
unimplemented!("IP address support in dynamic fields is not yet implemented")
}
diff --git a/crates/tantivy/src/fastfield/facet_reader.rs b/crates/tantivy/src/fastfield/facet_reader.rs
deleted file mode 100644
index 0fde3717..00000000
--- a/crates/tantivy/src/fastfield/facet_reader.rs
+++ /dev/null
@@ -1,195 +0,0 @@
-use crate::columnar::StrColumn;
-
-use crate::schema::Facet;
-use crate::termdict::TermOrdinal;
-use crate::DocId;
-
-/// The facet reader makes it possible to access the list of
-/// facets associated with a given document in a specific
-/// segment.
-///
-/// Rather than manipulating `Facet` object directly, the API
-/// exposes those in the form of list of `Facet` ordinal.
-///
-/// A segment ordinal can then be translated into a facet via
-/// `.facet_from_ord(...)`.
-///
-/// Facet ordinals are defined as their position in the sorted
-/// list of facets. This ordinal is segment local and
-/// only makes sense for a given segment.
-pub struct FacetReader {
- facet_column: StrColumn,
-}
-
-impl FacetReader {
- /// Creates a new `FacetReader`.
- ///
- /// A facet reader just wraps :
- /// - a `MultiValuedFastFieldReader` that makes it possible to
- /// access the list of facet ords for a given document.
- /// - a `TermDictionary` that helps associating a facet to
- /// an ordinal and vice versa.
- pub fn new(facet_column: StrColumn) -> FacetReader {
- FacetReader { facet_column }
- }
-
- /// Returns the size of the sets of facets in the segment.
- /// This does not take in account the documents that may be marked
- /// as deleted.
- ///
- /// `Facet` ordinals range from `0` to `num_facets() - 1`.
- pub fn num_facets(&self) -> usize {
- self.facet_column.num_terms()
- }
-
- /// Given a term ordinal returns the term associated with it.
- pub fn facet_from_ord(&self, facet_ord: TermOrdinal, output: &mut Facet) -> crate::Result<()> {
- let found_term = self.facet_column.ord_to_str(facet_ord, &mut output.0)?;
- assert!(found_term, "Term ordinal {facet_ord} no found.");
- Ok(())
- }
-
- /// Return the list of facet ordinals associated with a document.
- pub fn facet_ords(&self, doc: DocId) -> impl Iterator<Item = u64> + '_ {
- self.facet_column.ords().values_for_doc(doc)
- }
-
- /// Accessor to the facet dictionary.
- pub fn facet_dict(&self) -> &crate::columnar::Dictionary {
- self.facet_column.dictionary()
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::schema::{Facet, FacetOptions, SchemaBuilder, Value, STORED};
- use crate::{DocAddress, Index, IndexWriter, TantivyDocument};
-
- #[test]
- fn test_facet_only_indexed() {
- let mut schema_builder = SchemaBuilder::default();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))
- .unwrap();
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
- let mut facet_ords = Vec::new();
- facet_ords.extend(facet_reader.facet_ords(0u32));
- assert_eq!(&facet_ords, &[0u64]);
- assert_eq!(facet_reader.num_facets(), 1);
- let mut facet = Facet::default();
- facet_reader.facet_from_ord(0, &mut facet).unwrap();
- assert_eq!(facet.to_path_string(), "/a/b");
- let doc = searcher
- .doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))
- .unwrap();
- let value = doc
- .get_first(facet_field)
- .and_then(|v| v.as_value().as_facet());
- assert_eq!(value, None);
- }
-
- #[test]
- fn test_facet_several_facets_sorted() {
- let mut schema_builder = SchemaBuilder::default();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(facet_field=>Facet::from_text("/parent/child1").unwrap()))
- .unwrap();
- index_writer
- .add_document(doc!(
- facet_field=>Facet::from_text("/parent/child2").unwrap(),
- facet_field=>Facet::from_text("/parent/child1/blop").unwrap(),
- ))
- .unwrap();
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
- let mut facet_ords = Vec::new();
-
- facet_ords.extend(facet_reader.facet_ords(0u32));
- assert_eq!(&facet_ords, &[0u64]);
-
- facet_ords.clear();
- facet_ords.extend(facet_reader.facet_ords(1u32));
- assert_eq!(&facet_ords, &[1u64, 2u64]);
-
- assert_eq!(facet_reader.num_facets(), 3);
- let mut facet = Facet::default();
- facet_reader.facet_from_ord(0, &mut facet).unwrap();
- assert_eq!(facet.to_path_string(), "/parent/child1");
- facet_reader.facet_from_ord(1, &mut facet).unwrap();
- assert_eq!(facet.to_path_string(), "/parent/child1/blop");
- facet_reader.facet_from_ord(2, &mut facet).unwrap();
- assert_eq!(facet.to_path_string(), "/parent/child2");
- }
-
- #[test]
- fn test_facet_stored_and_indexed() -> crate::Result<()> {
- let mut schema_builder = SchemaBuilder::default();
- let facet_field = schema_builder.add_facet_field("facet", STORED);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer = index.writer_for_tests()?;
- index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
- index_writer.commit()?;
- let searcher = index.reader()?.searcher();
- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
- let mut facet_ords = Vec::new();
- facet_ords.extend(facet_reader.facet_ords(0u32));
- assert_eq!(&facet_ords, &[0u64]);
- let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))?;
- let value: Option<Facet> = doc
- .get_first(facet_field)
- .and_then(|v| v.as_facet())
- .map(|facet| Facet::from_encoded_string(facet.to_string()));
- assert_eq!(value, Facet::from_text("/a/b").ok());
- Ok(())
- }
-
- #[test]
- fn test_facet_not_populated_for_all_docs() -> crate::Result<()> {
- let mut schema_builder = SchemaBuilder::default();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer = index.writer_for_tests()?;
- index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
- index_writer.add_document(TantivyDocument::default())?;
- index_writer.commit()?;
- let searcher = index.reader()?.searcher();
- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
- let mut facet_ords = Vec::new();
- facet_ords.extend(facet_reader.facet_ords(0u32));
- assert_eq!(&facet_ords, &[0u64]);
- facet_ords.clear();
- facet_ords.extend(facet_reader.facet_ords(1u32));
- assert!(facet_ords.is_empty());
- Ok(())
- }
-
- #[test]
- fn test_facet_not_populated_for_any_docs() -> crate::Result<()> {
- let mut schema_builder = SchemaBuilder::default();
- schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer = index.writer_for_tests()?;
- index_writer.add_document(TantivyDocument::default())?;
- index_writer.add_document(TantivyDocument::default())?;
- index_writer.commit()?;
- let searcher = index.reader()?.searcher();
- let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
- assert!(facet_reader.facet_ords(0u32).next().is_none());
- assert!(facet_reader.facet_ords(1u32).next().is_none());
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/fastfield/mod.rs b/crates/tantivy/src/fastfield/mod.rs
index 6b1074b5..ca7a981a 100644
--- a/crates/tantivy/src/fastfield/mod.rs
+++ b/crates/tantivy/src/fastfield/mod.rs
@@ -8,7 +8,7 @@
//! fields given a document id.
//!
//! Fast fields are useful when a field is required for all or most of
-//! the `DocSet`: for instance for scoring, grouping, aggregation, filtering, or faceting.
+//! the `DocSet`: for instance for scoring, grouping, aggregation or filtering
//!
//!
//! Fields have to be declared as `FAST` in the schema.
@@ -24,7 +24,6 @@ use crate::columnar::MonotonicallyMappableToU64;
pub use self::alive_bitset::{intersect_alive_bitsets, write_alive_bitset, AliveBitSet};
pub use self::error::{FastFieldNotAvailableError, Result};
-pub use self::facet_reader::FacetReader;
pub use self::readers::FastFieldReaders;
pub use self::writer::FastFieldsWriter;
use crate::schema::Type;
@@ -32,7 +31,6 @@ use crate::DateTime;
mod alive_bitset;
mod error;
-mod facet_reader;
mod readers;
mod writer;
@@ -75,11 +73,9 @@ impl FastValue for DateTime {
#[cfg(test)]
mod tests {
- use std::net::Ipv6Addr;
- use std::ops::{Range, RangeInclusive};
+ use std::ops::RangeInclusive;
use std::path::Path;
- use crate::columnar::StrColumn;
use crate::common::{ByteCount, DateTimePrecision, HasLen, TerminatingWrite};
use once_cell::sync::Lazy;
use rand::prelude::SliceRandom;
@@ -91,11 +87,10 @@ mod tests {
use crate::index::SegmentId;
use crate::merge_policy::NoMergePolicy;
use crate::schema::{
- DateOptions, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder,
- TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
+ DateOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument, FAST,
+ INDEXED,
};
use crate::time::OffsetDateTime;
- use crate::tokenizer::{LowerCaser, RawTokenizer, TextAnalyzer, TokenizerManager};
use crate::{Index, IndexWriter, SegmentReader};
pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {
@@ -134,10 +129,7 @@ mod tests {
assert_eq!(file.len(), 80);
let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
- let column = fast_field_readers
- .u64("field")
- .unwrap()
- .first_or_default_col(0);
+ let column = fast_field_readers.u64("field").unwrap().values;
assert_eq!(column.get_val(0), 13u64);
assert_eq!(column.get_val(1), 14u64);
assert_eq!(column.get_val(2), 2u64);
@@ -184,10 +176,7 @@ mod tests {
let file = directory.open_read(path).unwrap();
assert_eq!(file.len(), 108);
let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
- let col = fast_field_readers
- .u64("field")
- .unwrap()
- .first_or_default_col(0);
+ let col = fast_field_readers.u64("field").unwrap().values;
assert_eq!(col.get_val(0), 4u64);
assert_eq!(col.get_val(1), 14_082_001u64);
assert_eq!(col.get_val(2), 3_052u64);
@@ -217,10 +206,7 @@ mod tests {
let file = directory.open_read(path).unwrap();
assert_eq!(file.len(), 81);
let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
- let fast_field_reader = fast_field_readers
- .u64("field")
- .unwrap()
- .first_or_default_col(0);
+ let fast_field_reader = fast_field_readers.u64("field").unwrap().values;
for doc in 0..10_000 {
assert_eq!(fast_field_reader.get_val(doc), 100_000u64);
}
@@ -250,10 +236,7 @@ mod tests {
assert_eq!(file.len(), 4476);
{
let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
- let col = fast_field_readers
- .u64("field")
- .unwrap()
- .first_or_default_col(0);
+ let col = fast_field_readers.u64("field").unwrap().values;
for doc in 1..10_000 {
assert_eq!(col.get_val(doc), 5_000_000_000_000_000_000u64 + doc as u64);
}
@@ -284,10 +267,7 @@ mod tests {
{
let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
- let col = fast_field_readers
- .i64("field")
- .unwrap()
- .first_or_default_col(0);
+ let col = fast_field_readers.i64("field").unwrap().values;
assert_eq!(col.min_value(), -100i64);
assert_eq!(col.max_value(), 9_999i64);
for (doc, i) in (-100i64..10_000i64).enumerate() {
@@ -302,65 +282,6 @@ mod tests {
Ok(())
}
- #[test]
- fn test_signed_intfastfield_default_val() {
- let path = Path::new("test");
- let directory: RamDirectory = RamDirectory::create();
- let mut schema_builder = Schema::builder();
- schema_builder.add_i64_field("field", FAST);
- let schema = schema_builder.build();
-
- {
- let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
- let doc = TantivyDocument::default();
- fast_field_writers.add_document(&doc).unwrap();
- fast_field_writers.serialize(&mut write, None).unwrap();
- write.terminate().unwrap();
- }
-
- let file = directory.open_read(path).unwrap();
- let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
- let col = fast_field_readers.i64("field").unwrap();
- assert_eq!(col.first(0), None);
-
- let col = fast_field_readers
- .i64("field")
- .unwrap()
- .first_or_default_col(0);
- assert_eq!(col.get_val(0), 0);
- let col = fast_field_readers
- .i64("field")
- .unwrap()
- .first_or_default_col(-100);
- assert_eq!(col.get_val(0), -100);
- }
-
- #[test]
- fn test_date_fastfield_default() {
- let path = Path::new("test");
- let directory: RamDirectory = RamDirectory::create();
- let mut schema_builder = Schema::builder();
- schema_builder.add_date_field("date", FAST);
- let schema = schema_builder.build();
- {
- let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
- let doc = TantivyDocument::default();
- fast_field_writers.add_document(&doc).unwrap();
- fast_field_writers.serialize(&mut write, None).unwrap();
- write.terminate().unwrap();
- }
-
- let file = directory.open_read(path).unwrap();
- let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
- let col = fast_field_readers
- .date("date")
- .unwrap()
- .first_or_default_col(DateTime::default());
- assert_eq!(col.get_val(0), DateTime::default());
- }
-
// Warning: this generates the same permutation at each call
pub fn generate_permutation() -> Vec<u64> {
let mut permutation: Vec<u64> = (0u64..100_000u64).collect();
@@ -390,10 +311,7 @@ mod tests {
}
let file = directory.open_read(path).unwrap();
let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
- let col = fast_field_readers
- .u64("field")
- .unwrap()
- .first_or_default_col(0);
+ let col = fast_field_readers.u64("field").unwrap().values;
for a in 0..n {
assert_eq!(col.get_val(a as u32), permutation[a]);
}
@@ -438,250 +356,6 @@ mod tests {
assert_eq!(reader.searcher().segment_readers().len(), 1);
}
- fn get_vals_for_docs(column: &Column<u64>, docs: Range<u32>) -> Vec<u64> {
- docs.into_iter()
- .flat_map(|doc| column.values_for_doc(doc))
- .collect()
- }
-
- #[test]
- fn test_text_fastfield() {
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("text", TEXT | FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- {
- // first segment
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- index_writer
- .add_document(doc!(
- text_field => "BBBBB", // term ord 1
- text_field => "AAAAA", // term ord 0
- ))
- .unwrap();
- index_writer.add_document(doc!()).unwrap();
- index_writer
- .add_document(doc!(
- text_field => "AAAAA", // term_ord 0
- ))
- .unwrap();
- index_writer
- .add_document(doc!(
- text_field => "AAAAA",
- text_field => "BBBBB",
- ))
- .unwrap();
- index_writer
- .add_document(doc!(
- text_field => "zumberthree", // term_ord 2, after merge term_ord 3
- ))
- .unwrap();
-
- index_writer.add_document(doc!()).unwrap();
- index_writer.commit().unwrap();
-
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 1);
- let segment_reader = searcher.segment_reader(0);
- let fast_fields = segment_reader.fast_fields();
- let str_column = fast_fields.str("text").unwrap().unwrap();
- assert!(str_column.ords().values_for_doc(0u32).eq([1, 0]),);
- assert!(str_column.ords().values_for_doc(1u32).next().is_none());
- assert!(str_column.ords().values_for_doc(2u32).eq([0]),);
- assert!(str_column.ords().values_for_doc(3u32).eq([0, 1]),);
- assert!(str_column.ords().values_for_doc(4u32).eq([2]),);
-
- let mut str_term = String::default();
- assert!(str_column.ord_to_str(0, &mut str_term).unwrap());
- assert_eq!("AAAAA", &str_term);
-
- let inverted_index = segment_reader.inverted_index(text_field).unwrap();
- assert_eq!(inverted_index.terms().num_terms(), 3);
- let mut bytes = vec![];
- assert!(inverted_index.terms().ord_to_term(0, &mut bytes).unwrap());
- assert_eq!(bytes, "aaaaa".as_bytes());
- }
-
- {
- // second segment
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-
- index_writer
- .add_document(doc!(
- text_field => "AAAAA", // term_ord 0
- ))
- .unwrap();
-
- index_writer
- .add_document(doc!(
- text_field => "CCCCC AAAAA", // term_ord 1, after merge 2
- ))
- .unwrap();
-
- index_writer.add_document(doc!()).unwrap();
- index_writer.commit().unwrap();
-
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 2);
- let segment_reader = searcher.segment_reader(1);
- let fast_fields = segment_reader.fast_fields();
- let text_fast_field = fast_fields.str("text").unwrap().unwrap();
-
- assert_eq!(&get_vals_for_docs(text_fast_field.ords(), 0..2), &[0, 1]);
- }
-
- // TODO uncomment once merging is available
- // Merging the segments
- {
- let segment_ids = index.searchable_segment_ids().unwrap();
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer.merge(&segment_ids).wait().unwrap();
- index_writer.wait_merging_threads().unwrap();
- }
-
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- let segment_reader = searcher.segment_reader(0);
- let fast_fields = segment_reader.fast_fields();
- let text_column = fast_fields.str("text").unwrap().unwrap();
-
- assert_eq!(
- get_vals_for_docs(text_column.ords(), 0..8),
- vec![1, 0, 0, 0, 1, 3 /* next segment */, 0, 2]
- );
- }
-
- #[test]
- fn test_string_fastfield_simple() {
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("text", TEXT | FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer = index.writer_for_tests().unwrap();
- writer.add_document(doc!(text_field=>"hello happy tax payer", text_field=>"aaa this string comes lexicographically before the other one.")).unwrap();
- writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let segment_reader = searcher.segment_reader(0);
- let str_column = segment_reader.fast_fields().str("text").unwrap().unwrap();
- // The string values are not sorted here.
- let term_ords: Vec<u64> = str_column.term_ords(0u32).collect();
- assert_eq!(&term_ords, &[1, 0]);
- }
-
- #[test]
- fn test_facet_fastfield_simple() {
- let mut schema_builder = Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer = index.writer_for_tests().unwrap();
- writer
- .add_document(doc!(facet_field=>Facet::from("/a/2"), facet_field=>Facet::from("/a/1")))
- .unwrap();
- writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let segment_reader = searcher.segment_reader(0);
- let facet_reader = segment_reader.facet_reader("facet").unwrap();
- // facets, contrary to strings are sorted.
- let mut facet_ords = Vec::new();
- facet_ords.extend(facet_reader.facet_ords(0u32));
- assert_eq!(&facet_ords, &[0, 1]);
- }
-
- #[test]
- fn test_string_fastfield() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("text", STRING | FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- {
- // first segment
- let mut index_writer = index.writer_for_tests()?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- index_writer.add_document(doc!(
- text_field => "BBBBB", // term_ord 1
- ))?;
- index_writer.add_document(doc!())?;
- index_writer.add_document(doc!(
- text_field => "AAAAA", // term_ord 0
- ))?;
- index_writer.add_document(doc!(
- text_field => "AAAAA", // term_ord 0
- ))?;
- index_writer.add_document(doc!(
- text_field => "zumberthree", // term_ord 2, after merge term_ord 3
- ))?;
-
- index_writer.add_document(doc!())?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 1);
- let segment_reader = searcher.segment_reader(0);
- let fast_fields = segment_reader.fast_fields();
- let text_col = fast_fields.str("text").unwrap().unwrap();
-
- assert_eq!(get_vals_for_docs(text_col.ords(), 0..6), vec![1, 0, 0, 2]);
-
- let inverted_index = segment_reader.inverted_index(text_field)?;
- assert_eq!(inverted_index.terms().num_terms(), 3);
- let mut bytes = vec![];
- assert!(inverted_index.terms().ord_to_term(0, &mut bytes)?);
- assert_eq!(bytes, "AAAAA".as_bytes());
- }
-
- {
- // second segment
- let mut index_writer = index.writer_for_tests()?;
-
- index_writer.add_document(doc!(
- text_field => "AAAAA", // term_ord 0
- ))?;
-
- index_writer.add_document(doc!(
- text_field => "CCCCC", // term_ord 1, after merge 2
- ))?;
-
- index_writer.add_document(doc!())?;
- index_writer.commit()?;
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 2);
- let segment_reader = searcher.segment_reader(1);
- let fast_fields = segment_reader.fast_fields();
- let text_fast_field = fast_fields.str("text").unwrap().unwrap();
-
- assert_eq!(&get_vals_for_docs(text_fast_field.ords(), 0..2), &[0, 1]);
- }
- // Merging the segments
- {
- let segment_ids = index.searchable_segment_ids()?;
- let mut index_writer: IndexWriter = index.writer_for_tests()?;
- index_writer.merge(&segment_ids).wait()?;
- index_writer.wait_merging_threads()?;
- }
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let segment_reader = searcher.segment_reader(0);
- let fast_fields = segment_reader.fast_fields();
- let text_fast_field = fast_fields.str("text").unwrap().unwrap();
-
- assert_eq!(
- get_vals_for_docs(text_fast_field.ords(), 0..9),
- vec![1, 0, 0, 3 /* next segment */, 0, 2]
- );
-
- Ok(())
- }
-
#[test]
fn test_datefastfield() -> crate::Result<()> {
let mut schema_builder = Schema::builder();
@@ -708,6 +382,7 @@ mod tests {
date_field => DateTime::from_u64(4i64.to_u64())
))?;
index_writer.add_document(doc!(
+ date_field => DateTime::from_u64(0i64.to_u64()),
multi_date_field => DateTime::from_u64(5i64.to_u64()),
multi_date_field => DateTime::from_u64(6i64.to_u64())
))?;
@@ -721,31 +396,11 @@ mod tests {
.column_opt::<DateTime>("date")
.unwrap()
.unwrap()
- .first_or_default_col(Default::default());
- let dates_fast_field = fast_fields
- .column_opt::<DateTime>("multi_date")
- .unwrap()
- .unwrap();
- let mut dates = Vec::new();
- {
- assert_eq!(date_fast_field.get_val(0).into_timestamp_nanos(), 1i64);
- dates_fast_field.fill_vals(0u32, &mut dates);
- assert_eq!(dates.len(), 2);
- assert_eq!(dates[0].into_timestamp_nanos(), 2i64);
- assert_eq!(dates[1].into_timestamp_nanos(), 3i64);
- }
- {
- assert_eq!(date_fast_field.get_val(1).into_timestamp_nanos(), 4i64);
- dates_fast_field.fill_vals(1u32, &mut dates);
- assert!(dates.is_empty());
- }
- {
- assert_eq!(date_fast_field.get_val(2).into_timestamp_nanos(), 0i64);
- dates_fast_field.fill_vals(2u32, &mut dates);
- assert_eq!(dates.len(), 2);
- assert_eq!(dates[0].into_timestamp_nanos(), 5i64);
- assert_eq!(dates[1].into_timestamp_nanos(), 6i64);
- }
+ .values;
+
+ assert_eq!(date_fast_field.get_val(0).into_timestamp_nanos(), 1i64);
+ assert_eq!(date_fast_field.get_val(1).into_timestamp_nanos(), 4i64);
+ assert_eq!(date_fast_field.get_val(2).into_timestamp_nanos(), 0i64);
Ok(())
}
@@ -815,38 +470,6 @@ mod tests {
}
}
- #[test]
- pub fn test_fastfield_bool_default_value() {
- let path = Path::new("test_bool");
- let directory: RamDirectory = RamDirectory::create();
- let mut schema_builder = Schema::builder();
- schema_builder.add_bool_field("field_bool", FAST);
- let schema = schema_builder.build();
- {
- let mut write: WritePtr = directory.open_write(path).unwrap();
- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
- let doc = TantivyDocument::default();
- fast_field_writers.add_document(&doc).unwrap();
- fast_field_writers.serialize(&mut write, None).unwrap();
- write.terminate().unwrap();
- }
- let file = directory.open_read(path).unwrap();
- assert_eq!(file.len(), 86);
- let fastfield_readers = FastFieldReaders::open(file, schema).unwrap();
- let col = fastfield_readers.bool("field_bool").unwrap();
- assert_eq!(col.first(0), None);
- let col = fastfield_readers
- .bool("field_bool")
- .unwrap()
- .first_or_default_col(false);
- assert_eq!(col.get_val(0), false);
- let col = fastfield_readers
- .bool("field_bool")
- .unwrap()
- .first_or_default_col(true);
- assert_eq!(col.get_val(0), true);
- }
-
fn get_index(docs: &[crate::TantivyDocument], schema: &Schema) -> crate::Result<RamDirectory> {
let directory: RamDirectory = RamDirectory::create();
{
@@ -934,18 +557,14 @@ mod tests {
let reader = index.reader().unwrap();
let searcher = reader.searcher();
let segment = &searcher.segment_readers()[0];
- let field = segment
- .fast_fields()
- .u64("url_norm_hash")
- .unwrap()
- .first_or_default_col(0);
+ let field = segment.fast_fields().u64("url_norm_hash").unwrap().values;
let numbers = [100, 200, 300];
let test_range = |range: RangeInclusive<u64>| {
- let expexted_count = numbers.iter().filter(|num| range.contains(num)).count();
+ let expected_count = numbers.iter().filter(|num| range.contains(num)).count();
let mut vec = vec![];
field.get_row_ids_for_value_range(range, 0..u32::MAX, &mut vec);
- assert_eq!(vec.len(), expexted_count);
+ assert_eq!(vec.len(), expected_count);
};
test_range(50..=50);
test_range(150..=150);
@@ -957,31 +576,6 @@ mod tests {
test_range(100..=299);
}
- #[test]
- fn test_ip_addr_columnar_simple() {
- let mut schema_builder = Schema::builder();
- let ip_field = schema_builder.add_u64_field("ip", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- let ip_addr = Ipv6Addr::new(1, 2, 3, 4, 5, 1, 2, 3);
- index_writer
- .add_document(TantivyDocument::default())
- .unwrap();
- index_writer.add_document(doc!(ip_field=>ip_addr)).unwrap();
- index_writer
- .add_document(TantivyDocument::default())
- .unwrap();
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let fastfields = searcher.segment_reader(0u32).fast_fields();
- let column: Column<Ipv6Addr> = fastfields.column_opt("ip").unwrap().unwrap();
- assert_eq!(column.num_docs(), 3);
- assert_eq!(column.first(0), None);
- assert_eq!(column.first(1), Some(ip_addr));
- assert_eq!(column.first(2), None);
- }
-
#[test]
fn test_mapping_bug_docids_for_value_range() {
let mut schema_builder = Schema::builder();
@@ -1012,11 +606,7 @@ mod tests {
let reader = index.reader().unwrap();
let searcher = reader.searcher();
let segment = &searcher.segment_readers()[0];
- let field = segment
- .fast_fields()
- .u64("url_norm_hash")
- .unwrap()
- .first_or_default_col(0);
+ let field = segment.fast_fields().u64("url_norm_hash").unwrap().values;
let numbers = [1000, 1001, 1003];
let test_range = |range: RangeInclusive<u64>| {
@@ -1050,40 +640,6 @@ mod tests {
test_range_variant(1000, 1002);
}
- #[test]
- fn test_json_object_fast_field() {
- let mut schema_builder = Schema::builder();
- let without_fast_field = schema_builder.add_json_field("without", STORED);
- let with_fast_field = schema_builder.add_json_field("with", STORED | FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer = index.writer_for_tests().unwrap();
- writer
- .add_document(doc!(without_fast_field=>json!({"hello": "without"})))
- .unwrap();
- writer
- .add_document(doc!(with_fast_field=>json!({"hello": "with"})))
- .unwrap();
- writer
- .add_document(doc!(with_fast_field=>json!({"hello": "with2"})))
- .unwrap();
- writer
- .add_document(doc!(with_fast_field=>json!({"hello": "with1"})))
- .unwrap();
- writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let segment_reader = searcher.segment_reader(0u32);
- let fast_fields = segment_reader.fast_fields();
- let column_without_opt = fast_fields.str("without.hello");
- assert!(column_without_opt.is_err());
- let column_with_opt: Option<StrColumn> = fast_fields.str("with.hello").unwrap();
- let column_with: StrColumn = column_with_opt.unwrap();
- assert!(column_with.term_ords(0).next().is_none());
- assert!(column_with.term_ords(1).eq([0]));
- assert!(column_with.term_ords(2).eq([2]));
- assert!(column_with.term_ords(3).eq([1]));
- }
-
#[test]
fn test_fast_field_in_json_field_expand_dots_disabled() {
let mut schema_builder = Schema::builder();
@@ -1106,33 +662,7 @@ mod tests {
.column_opt::<i64>(r"json.attr\.age")
.unwrap()
.unwrap();
- let vals: Vec<i64> = column.values_for_doc(0u32).collect();
- assert_eq!(&vals, &[32])
- }
-
- #[test]
- fn test_fast_field_in_json_field_with_tokenizer() {
- let mut schema_builder = Schema::builder();
- let json_option = JsonObjectOptions::default().set_fast(Some("default"));
- let json = schema_builder.add_json_field("json", json_option);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(json => json!({"age": 32})))
- .unwrap();
- index_writer
- .add_document(doc!(json => json!({"age": "NEW"})))
- .unwrap();
-
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let fast_fields = searcher.segment_reader(0u32).fast_fields();
-
- let ff_str = fast_fields.str("json.age").unwrap().unwrap();
- let mut output = String::new();
- ff_str.ord_to_str(0, &mut output).unwrap();
- assert_eq!(output, "new");
+ assert_eq!(column.first(0u32), Some(32))
}
#[test]
@@ -1156,8 +686,7 @@ mod tests {
.column_opt::<i64>(test_column_name)
.unwrap()
.unwrap();
- let vals: Vec<i64> = column.values_for_doc(0u32).collect();
- assert_eq!(&vals, &[32]);
+ assert_eq!(column.first(0u32), Some(32))
}
}
@@ -1178,8 +707,7 @@ mod tests {
.column_opt::<i64>("field.with.dot")
.unwrap()
.unwrap();
- let vals: Vec<i64> = column.values_for_doc(0u32).collect();
- assert_eq!(&vals, &[32]);
+ assert_eq!(column.first(0u32), Some(32));
}
#[test]
@@ -1200,76 +728,7 @@ mod tests {
.column_opt::<i64>("jsonfield.attr.age")
.unwrap()
.unwrap();
- let vals: Vec<i64> = column.values_for_doc(0u32).collect();
- assert_eq!(&vals, &[33]);
- }
-
- #[test]
- fn test_fast_field_tokenizer() {
- let mut schema_builder = Schema::builder();
- let opt = TextOptions::default().set_fast(Some("custom_lowercase"));
- let text_field = schema_builder.add_text_field("text", opt);
- let schema = schema_builder.build();
- let ff_tokenizer_manager = TokenizerManager::default();
- ff_tokenizer_manager.register(
- "custom_lowercase",
- TextAnalyzer::builder(RawTokenizer::default())
- .filter(LowerCaser)
- .build(),
- );
-
- let mut index = Index::create_in_ram(schema);
- index.set_fast_field_tokenizers(ff_tokenizer_manager);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(text_field => "Test1 test2"))
- .unwrap();
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();
- let column = fast_field_reader.str("text").unwrap().unwrap();
- let mut out = String::new();
- column.ord_to_str(0u64, &mut out).unwrap();
- assert_eq!(&out, "test1 test2");
- }
-
- #[test]
- fn test_text_fast_field_tokenizer() {
- let mut schema_builder = Schema::builder();
-
- let text_fieldtype = crate::schema::TextOptions::default()
- .set_indexing_options(
- crate::schema::TextFieldIndexing::default()
- .set_index_option(crate::schema::IndexRecordOption::WithFreqs)
- .set_tokenizer("raw"),
- )
- .set_fast(Some("default"))
- .set_stored();
-
- let log_field = schema_builder.add_text_field("log_level", text_fieldtype);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(log_field => "info"))
- .unwrap();
- index_writer
- .add_document(doc!(log_field => "INFO"))
- .unwrap();
- index_writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();
-
- let text_fast_field = fast_field_reader.str("log_level").unwrap().unwrap();
- let mut buffer = String::new();
- assert!(text_fast_field.ord_to_str(0, &mut buffer).unwrap());
- assert_eq!(buffer, "info");
- assert!(!text_fast_field.ord_to_str(1, &mut buffer).unwrap());
-
- assert!(text_fast_field.term_ords(0).eq([0].into_iter()));
- assert!(text_fast_field.term_ords(1).eq([0].into_iter()));
- assert!(text_fast_field.ords().values_for_doc(0u32).eq([0]));
- assert!(text_fast_field.ords().values_for_doc(1u32).eq([0]));
+ assert_eq!(column.first(0u32), Some(33))
}
#[test]
@@ -1294,13 +753,11 @@ mod tests {
.column_opt::<i64>("jsonfield.attr.age")
.unwrap()
.unwrap();
- let vals: Vec<i64> = column.values_for_doc(0u32).collect();
- assert_eq!(&vals, &[33]);
+ assert_eq!(column.first(0u32), Some(33));
let column = fast_field_reader
.column_opt::<i64>("jsonfield\\.attr.age")
.unwrap()
.unwrap();
- let vals: Vec<i64> = column.values_for_doc(0u32).collect();
- assert_eq!(&vals, &[33]);
+ assert_eq!(column.first(0u32), Some(33));
}
}
diff --git a/crates/tantivy/src/fastfield/readers.rs b/crates/tantivy/src/fastfield/readers.rs
index 401de1d6..735f9835 100644
--- a/crates/tantivy/src/fastfield/readers.rs
+++ b/crates/tantivy/src/fastfield/readers.rs
@@ -1,10 +1,9 @@
use std::io;
-use std::net::Ipv6Addr;
use std::sync::Arc;
use crate::columnar::{
- BytesColumn, Column, ColumnType, ColumnValues, ColumnarReader, DynamicColumn,
- DynamicColumnHandle, HasAssociatedColumnType, StrColumn,
+ BytesColumn, Column, ColumnType, ColumnarReader, DynamicColumn, DynamicColumnHandle,
+ HasAssociatedColumnType,
};
use crate::common::ByteCount;
@@ -139,20 +138,6 @@ impl FastFieldReaders {
.sum())
}
- /// Returns a typed column value object.
- ///
- /// In that column value:
- /// - Rows with no value are associated with the default value.
- /// - Rows with several values are associated with the first value.
- pub fn column_first_or_default<T>(&self, field: &str) -> crate::Result<Arc<dyn ColumnValues<T>>>
- where
- T: PartialOrd + Copy + HasAssociatedColumnType + Send + Sync + 'static,
- DynamicColumn: Into<Option<Column<T>>>,
- {
- let col: Column<T> = self.column(field)?;
- Ok(col.first_or_default_col(T::default_value()))
- }
-
/// Returns a typed column associated to a given field name.
///
/// Returns an error if no column associated with that field_name exists.
@@ -183,24 +168,6 @@ impl FastFieldReaders {
self.column(field)
}
- /// Returns the `ip` fast field reader reader associated to `field`.
- ///
- /// If `field` is not a u128 fast field, this method returns an Error.
- pub fn ip_addr(&self, field: &str) -> crate::Result<Column<Ipv6Addr>> {
- self.column(field)
- }
-
- /// Returns a `str` column.
- pub fn str(&self, field_name: &str) -> crate::Result<Option<StrColumn>> {
- let Some(dynamic_column_handle) =
- self.dynamic_column_handle(field_name, ColumnType::Str)?
- else {
- return Ok(None);
- };
- let dynamic_column = dynamic_column_handle.open()?;
- Ok(dynamic_column.into())
- }
-
/// Returns a `bytes` column.
pub fn bytes(&self, field_name: &str) -> crate::Result<Option<BytesColumn>> {
let Some(dynamic_column_handle) =
@@ -460,16 +427,13 @@ mod tests {
assert_eq!(id_columns.first().unwrap().column_type(), ColumnType::U64);
let foo_columns = fast_fields.dynamic_column_handles("json.foo").unwrap();
- assert_eq!(foo_columns.len(), 3);
+ assert_eq!(foo_columns.len(), 2);
assert!(foo_columns
.iter()
.any(|column| column.column_type() == ColumnType::I64));
assert!(foo_columns
.iter()
.any(|column| column.column_type() == ColumnType::Bool));
- assert!(foo_columns
- .iter()
- .any(|column| column.column_type() == ColumnType::Str));
println!("*** {:?}", fast_fields.columnar().list_columns());
}
diff --git a/crates/tantivy/src/fastfield/writer.rs b/crates/tantivy/src/fastfield/writer.rs
index a8c7d1a5..9eefa602 100644
--- a/crates/tantivy/src/fastfield/writer.rs
+++ b/crates/tantivy/src/fastfield/writer.rs
@@ -2,11 +2,10 @@ use std::io;
use crate::columnar::{ColumnarWriter, NumericalValue};
use crate::common::{DateTimePrecision, JsonPathWriter};
-use crate::tokenizer_api::Token;
use crate::indexer::doc_id_mapping::DocIdMapping;
use crate::schema::document::{Document, ReferenceValue, ReferenceValueLeaf, Value};
-use crate::schema::{value_type_to_column_type, Field, FieldType, Schema, Type};
+use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};
use crate::tokenizer::{TextAnalyzer, TokenizerManager};
use crate::{DocId, TantivyError};
@@ -81,7 +80,7 @@ impl FastFieldsWriter {
}
}
- let sort_values_within_row = value_type == Type::Facet;
+ let sort_values_within_row = false;
if let Some(column_type) = value_type_to_column_type(value_type) {
columnar_writer.record_column_type(
field_entry.name(),
@@ -142,19 +141,7 @@ impl FastFieldsWriter {
match value.as_value() {
ReferenceValue::Leaf(leaf) => match leaf {
ReferenceValueLeaf::Null => {}
- ReferenceValueLeaf::Str(val) => {
- if let Some(tokenizer) =
- &mut self.per_field_tokenizer[field.field_id() as usize]
- {
- let mut token_stream = tokenizer.token_stream(val);
- token_stream.process(&mut |token: &Token| {
- self.columnar_writer
- .record_str(doc_id, field_name, &token.text);
- })
- } else {
- self.columnar_writer.record_str(doc_id, field_name, val);
- }
- }
+ ReferenceValueLeaf::Str(_) => {}
ReferenceValueLeaf::U64(val) => {
self.columnar_writer.record_numerical(
doc_id,
@@ -182,24 +169,14 @@ impl FastFieldsWriter {
self.columnar_writer
.record_datetime(doc_id, field_name, truncated_datetime);
}
- ReferenceValueLeaf::Facet(val) => {
- self.columnar_writer.record_str(doc_id, field_name, val);
- }
ReferenceValueLeaf::Bytes(val) => {
self.columnar_writer.record_bytes(doc_id, field_name, val);
}
- ReferenceValueLeaf::IpAddr(val) => {
- self.columnar_writer.record_ip_addr(doc_id, field_name, val);
- }
+ ReferenceValueLeaf::IpAddr(_) => {}
ReferenceValueLeaf::Bool(val) => {
self.columnar_writer.record_bool(doc_id, field_name, val);
}
- ReferenceValueLeaf::PreTokStr(val) => {
- for token in &val.tokens {
- self.columnar_writer
- .record_str(doc_id, field_name, &token.text);
- }
- }
+ ReferenceValueLeaf::PreTokStr(_) => {}
},
ReferenceValue::Array(val) => {
// TODO: Check this is the correct behaviour we want.
@@ -285,16 +262,7 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
match json_val.as_value() {
ReferenceValue::Leaf(leaf) => match leaf {
ReferenceValueLeaf::Null => {} // TODO: Handle null
- ReferenceValueLeaf::Str(val) => {
- if let Some(text_analyzer) = tokenizer.as_mut() {
- let mut token_stream = text_analyzer.token_stream(val);
- token_stream.process(&mut |token| {
- columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);
- })
- } else {
- columnar_writer.record_str(doc, json_path_writer.as_str(), val);
- }
- }
+ ReferenceValueLeaf::Str(_) => {}
ReferenceValueLeaf::U64(val) => {
columnar_writer.record_numerical(
doc,
@@ -322,9 +290,6 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
ReferenceValueLeaf::Date(val) => {
columnar_writer.record_datetime(doc, json_path_writer.as_str(), val);
}
- ReferenceValueLeaf::Facet(_) => {
- unimplemented!("Facet support in dynamic fields is not yet implemented")
- }
ReferenceValueLeaf::Bytes(_) => {
// TODO: This can be re added once it is added to the JSON Utils section as well.
// columnar_writer.record_bytes(doc, json_path_writer.as_str(), val);
@@ -363,140 +328,3 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
}
}
}
-
-#[cfg(test)]
-mod tests {
- use crate::columnar::{Column, ColumnarReader, ColumnarWriter, StrColumn};
- use crate::common::JsonPathWriter;
-
- use super::record_json_value_to_columnar_writer;
- use crate::fastfield::writer::JSON_DEPTH_LIMIT;
- use crate::DocId;
-
- fn test_columnar_from_jsons_aux(
- json_docs: &[serde_json::Value],
- expand_dots: bool,
- ) -> ColumnarReader {
- let mut columnar_writer = ColumnarWriter::default();
- let mut json_path = JsonPathWriter::default();
- json_path.set_expand_dots(expand_dots);
- for (doc, json_doc) in json_docs.iter().enumerate() {
- record_json_value_to_columnar_writer(
- doc as u32,
- json_doc,
- JSON_DEPTH_LIMIT,
- &mut json_path,
- &mut columnar_writer,
- &mut None,
- );
- }
- let mut buffer = Vec::new();
- columnar_writer
- .serialize(json_docs.len() as DocId, None, &mut buffer)
- .unwrap();
- ColumnarReader::open(buffer).unwrap()
- }
-
- #[test]
- fn test_json_fastfield_record_simple() {
- let json_doc = serde_json::json!({
- "float": 1.02,
- "text": "hello happy tax payer",
- "nested": {"child": 3, "child2": 5},
- "arr": ["hello", "happy", "tax", "payer"]
- });
- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);
- let columns = columnar_reader.list_columns().unwrap();
- {
- assert_eq!(columns[0].0, "arr");
- let column_arr_opt: Option<StrColumn> = columns[0].1.open().unwrap().into();
- assert!(column_arr_opt
- .unwrap()
- .term_ords(0)
- .eq([1, 0, 3, 2].into_iter()));
- }
- {
- assert_eq!(columns[1].0, "float");
- let column_float_opt: Option<Column<f64>> = columns[1].1.open().unwrap().into();
- assert!(column_float_opt
- .unwrap()
- .values_for_doc(0)
- .eq([1.02f64].into_iter()));
- }
- {
- assert_eq!(columns[2].0, "nested\u{1}child");
- let column_nest_child_opt: Option<Column<i64>> = columns[2].1.open().unwrap().into();
- assert!(column_nest_child_opt
- .unwrap()
- .values_for_doc(0)
- .eq([3].into_iter()));
- }
- {
- assert_eq!(columns[3].0, "nested\u{1}child2");
- let column_nest_child2_opt: Option<Column<i64>> = columns[3].1.open().unwrap().into();
- assert!(column_nest_child2_opt
- .unwrap()
- .values_for_doc(0)
- .eq([5].into_iter()));
- }
- {
- assert_eq!(columns[4].0, "text");
- let column_text_opt: Option<StrColumn> = columns[4].1.open().unwrap().into();
- assert!(column_text_opt.unwrap().term_ords(0).eq([0].into_iter()));
- }
- }
-
- #[test]
- fn test_json_fastfield_deep_obj() {
- let json_doc = serde_json::json!(
- {"a": {"a": {"a": {"a": {"a":
- {"a": {"a": {"a": {"a": {"a":
- {"a": {"a": {"a": {"a": {"a":
- {"a": {"a": {"a": {"depth_accepted": 19, "a": { "depth_truncated": 20}
- }}}}}}}}}}}}}}}}}}});
- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);
- let columns = columnar_reader.list_columns().unwrap();
- assert_eq!(columns.len(), 1);
- assert!(columns[0].0.ends_with("a\u{1}a\u{1}a\u{1}depth_accepted"));
- }
-
- #[test]
- fn test_json_fastfield_deep_arr() {
- let json_doc = json!(
- {"obj":
- [[[[[,
- [[[[[,
- [[[[[,
- [[18, [19, //< within limits
- [20]]]]]]]]]]]]]]]]]]]});
- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);
- let columns = columnar_reader.list_columns().unwrap();
- assert_eq!(columns.len(), 1);
- assert_eq!(columns[0].0, "obj");
- let dynamic_column = columns[0].1.open().unwrap();
- let col: Option<Column<i64>> = dynamic_column.into();
- let vals: Vec<i64> = col.unwrap().values_for_doc(0).collect();
- assert_eq!(&vals, &[18, 19])
- }
-
- #[test]
- fn test_json_fast_field_do_not_expand_dots() {
- let json_doc = json!({"field.with.dots": {"child.with.dot": "hello"}});
- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], false);
- let columns = columnar_reader.list_columns().unwrap();
- assert_eq!(columns.len(), 1);
- assert_eq!(columns[0].0, "field.with.dots\u{1}child.with.dot");
- }
-
- #[test]
- fn test_json_fast_field_expand_dots() {
- let json_doc = json!({"field.with.dots": {"child.with.dot": "hello"}});
- let columnar_reader = test_columnar_from_jsons_aux(&[json_doc], true);
- let columns = columnar_reader.list_columns().unwrap();
- assert_eq!(columns.len(), 1);
- assert_eq!(
- columns[0].0,
- "field\u{1}with\u{1}dots\u{1}child\u{1}with\u{1}dot"
- );
- }
-}
diff --git a/crates/tantivy/src/index/segment_reader.rs b/crates/tantivy/src/index/segment_reader.rs
index c927a7d0..c2f2b49f 100644
--- a/crates/tantivy/src/index/segment_reader.rs
+++ b/crates/tantivy/src/index/segment_reader.rs
@@ -8,7 +8,7 @@ use itertools::Itertools;
use crate::directory::{CompositeFile, FileSlice};
use crate::error::DataCorruption;
-use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};
+use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FastFieldReaders};
use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
use crate::index::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
use crate::json_utils::json_path_sep_to_dot;
@@ -92,22 +92,6 @@ impl SegmentReader {
&self.fast_fields_readers
}
- /// Accessor to the `FacetReader` associated with a given `Field`.
- pub fn facet_reader(&self, field_name: &str) -> crate::Result<FacetReader> {
- let schema = self.schema();
- let field = schema.get_field(field_name)?;
- let field_entry = schema.get_field_entry(field);
- if field_entry.field_type().value_type() != Type::Facet {
- return Err(crate::TantivyError::SchemaError(format!(
- "`{field_name}` is not a facet field.`"
- )));
- }
- let Some(facet_column) = self.fast_fields().str(field_name)? else {
- panic!("Facet Field `{field_name}` is missing. This should not happen");
- };
- Ok(FacetReader::new(facet_column))
- }
-
/// Accessor to the segment's `Field norms`'s reader.
///
/// Field norms are the length (in tokens) of the fields.
diff --git a/crates/tantivy/src/indexer/doc_id_mapping.rs b/crates/tantivy/src/indexer/doc_id_mapping.rs
index 9869d78a..1ebeb42b 100644
--- a/crates/tantivy/src/indexer/doc_id_mapping.rs
+++ b/crates/tantivy/src/indexer/doc_id_mapping.rs
@@ -467,22 +467,11 @@ mod tests_indexsorting {
let segment_reader = searcher.segment_reader(0);
let fast_fields = segment_reader.fast_fields();
- let fast_field = fast_fields
- .u64("my_number")
- .unwrap()
- .first_or_default_col(999);
+ let fast_field = fast_fields.u64("my_number").unwrap().values;
assert_eq!(fast_field.get_val(0), 10u64);
assert_eq!(fast_field.get_val(1), 20u64);
assert_eq!(fast_field.get_val(2), 30u64);
- let multifield = fast_fields.u64("multi_numbers").unwrap();
- let vals: Vec<u64> = multifield.values_for_doc(0u32).collect();
- assert_eq!(vals, &[] as &[u64]);
- let vals: Vec<_> = multifield.values_for_doc(1u32).collect();
- assert_eq!(vals, &[5, 6]);
-
- let vals: Vec<_> = multifield.values_for_doc(2u32).collect();
- assert_eq!(vals, &[3]);
Ok(())
}
@@ -523,10 +512,7 @@ mod tests_indexsorting {
let segment_reader = searcher.segment_reader(0);
let fast_fields = segment_reader.fast_fields();
- let fast_field = fast_fields
- .date("date")
- .unwrap()
- .first_or_default_col(DateTime::from_timestamp_secs(0));
+ let fast_field = fast_fields.date("date").unwrap().values;
assert_eq!(fast_field.get_val(0), DateTime::from_timestamp_secs(1001));
assert_eq!(fast_field.get_val(1), DateTime::from_timestamp_secs(1000));
assert_eq!(fast_field.get_val(2), DateTime::from_timestamp_secs(999));
diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rs
index a6c3924c..0392789f 100644
--- a/crates/tantivy/src/indexer/index_writer.rs
+++ b/crates/tantivy/src/indexer/index_writer.rs
@@ -770,10 +770,8 @@ impl<D: Document> Drop for IndexWriter<D> {
#[cfg(test)]
mod tests {
use std::collections::{HashMap, HashSet};
- use std::net::Ipv6Addr;
- use crate::columnar::{Cardinality, Column, MonotonicallyMappableToU128};
- use itertools::Itertools;
+ use crate::columnar::{Column, MonotonicallyMappableToU128};
use proptest::prop_oneof;
use super::super::operation::UserOperation;
@@ -784,14 +782,15 @@ mod tests {
use crate::indexer::NoMergePolicy;
use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
use crate::schema::{
- self, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions, Schema,
- TextFieldIndexing, TextOptions, Value, FAST, INDEXED, STORED, STRING, TEXT,
+ self, IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions, Value,
+ FAST, INDEXED, STORED, STRING, TEXT,
};
use crate::store::DOCSTORE_CACHE_CAPACITY;
use crate::{
DateTime, DocAddress, Index, IndexSettings, IndexSortByField, IndexWriter, Order,
ReloadPolicy, TantivyDocument, Term,
};
+ use proptest::prelude::*;
const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do \
eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad \
@@ -1378,59 +1377,6 @@ mod tests {
assert!(commit_again.is_ok());
}
- #[test]
- fn test_delete_and_merge_removes_terms_fast_field_dict() {
- let mut schema_builder = schema::Schema::builder();
- let text_field = schema_builder.add_text_field("text", STRING | FAST);
- let schema = schema_builder.build();
-
- let index = Index::builder().schema(schema).create_in_ram().unwrap();
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(text_field => "one"))
- .unwrap();
- index_writer
- .add_document(doc!(text_field => "two"))
- .unwrap();
- index_writer
- .add_document(doc!(text_field => "three"))
- .unwrap();
- index_writer.commit().unwrap();
- let index_reader = index.reader().unwrap();
- let searcher = index_reader.searcher();
- let segment_reader = searcher.segment_reader(0);
- let text_fast_field = segment_reader.fast_fields().str("text").unwrap().unwrap();
- let mut buffer = String::new();
- assert!(text_fast_field.ord_to_str(0, &mut buffer).unwrap());
- assert_eq!(buffer, "one");
- assert!(text_fast_field.ord_to_str(1, &mut buffer).unwrap());
- assert_eq!(buffer, "three");
- assert!(text_fast_field.ord_to_str(2, &mut buffer).unwrap());
- assert_eq!(buffer, "two");
- assert!(!text_fast_field.ord_to_str(3, &mut buffer).unwrap());
-
- assert_eq!(segment_reader.max_doc(), 3);
- index_writer.delete_term(Term::from_field_text(text_field, "three"));
- index_writer.commit().unwrap();
- index_writer
- .merge(&[segment_reader.segment_id()])
- .wait()
- .unwrap();
- index_reader.reload().unwrap();
- let searcher = index_reader.searcher();
- let segment_reader = searcher.segment_reader(0);
- assert_eq!(segment_reader.max_doc(), 2);
- let text_fast_field = segment_reader.fast_fields().str("text").unwrap().unwrap();
- let mut buffer = String::new();
- assert!(text_fast_field.ord_to_str(0, &mut buffer).unwrap());
- assert_eq!(buffer, "one");
- assert!(text_fast_field.ord_to_str(1, &mut buffer).unwrap());
- assert_eq!(buffer, "two");
- assert!(!text_fast_field.ord_to_str(2, &mut buffer).unwrap());
- assert!(text_fast_field.term_ords(0).eq([0].into_iter()));
- assert!(text_fast_field.term_ords(1).eq([1].into_iter()));
- }
-
#[test]
fn test_delete_with_sort_by_field() -> crate::Result<()> {
let mut schema_builder = schema::Schema::builder();
@@ -1475,7 +1421,7 @@ mod tests {
let in_order_alive_ids: Vec<u64> = segment_reader
.doc_ids_alive()
- .flat_map(|doc| fast_field_reader.values_for_doc(doc))
+ .flat_map(|doc| fast_field_reader.first(doc))
.collect();
assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);
Ok(())
@@ -1535,7 +1481,7 @@ mod tests {
let fast_field_reader = segment_reader.fast_fields().u64("id")?;
let in_order_alive_ids: Vec<u64> = segment_reader
.doc_ids_alive()
- .flat_map(|doc| fast_field_reader.values_for_doc(doc))
+ .flat_map(|doc| fast_field_reader.first(doc))
.collect();
assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 2, 0]);
Ok(())
@@ -1665,28 +1611,6 @@ mod tests {
(existing_ids, deleted_ids)
}
- fn get_id_list(ops: &[IndexingOp]) -> Vec<u64> {
- let mut id_list = Vec::new();
- for op in ops {
- match op {
- IndexingOp::AddDoc { id, value: _ } => {
- id_list.push(*id);
- }
- IndexingOp::AddMultipleDoc { id, .. } => {
- id_list.push(*id);
- }
- IndexingOp::DeleteDoc { id } => {
- id_list.retain(|el| el != id);
- }
- IndexingOp::DeleteDocQuery { id } => {
- id_list.retain(|el| el != id);
- }
- _ => {}
- }
- }
- id_list
- }
-
fn test_operation_strategy(
ops: &[IndexingOp],
sort_index: bool,
@@ -1694,12 +1618,8 @@ mod tests {
) -> crate::Result<Index> {
let mut schema_builder = schema::Schema::builder();
let json_field = schema_builder.add_json_field("json", FAST | TEXT | STORED);
- let ip_field = schema_builder.add_ip_addr_field("ip", FAST | INDEXED | STORED);
- let ips_field = schema_builder
- .add_ip_addr_field("ips", IpAddrOptions::default().set_fast().set_indexed());
let i64_field = schema_builder.add_i64_field("i64", INDEXED);
let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);
- let id_opt_field = schema_builder.add_u64_field("id_opt", FAST | INDEXED | STORED);
let f64_field = schema_builder.add_f64_field("f64", INDEXED);
let date_field = schema_builder.add_date_field("date", INDEXED);
let bytes_field = schema_builder.add_bytes_field("bytes", FAST | INDEXED | STORED);
@@ -1715,22 +1635,12 @@ mod tests {
);
let large_text_field = schema_builder.add_text_field("large_text_field", TEXT | STORED);
- let multi_text_fields = schema_builder.add_text_field("multi_text_fields", TEXT | STORED);
- let multi_numbers = schema_builder.add_u64_field(
- "multi_numbers",
- NumericOptions::default().set_fast().set_stored(),
- );
- let multi_bools = schema_builder.add_bool_field(
- "multi_bools",
- NumericOptions::default().set_fast().set_stored(),
- );
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
let schema = schema_builder.build();
let settings = if sort_index {
IndexSettings {
sort_by_field: Some(IndexSortByField {
- field: "id_opt".to_string(),
+ field: "id".to_string(),
order: Order::Asc,
}),
..Default::default()
@@ -1752,49 +1662,27 @@ mod tests {
// Every 3rd doc has only id field
let id_is_full_doc = |id| id % 3 != 0;
- let multi_text_field_text1 = "test1 test2 test3 test1 test2 test3";
- // rotate left
- let multi_text_field_text2 = "test2 test3 test1 test2 test3 test1";
- // rotate right
- let multi_text_field_text3 = "test3 test1 test2 test3 test1 test2";
-
- let ip_from_id = |id| Ipv6Addr::from_u128(id as u128);
-
let add_docs = |index_writer: &mut IndexWriter,
id: u64,
value: IndexValue,
num: u64|
-> crate::Result<()> {
- let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
- let ip = ip_from_id(id);
let doc = if !id_is_full_doc(id) {
// every 3rd doc has no ip field
doc!(
id_field=>id,
)
} else {
- let json = json!({"date1": format!("2022-{id}-01T00:00:01Z"), "date2": format!("{id}-05-01T00:00:01Z"), "id": id, "ip": ip.to_string(), "val": value});
+ let json = json!({"date1": format!("2022-{id}-01T00:00:01Z"), "date2": format!("{id}-05-01T00:00:01Z"), "id": id, "val": value});
doc!(id_field=>id,
json_field=>json,
bytes_field => id.to_le_bytes().as_slice(),
- id_opt_field => id,
- ip_field => ip,
- ips_field => ip,
- ips_field => ip,
- multi_numbers=> id,
- multi_numbers => id,
bool_field => (id % 2u64) != 0,
i64_field => id as i64,
f64_field => id as f64,
date_field => DateTime::from_timestamp_secs(id as i64),
- multi_bools => (id % 2u64) != 0,
- multi_bools => (id % 2u64) == 0,
text_field => id.to_string(),
- facet_field => facet,
large_text_field => LOREM,
- multi_text_fields => multi_text_field_text1,
- multi_text_fields => multi_text_field_text2,
- multi_text_fields => multi_text_field_text3,
)
};
for _ in 0..num {
@@ -1864,7 +1752,7 @@ mod tests {
let ff_reader = segment_reader.fast_fields().u64("id").unwrap();
segment_reader
.doc_ids_alive()
- .flat_map(move |doc| ff_reader.values_for_doc(doc).collect_vec().into_iter())
+ .flat_map(move |doc| ff_reader.first(doc).into_iter())
})
.collect();
@@ -1875,29 +1763,12 @@ mod tests {
let ff_reader = segment_reader.fast_fields().u64("id").unwrap();
segment_reader
.doc_ids_alive()
- .flat_map(move |doc| ff_reader.values_for_doc(doc).collect_vec().into_iter())
+ .flat_map(move |doc| ff_reader.first(doc).into_iter())
})
.collect();
let (expected_ids_and_num_occurrences, deleted_ids) = expected_ids(ops);
- let id_list = get_id_list(ops);
-
- // multivalue fast field content
- let mut all_ips = Vec::new();
- let mut num_ips = 0;
- for segment_reader in searcher.segment_readers().iter() {
- let ip_reader: Column<Ipv6Addr> = segment_reader
- .fast_fields()
- .column_opt("ips")
- .unwrap()
- .unwrap();
- for doc in segment_reader.doc_ids_alive() {
- all_ips.extend(ip_reader.values_for_doc(doc));
- }
- num_ips += ip_reader.values.num_vals();
- }
-
let num_docs_expected = expected_ids_and_num_occurrences
.values()
.map(|id_occurrences| *id_occurrences as usize)
@@ -1927,17 +1798,6 @@ mod tests {
);
if force_end_merge && num_segments_before_merge > 1 && num_segments_after_merge == 1 {
- let mut expected_multi_ips: Vec<_> = id_list
- .iter()
- .filter(|id| id_is_full_doc(**id))
- .flat_map(|id| vec![ip_from_id(*id), ip_from_id(*id)])
- .collect();
- assert_eq!(num_ips, expected_multi_ips.len() as u32);
-
- expected_multi_ips.sort();
- all_ips.sort();
- assert_eq!(expected_multi_ips, all_ips);
-
// Test fastfield num_docs
let num_docs: usize = searcher
.segment_readers()
@@ -1945,7 +1805,7 @@ mod tests {
.map(|segment_reader| {
let ff_reader = segment_reader
.fast_fields()
- .column_opt::<Ipv6Addr>("ips")
+ .column_opt::<i64>("i64")
.unwrap()
.unwrap();
ff_reader.num_docs() as usize
@@ -1954,85 +1814,6 @@ mod tests {
assert_eq!(num_docs, num_docs_expected);
}
- // Load all ips addr
- let mut ips: HashSet<Ipv6Addr> = Default::default();
- for reader in searcher.segment_readers() {
- if let Some(ff_reader) = reader.fast_fields().column_opt::<Ipv6Addr>("ips").unwrap() {
- for doc in reader.doc_ids_alive() {
- ips.extend(ff_reader.values_for_doc(doc));
- }
- }
- }
-
- let expected_ips = expected_ids_and_num_occurrences
- .keys()
- .flat_map(|id| {
- if !id_is_full_doc(*id) {
- None
- } else {
- Some(Ipv6Addr::from_u128(*id as u128))
- }
- })
- .collect::<HashSet<_>>();
- assert_eq!(ips, expected_ips);
-
- let expected_ips = expected_ids_and_num_occurrences
- .keys()
- .filter_map(|id| {
- if !id_is_full_doc(*id) {
- None
- } else {
- Some(Ipv6Addr::from_u128(*id as u128))
- }
- })
- .collect::<HashSet<_>>();
-
- let mut ips: HashSet<Ipv6Addr> = Default::default();
- for reader in searcher.segment_readers() {
- if let Some(ff_reader) = reader.fast_fields().column_opt::<Ipv6Addr>("ips").unwrap() {
- for doc in reader.doc_ids_alive() {
- ips.extend(ff_reader.values_for_doc(doc));
- }
- }
- }
- assert_eq!(ips, expected_ips);
-
- // multivalue fast field tests
- for segment_reader in searcher.segment_readers().iter() {
- let id_reader = segment_reader.fast_fields().u64("id").unwrap();
- let ff_reader = segment_reader
- .fast_fields()
- .column_opt("multi_numbers")
- .unwrap()
- .unwrap();
- let bool_ff_reader = segment_reader
- .fast_fields()
- .column_opt::<bool>("multi_bools")
- .unwrap()
- .unwrap();
- for doc in segment_reader.doc_ids_alive() {
- let id = id_reader.first(doc).unwrap();
-
- let vals: Vec<u64> = ff_reader.values_for_doc(doc).collect();
- if id_is_full_doc(id) {
- assert_eq!(vals.len(), 2);
- assert_eq!(vals[0], vals[1]);
- assert!(expected_ids_and_num_occurrences.contains_key(&vals[0]));
- assert_eq!(id_reader.first(doc), Some(vals[0]));
- } else {
- assert_eq!(vals.len(), 0);
- }
-
- let bool_vals: Vec<bool> = bool_ff_reader.values_for_doc(doc).collect();
- if id_is_full_doc(id) {
- assert_eq!(bool_vals.len(), 2);
- assert_ne!(bool_vals[0], bool_vals[1]);
- } else {
- assert_eq!(bool_vals.len(), 0);
- }
- }
- }
-
// doc store tests
for segment_reader in searcher.segment_readers().iter() {
let store_reader = segment_reader
@@ -2059,40 +1840,9 @@ mod tests {
.as_u64()
.unwrap();
assert!(expected_ids_and_num_occurrences.contains_key(&id));
- if id_is_full_doc(id) {
- let id2 = store_reader
- .get::<TantivyDocument>(doc_id)
- .unwrap()
- .get_first(multi_numbers)
- .unwrap()
- .as_u64()
- .unwrap();
- assert_eq!(id, id2);
- let bool = store_reader
- .get::<TantivyDocument>(doc_id)
- .unwrap()
- .get_first(bool_field)
- .unwrap()
- .as_bool()
- .unwrap();
- let doc = store_reader.get::<TantivyDocument>(doc_id).unwrap();
- let mut bool2 = doc.get_all(multi_bools);
- assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());
- assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());
- assert!(bool2.next().is_none())
- }
}
}
// test search
- let do_search = |term: &str, field| {
- let query = QueryParser::for_index(&index, vec![field])
- .parse_query(term)
- .unwrap();
- let top_docs: Vec<(f32, DocAddress)> =
- searcher.search(&query, &TopDocs::with_limit(1000)).unwrap();
-
- top_docs.iter().map(|el| el.1).collect::<Vec<_>>()
- };
let count_search = |term: &str, field| {
let query = QueryParser::for_index(&index, vec![field])
.parse_query(term)
@@ -2117,18 +1867,6 @@ mod tests {
assert_eq!(get_num_hits(i64_field), count);
assert_eq!(get_num_hits(f64_field), count);
- // Test multi text
- if num_docs_with_values < 1000 {
- assert_eq!(
- do_search("\"test1 test2\"", multi_text_fields).len(),
- num_docs_with_values
- );
- assert_eq!(
- do_search("\"test2 test3\"", multi_text_fields).len(),
- num_docs_with_values
- );
- }
-
// Test bytes
let term = Term::from_field_bytes(bytes_field, existing_id.to_le_bytes().as_slice());
assert_eq!(count_search2(term) as u64, count);
@@ -2158,156 +1896,17 @@ mod tests {
Term::from_field_date(date_field, DateTime::from_timestamp_secs(deleted_id as i64));
assert_eq!(count_search2(term), 0);
}
- // search ip address
- //
- for (existing_id, count) in &expected_ids_and_num_occurrences {
- let (existing_id, count) = (*existing_id, *count);
- if !id_is_full_doc(existing_id) {
- continue;
- }
- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;
- let ip_addr = Ipv6Addr::from_u128(existing_id as u128);
- // Test incoming ip as ipv6
- assert_eq!(do_search_ip_field(&format!("\"{ip_addr}\"")), count);
-
- let term = Term::from_field_ip_addr(ip_field, ip_addr);
- assert_eq!(count_search2(term) as u64, count);
-
- // Test incoming ip as ipv4
- if let Some(ip_addr) = ip_addr.to_ipv4_mapped() {
- assert_eq!(do_search_ip_field(&format!("\"{ip_addr}\"")), count);
- }
- }
-
- // Range query
- //
- // Take half as sample
- let mut sample: Vec<_> = expected_ids_and_num_occurrences.iter().collect();
- sample.sort_by_key(|(k, _num_occurences)| *k);
- // sample.truncate(sample.len() / 2);
- if !sample.is_empty() {
- let (left_sample, right_sample) = sample.split_at(sample.len() / 2);
-
- let calc_expected_count = |sample: &[(&u64, &u64)]| {
- sample
- .iter()
- .filter(|(id, _)| id_is_full_doc(**id))
- .map(|(_id, num_occurences)| **num_occurences)
- .sum::<u64>()
- };
- fn gen_query_inclusive<T1: ToString, T2: ToString>(
- field: &str,
- from: T1,
- to: T2,
- ) -> String {
- format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())
- }
-
- // Query first half
- let expected_count = calc_expected_count(left_sample);
- if !left_sample.is_empty() && expected_count < 1000 {
- let start_range = *left_sample[0].0;
- let end_range = *left_sample.last().unwrap().0;
- let query = gen_query_inclusive("id_opt", start_range, end_range);
- assert_eq!(count_search(&query, id_opt_field) as u64, expected_count);
-
- // Range query on ip field
- let ip1 = ip_from_id(start_range);
- let ip2 = ip_from_id(end_range);
- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;
- let query = gen_query_inclusive("ip", ip1, ip2);
- assert_eq!(do_search_ip_field(&query), expected_count);
- let query = gen_query_inclusive("ip", "*", ip2);
- assert_eq!(do_search_ip_field(&query), expected_count);
- // Range query on multi value field
- let query = gen_query_inclusive("ips", ip1, ip2);
- assert_eq!(do_search_ip_field(&query), expected_count);
- let query = gen_query_inclusive("ips", "*", ip2);
- assert_eq!(do_search_ip_field(&query), expected_count);
- }
- // Query second half
- let expected_count = calc_expected_count(right_sample);
- if !right_sample.is_empty() && expected_count < 1000 {
- let start_range = *right_sample[0].0;
- let end_range = *right_sample.last().unwrap().0;
- // Range query on id opt field
- let query =
- gen_query_inclusive("id_opt", start_range.to_string(), end_range.to_string());
- assert_eq!(count_search(&query, id_opt_field) as u64, expected_count);
-
- // Range query on ip field
- let ip1 = ip_from_id(start_range);
- let ip2 = ip_from_id(end_range);
- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;
- let query = gen_query_inclusive("ip", ip1, ip2);
- assert_eq!(do_search_ip_field(&query), expected_count);
- let query = gen_query_inclusive("ip", ip1, "*");
- assert_eq!(do_search_ip_field(&query), expected_count);
- // Range query on multi value field
- let query = gen_query_inclusive("ips", ip1, ip2);
- assert_eq!(do_search_ip_field(&query), expected_count);
- let query = gen_query_inclusive("ips", ip1, "*");
- assert_eq!(do_search_ip_field(&query), expected_count);
- }
- }
-
- // ip range query on fast field
- //
- for (existing_id, count) in expected_ids_and_num_occurrences.iter().take(10) {
- let (existing_id, count) = (*existing_id, *count);
- if !id_is_full_doc(existing_id) {
- continue;
- }
- let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {
- format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())
- };
- let ip = ip_from_id(existing_id);
-
- let do_search_ip_field = |term: &str| count_search(term, ip_field) as u64;
- // Range query on single value field
- let query = gen_query_inclusive("ip", ip, ip);
- assert_eq!(do_search_ip_field(&query), count);
-
- // Range query on multi value field
- let query = gen_query_inclusive("ips", ip, ip);
- assert_eq!(do_search_ip_field(&query), count);
- }
-
- // test facets
- for segment_reader in searcher.segment_readers().iter() {
- let facet_reader = segment_reader.facet_reader("facet").unwrap();
- let ff_reader = segment_reader
- .fast_fields()
- .u64("id")
- .unwrap()
- .first_or_default_col(9999);
- for doc_id in segment_reader.doc_ids_alive() {
- let id = ff_reader.get_val(doc_id);
- if !id_is_full_doc(id) {
- continue;
- }
- let facet_ords: Vec<u64> = facet_reader.facet_ords(doc_id).collect();
- assert_eq!(facet_ords.len(), 1);
- let mut facet = Facet::default();
- facet_reader
- .facet_from_ord(facet_ords[0], &mut facet)
- .unwrap();
- let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));
-
- assert_eq!(facet, facet_expected);
- }
- }
// Test if index property is in sort order
if sort_index {
- // load all id_opt in each segment and check they are in order
+ // load all id in each segment and check they are in order
for reader in searcher.segment_readers() {
- let (ff_reader, _) = reader.fast_fields().u64_lenient("id_opt").unwrap().unwrap();
+ let (ff_reader, _) = reader.fast_fields().u64_lenient("id").unwrap().unwrap();
let mut ids_in_segment: Vec<u64> = Vec::new();
for doc in 0..reader.num_docs() {
- ids_in_segment.extend(ff_reader.values_for_doc(doc));
+ ids_in_segment.extend(ff_reader.first(doc));
}
assert!(is_sorted(&ids_in_segment));
@@ -2374,40 +1973,6 @@ mod tests {
.is_ok());
}
- #[test]
- fn test_ip_range_query_multivalue_bug() {
- assert!(test_operation_strategy(
- &[
- IndexingOp::add(2),
- IndexingOp::Commit,
- IndexingOp::add(1),
- IndexingOp::add(1),
- IndexingOp::Commit,
- IndexingOp::Merge
- ],
- true,
- false
- )
- .is_ok());
- }
-
- #[test]
- fn test_ff_num_ips_regression() {
- assert!(test_operation_strategy(
- &[
- IndexingOp::add(13),
- IndexingOp::add(1),
- IndexingOp::Commit,
- IndexingOp::DeleteDocQuery { id: 13 },
- IndexingOp::add(1),
- IndexingOp::Commit,
- ],
- false,
- true
- )
- .is_ok());
- }
-
#[test]
fn test_minimal_sort_force_end_merge() {
assert!(test_operation_strategy(
@@ -2455,17 +2020,8 @@ mod tests {
.column_opt("id")
.unwrap()
.unwrap();
- let val_col: Column = segment_reader
- .fast_fields()
- .column_opt("val")
- .unwrap()
- .unwrap();
- assert_eq!(id_col.get_cardinality(), Cardinality::Full);
- assert_eq!(val_col.get_cardinality(), Cardinality::Multivalued);
assert_eq!(id_col.first(0u32), Some(1u64));
assert_eq!(id_col.first(1u32), Some(2u64));
- assert!(val_col.values_for_doc(0u32).eq([1u64, 1u64].into_iter()));
- assert!(val_col.values_for_doc(1u32).eq([2u64, 2u64].into_iter()));
}
#[test]
@@ -2501,52 +2057,48 @@ mod tests {
assert!(test_operation_strategy(&[IndexingOp::add(3),], true, true).is_ok());
}
- use proptest::prelude::*;
-
- proptest! {
+ // proptest! {
- #![proptest_config(ProptestConfig::with_cases(20))]
- #[test]
- fn test_delete_with_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
- assert!(test_operation_strategy(&ops[..], true, false).is_ok());
- }
+ // #![proptest_config(ProptestConfig::with_cases(20))]
+ // #[test]
+ // fn test_delete_with_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
+ // assert!(test_operation_strategy(&ops[..], true, false).is_ok());
+ // }
- #[test]
- fn test_delete_without_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
- assert!(test_operation_strategy(&ops[..], false, false).is_ok());
- }
+ // #[test]
+ // fn test_delete_without_sort_proptest_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
+ // assert!(test_operation_strategy(&ops[..], false, false).is_ok());
+ // }
- #[test]
- fn test_delete_with_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
- assert!(test_operation_strategy(&ops[..], true, true).is_ok());
- }
+ // #[test]
+ // fn test_delete_with_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
+ // assert!(test_operation_strategy(&ops[..], true, true).is_ok());
+ // }
- #[test]
- fn test_delete_without_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
- assert!(test_operation_strategy(&ops[..], false, true).is_ok());}
+ // #[test]
+ // fn test_delete_without_sort_proptest_with_merge_adding(ops in proptest::collection::vec(adding_operation_strategy(), 1..100)) {
+ // assert!(test_operation_strategy(&ops[..], false, true).is_ok());}
- #[test]
- fn test_delete_with_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {
- assert!(test_operation_strategy(&ops[..], true, false).is_ok());
- }
-
- #[test]
- fn test_delete_without_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {
- assert!(test_operation_strategy(&ops[..], false, false).is_ok());
- }
-
- #[test]
- fn test_delete_with_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {
- assert!(test_operation_strategy(&ops[..], true, true).is_ok());
- }
+ // #[test]
+ // fn test_delete_with_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {
+ // assert!(test_operation_strategy(&ops[..], true, false).is_ok());
+ // }
+ // #[test]
+ // fn test_delete_without_sort_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {
+ // assert!(test_operation_strategy(&ops[..], false, false).is_ok());
+ // }
+ // #[test]
+ // fn test_delete_with_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..10)) {
+ // assert!(test_operation_strategy(&ops[..], true, true).is_ok());
+ // }
- #[test]
- fn test_delete_without_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..100)) {
- assert!(test_operation_strategy(&ops[..], false, true).is_ok());
- }
- }
+ // #[test]
+ // fn test_delete_without_sort_proptest_with_merge(ops in proptest::collection::vec(balanced_operation_strategy(), 1..100)) {
+ // assert!(test_operation_strategy(&ops[..], false, true).is_ok());
+ // }
+ // }
#[test]
fn test_delete_with_sort_by_field_last_opstamp_is_not_max() -> crate::Result<()> {
@@ -2590,22 +2142,22 @@ mod tests {
Ok(())
}
- #[test]
- fn test_delete_bug_reproduction_ip_addr() {
- use IndexingOp::*;
- let ops = &[
- IndexingOp::add(1),
- IndexingOp::add(2),
- Commit,
- IndexingOp::add(3),
- DeleteDoc { id: 1 },
- Commit,
- Merge,
- IndexingOp::add(4),
- Commit,
- ];
- test_operation_strategy(&ops[..], false, true).unwrap();
- }
+ // #[test]
+ // fn test_delete_bug_reproduction_ip_addr() {
+ // use IndexingOp::*;
+ // let ops = &[
+ // IndexingOp::add(1),
+ // IndexingOp::add(2),
+ // Commit,
+ // IndexingOp::add(3),
+ // DeleteDoc { id: 1 },
+ // Commit,
+ // Merge,
+ // IndexingOp::add(4),
+ // Commit,
+ // ];
+ // test_operation_strategy(&ops[..], false, true).unwrap();
+ // }
#[test]
fn test_merge_regression_1() {
diff --git a/crates/tantivy/src/indexer/merger.rs b/crates/tantivy/src/indexer/merger.rs
index 0a325989..a1f54779 100644
--- a/crates/tantivy/src/indexer/merger.rs
+++ b/crates/tantivy/src/indexer/merger.rs
@@ -310,7 +310,7 @@ impl IndexMerger {
.ok_or_else(|| FastFieldNotAvailableError {
field_name: sort_by_field.field.to_string(),
})?;
- Ok(value_accessor.first_or_default_col(0u64))
+ Ok(value_accessor.values)
}
/// Collecting value_accessors into a vec to bind the lifetime.
pub(crate) fn get_reader_with_sort_field_accessor(
@@ -794,18 +794,16 @@ mod tests {
use crate::collector::tests::{
BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,
};
- use crate::collector::{Count, FacetCollector};
use crate::index::{Index, SegmentId};
use crate::indexer::NoMergePolicy;
- use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
+ use crate::query::{BooleanQuery, EnableScoring, Scorer, TermQuery};
use crate::schema::{
- Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,
- TextFieldIndexing, Value, INDEXED, TEXT,
+ IndexRecordOption, NumericOptions, TantivyDocument, Term, TextFieldIndexing, Value,
+ INDEXED, TEXT,
};
use crate::time::OffsetDateTime;
use crate::{
- assert_nearly_equals, schema, DateTime, DocAddress, DocId, DocSet, IndexSettings,
- IndexSortByField, IndexWriter, Order, Searcher,
+ assert_nearly_equals, schema, DateTime, DocAddress, DocId, DocSet, IndexWriter, Searcher,
};
#[test]
@@ -965,599 +963,359 @@ mod tests {
Ok(())
}
- #[test]
- fn test_index_merger_with_deletes() -> crate::Result<()> {
- let mut schema_builder = schema::Schema::builder();
- let text_fieldtype = schema::TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
- )
- .set_stored();
- let text_field = schema_builder.add_text_field("text", text_fieldtype);
- let score_fieldtype = schema::NumericOptions::default().set_fast();
- let score_field = schema_builder.add_u64_field("score", score_fieldtype);
- let bytes_score_field = schema_builder.add_bytes_field("score_bytes", FAST);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer = index.writer_for_tests()?;
- let reader = index.reader().unwrap();
- let search_term = |searcher: &Searcher, term: Term| {
- let collector = FastFieldTestCollector::for_field("score");
- // let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);
- let term_query = TermQuery::new(term, IndexRecordOption::Basic);
- // searcher
- // .search(&term_query, &(collector, bytes_collector))
- // .map(|(scores, bytes)| {
- // let mut score_bytes = &bytes[..];
- // for &score in &scores {
- // assert_eq!(score as u32, score_bytes.read_u32::<BigEndian>().unwrap());
- // }
- // scores
- // })
- searcher.search(&term_query, &collector)
- };
-
- let empty_vec = Vec::<u64>::new();
- {
- // a first commit
- index_writer.add_document(doc!(
- text_field => "a b d",
- score_field => 1u64,
- bytes_score_field => vec![0u8, 0, 0, 1],
- ))?;
- index_writer.add_document(doc!(
- text_field => "b c",
- score_field => 2u64,
- bytes_score_field => vec![0u8, 0, 0, 2],
- ))?;
- index_writer.delete_term(Term::from_field_text(text_field, "c"));
- index_writer.add_document(doc!(
- text_field => "c d",
- score_field => 3u64,
- bytes_score_field => vec![0u8, 0, 0, 3],
- ))?;
- index_writer.commit()?;
- reader.reload()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "a"))?,
- vec![1]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "b"))?,
- vec![1]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "c"))?,
- vec![3]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "d"))?,
- vec![1, 3]
- );
- }
- {
- // a second commit
- index_writer.add_document(doc!(
- text_field => "a d e",
- score_field => 4_000u64,
- bytes_score_field => vec![0u8, 0, 0, 4],
- ))?;
- index_writer.add_document(doc!(
- text_field => "e f",
- score_field => 5_000u64,
- bytes_score_field => vec![0u8, 0, 0, 5],
- ))?;
- index_writer.delete_term(Term::from_field_text(text_field, "a"));
- index_writer.delete_term(Term::from_field_text(text_field, "f"));
- index_writer.add_document(doc!(
- text_field => "f g",
- score_field => 6_000u64,
- bytes_score_field => vec![0u8, 0, 23, 112],
- ))?;
- index_writer.add_document(doc!(
- text_field => "g h",
- score_field => 7_000u64,
- bytes_score_field => vec![0u8, 0, 27, 88],
- ))?;
- index_writer.commit()?;
- reader.reload()?;
- let searcher = reader.searcher();
-
- assert_eq!(searcher.segment_readers().len(), 2);
- assert_eq!(searcher.num_docs(), 3);
- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].max_doc(), 4);
- assert_eq!(searcher.segment_readers()[1].num_docs(), 1);
- assert_eq!(searcher.segment_readers()[1].max_doc(), 3);
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "a"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "b"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "c"))?,
- vec![3]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "d"))?,
- vec![3]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "e"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "f"))?,
- vec![6_000]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "g"))?,
- vec![6_000, 7_000]
- );
-
- let score_field_reader = searcher
- .segment_reader(0)
- .fast_fields()
- .u64("score")
- .unwrap();
- assert_eq!(score_field_reader.min_value(), 4000);
- assert_eq!(score_field_reader.max_value(), 7000);
-
- let score_field_reader = searcher
- .segment_reader(1)
- .fast_fields()
- .u64("score")
- .unwrap();
- assert_eq!(score_field_reader.min_value(), 1);
- assert_eq!(score_field_reader.max_value(), 3);
- }
- {
- // merging the segments
- let segment_ids = index.searchable_segment_ids()?;
- index_writer.merge(&segment_ids).wait()?;
- reader.reload()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 1);
- assert_eq!(searcher.num_docs(), 3);
- assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
- assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "a"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "b"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "c"))?,
- vec![3]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "d"))?,
- vec![3]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "e"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "f"))?,
- vec![6_000]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "g"))?,
- vec![6_000, 7_000]
- );
- let score_field_reader = searcher
- .segment_reader(0)
- .fast_fields()
- .u64("score")
- .unwrap();
- assert_eq!(score_field_reader.min_value(), 3);
- assert_eq!(score_field_reader.max_value(), 7000);
- }
- {
- // test a commit with only deletes
- index_writer.delete_term(Term::from_field_text(text_field, "c"));
- index_writer.commit()?;
-
- reader.reload()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 1);
- assert_eq!(searcher.num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "a"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "b"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "c"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "d"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "e"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "f"))?,
- vec![6_000]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "g"))?,
- vec![6_000, 7_000]
- );
- let score_field_reader = searcher
- .segment_reader(0)
- .fast_fields()
- .u64("score")
- .unwrap();
- assert_eq!(score_field_reader.min_value(), 3);
- assert_eq!(score_field_reader.max_value(), 7000);
- }
- {
- // Test merging a single segment in order to remove deletes.
- let segment_ids = index.searchable_segment_ids()?;
- index_writer.merge(&segment_ids).wait()?;
- reader.reload()?;
-
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 1);
- assert_eq!(searcher.num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
- assert_eq!(searcher.segment_readers()[0].max_doc(), 2);
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "a"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "b"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "c"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "d"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "e"))?,
- empty_vec
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "f"))?,
- vec![6_000]
- );
- assert_eq!(
- search_term(&searcher, Term::from_field_text(text_field, "g"))?,
- vec![6_000, 7_000]
- );
- let score_field_reader = searcher
- .segment_reader(0)
- .fast_fields()
- .u64("score")
- .unwrap();
- assert_eq!(score_field_reader.min_value(), 6000);
- assert_eq!(score_field_reader.max_value(), 7000);
- }
-
- {
- // Test removing all docs
- index_writer.delete_term(Term::from_field_text(text_field, "g"));
- index_writer.commit()?;
- let segment_ids = index.searchable_segment_ids()?;
- reader.reload()?;
-
- let searcher = reader.searcher();
- assert!(segment_ids.is_empty());
- assert!(searcher.segment_readers().is_empty());
- assert_eq!(searcher.num_docs(), 0);
- }
- Ok(())
- }
-
- #[test]
- fn test_merge_facets_sort_none() {
- test_merge_facets(None, true)
- }
-
- #[test]
- fn test_merge_facets_sort_asc() {
- // In the merge case this will go through the doc_id mapping code
- test_merge_facets(
- Some(IndexSettings {
- sort_by_field: Some(IndexSortByField {
- field: "intval".to_string(),
- order: Order::Desc,
- }),
- ..Default::default()
- }),
- true,
- );
- // In the merge case this will not go through the doc_id mapping code, because the data
- // sorted and disjunct
- test_merge_facets(
- Some(IndexSettings {
- sort_by_field: Some(IndexSortByField {
- field: "intval".to_string(),
- order: Order::Desc,
- }),
- ..Default::default()
- }),
- false,
- );
- }
-
- #[test]
- fn test_merge_facets_sort_desc() {
- // In the merge case this will go through the doc_id mapping code
- test_merge_facets(
- Some(IndexSettings {
- sort_by_field: Some(IndexSortByField {
- field: "intval".to_string(),
- order: Order::Desc,
- }),
- ..Default::default()
- }),
- true,
- );
- // In the merge case this will not go through the doc_id mapping code, because the data
- // sorted and disjunct
- test_merge_facets(
- Some(IndexSettings {
- sort_by_field: Some(IndexSortByField {
- field: "intval".to_string(),
- order: Order::Desc,
- }),
- ..Default::default()
- }),
- false,
- );
- }
-
- // force_segment_value_overlap forces the int value for sorting to have overlapping min and max
- // ranges between segments so that merge algorithm can't apply certain optimizations
- fn test_merge_facets(index_settings: Option<IndexSettings>, force_segment_value_overlap: bool) {
- let mut schema_builder = schema::Schema::builder();
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
- let int_options = NumericOptions::default().set_fast().set_indexed();
- let int_field = schema_builder.add_u64_field("intval", int_options);
- let mut index_builder = Index::builder().schema(schema_builder.build());
- if let Some(settings) = index_settings {
- index_builder = index_builder.settings(settings);
- }
- let index = index_builder.create_in_ram().unwrap();
- // let index = Index::create_in_ram(schema_builder.build());
- let reader = index.reader().unwrap();
- let mut int_val = 0;
- {
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- let index_doc =
- |index_writer: &mut IndexWriter, doc_facets: &[&str], int_val: &mut u64| {
- let mut doc = TantivyDocument::default();
- for facet in doc_facets {
- doc.add_facet(facet_field, Facet::from(facet));
- }
- doc.add_u64(int_field, *int_val);
- *int_val += 1;
- index_writer.add_document(doc).unwrap();
- };
-
- index_doc(
- &mut index_writer,
- &["/top/a/firstdoc", "/top/b"],
- &mut int_val,
- );
- index_doc(
- &mut index_writer,
- &["/top/a/firstdoc", "/top/b", "/top/c"],
- &mut int_val,
- );
- index_doc(&mut index_writer, &["/top/a", "/top/b"], &mut int_val);
- index_doc(&mut index_writer, &["/top/a"], &mut int_val);
-
- index_doc(&mut index_writer, &["/top/b", "/top/d"], &mut int_val);
- if force_segment_value_overlap {
- index_doc(&mut index_writer, &["/top/d"], &mut 0);
- index_doc(&mut index_writer, &["/top/e"], &mut 10);
- index_writer.commit().expect("committed");
- index_doc(&mut index_writer, &["/top/a"], &mut 5); // 5 is between 0 - 10 so the
- // segments don' have disjunct
- // ranges
- } else {
- index_doc(&mut index_writer, &["/top/d"], &mut int_val);
- index_doc(&mut index_writer, &["/top/e"], &mut int_val);
- index_writer.commit().expect("committed");
- index_doc(&mut index_writer, &["/top/a"], &mut int_val);
- }
- index_doc(&mut index_writer, &["/top/b"], &mut int_val);
- index_doc(&mut index_writer, &["/top/c"], &mut int_val);
- index_writer.commit().expect("committed");
-
- index_doc(&mut index_writer, &["/top/e", "/top/f"], &mut int_val);
- index_writer.commit().expect("committed");
- }
-
- reader.reload().unwrap();
- let test_searcher = |expected_num_docs: usize, expected: &[(&str, u64)]| {
- let searcher = reader.searcher();
- let mut facet_collector = FacetCollector::for_field("facet");
- facet_collector.add_facet(Facet::from("/top"));
- let (count, facet_counts) = searcher
- .search(&AllQuery, &(Count, facet_collector))
- .unwrap();
- assert_eq!(count, expected_num_docs);
- let facets: Vec<(String, u64)> = facet_counts
- .get("/top")
- .map(|(facet, count)| (facet.to_string(), count))
- .collect();
- assert_eq!(
- facets,
- expected
- .iter()
- .map(|&(facet_str, count)| (String::from(facet_str), count))
- .collect::<Vec<_>>()
- );
- };
- test_searcher(
- 11,
- &[
- ("/top/a", 5),
- ("/top/b", 5),
- ("/top/c", 2),
- ("/top/d", 2),
- ("/top/e", 2),
- ("/top/f", 1),
- ],
- );
- // Merging the segments
- {
- let segment_ids = index
- .searchable_segment_ids()
- .expect("Searchable segments failed.");
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .merge(&segment_ids)
- .wait()
- .expect("Merging failed");
- index_writer.wait_merging_threads().unwrap();
- reader.reload().unwrap();
- test_searcher(
- 11,
- &[
- ("/top/a", 5),
- ("/top/b", 5),
- ("/top/c", 2),
- ("/top/d", 2),
- ("/top/e", 2),
- ("/top/f", 1),
- ],
- );
- }
-
- // Deleting one term
- {
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- let facet = Facet::from_path(vec!["top", "a", "firstdoc"]);
- let facet_term = Term::from_facet(facet_field, &facet);
- index_writer.delete_term(facet_term);
- index_writer.commit().unwrap();
- reader.reload().unwrap();
- test_searcher(
- 9,
- &[
- ("/top/a", 3),
- ("/top/b", 3),
- ("/top/c", 1),
- ("/top/d", 2),
- ("/top/e", 2),
- ("/top/f", 1),
- ],
- );
- }
- }
-
- #[test]
- fn test_bug_merge() -> crate::Result<()> {
- let mut schema_builder = schema::Schema::builder();
- let int_field = schema_builder.add_u64_field("intvals", INDEXED);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer.add_document(doc!(int_field => 1u64))?;
- index_writer.commit().expect("commit failed");
- index_writer.add_document(doc!(int_field => 1u64))?;
- index_writer.commit().expect("commit failed");
- let reader = index.reader()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 2);
- index_writer.delete_term(Term::from_field_u64(int_field, 1));
- let segment_ids = index
- .searchable_segment_ids()
- .expect("Searchable segments failed.");
- index_writer.merge(&segment_ids).wait()?;
- reader.reload()?;
- // commit has not been called yet. The document should still be
- // there.
- assert_eq!(reader.searcher().num_docs(), 2);
- Ok(())
- }
-
- #[test]
- fn test_merge_multivalued_int_fields_all_deleted() -> crate::Result<()> {
- let mut schema_builder = schema::Schema::builder();
- let int_options = NumericOptions::default().set_fast().set_indexed();
- let int_field = schema_builder.add_u64_field("intvals", int_options);
- let index = Index::create_in_ram(schema_builder.build());
- let reader = index.reader()?;
- {
- let mut index_writer = index.writer_for_tests()?;
- let mut doc = TantivyDocument::default();
- doc.add_u64(int_field, 1);
- index_writer.add_document(doc.clone())?;
- index_writer.commit()?;
- index_writer.add_document(doc)?;
- index_writer.commit()?;
- index_writer.delete_term(Term::from_field_u64(int_field, 1));
- let segment_ids = index.searchable_segment_ids()?;
- index_writer.merge(&segment_ids).wait()?;
-
- // assert delete has not been committed
- reader.reload()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 2);
-
- index_writer.commit()?;
-
- index_writer.wait_merging_threads()?;
- }
-
- reader.reload()?;
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 0);
- Ok(())
- }
+ // #[test]
+ // fn test_index_merger_with_deletes() -> crate::Result<()> {
+ // let mut schema_builder = schema::Schema::builder();
+ // let text_fieldtype = schema::TextOptions::default()
+ // .set_indexing_options(
+ // TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
+ // )
+ // .set_stored();
+ // let text_field = schema_builder.add_text_field("text", text_fieldtype);
+ // let score_fieldtype = schema::NumericOptions::default().set_fast();
+ // let score_field = schema_builder.add_u64_field("score", score_fieldtype);
+ // let bytes_score_field = schema_builder.add_bytes_field("score_bytes", FAST);
+ // let index = Index::create_in_ram(schema_builder.build());
+ // let mut index_writer = index.writer_for_tests()?;
+ // let reader = index.reader().unwrap();
+ // let search_term = |searcher: &Searcher, term: Term| {
+ // let collector = FastFieldTestCollector::for_field("score");
+ // // let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);
+ // let term_query = TermQuery::new(term, IndexRecordOption::Basic);
+ // // searcher
+ // // .search(&term_query, &(collector, bytes_collector))
+ // // .map(|(scores, bytes)| {
+ // // let mut score_bytes = &bytes[..];
+ // // for &score in &scores {
+ // // assert_eq!(score as u32, score_bytes.read_u32::<BigEndian>().unwrap());
+ // // }
+ // // scores
+ // // })
+ // searcher.search(&term_query, &collector)
+ // };
+
+ // let empty_vec = Vec::<u64>::new();
+ // {
+ // // a first commit
+ // index_writer.add_document(doc!(
+ // text_field => "a b d",
+ // score_field => 1u64,
+ // bytes_score_field => vec![0u8, 0, 0, 1],
+ // ))?;
+ // index_writer.add_document(doc!(
+ // text_field => "b c",
+ // score_field => 2u64,
+ // bytes_score_field => vec![0u8, 0, 0, 2],
+ // ))?;
+ // index_writer.delete_term(Term::from_field_text(text_field, "c"));
+ // index_writer.add_document(doc!(
+ // text_field => "c d",
+ // score_field => 3u64,
+ // bytes_score_field => vec![0u8, 0, 0, 3],
+ // ))?;
+ // index_writer.commit()?;
+ // reader.reload()?;
+ // let searcher = reader.searcher();
+ // assert_eq!(searcher.num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,
+ // vec![1]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,
+ // vec![1]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,
+ // vec![3]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,
+ // vec![1, 3]
+ // );
+ // }
+ // {
+ // // a second commit
+ // index_writer.add_document(doc!(
+ // text_field => "a d e",
+ // score_field => 4_000u64,
+ // bytes_score_field => vec![0u8, 0, 0, 4],
+ // ))?;
+ // index_writer.add_document(doc!(
+ // text_field => "e f",
+ // score_field => 5_000u64,
+ // bytes_score_field => vec![0u8, 0, 0, 5],
+ // ))?;
+ // index_writer.delete_term(Term::from_field_text(text_field, "a"));
+ // index_writer.delete_term(Term::from_field_text(text_field, "f"));
+ // index_writer.add_document(doc!(
+ // text_field => "f g",
+ // score_field => 6_000u64,
+ // bytes_score_field => vec![0u8, 0, 23, 112],
+ // ))?;
+ // index_writer.add_document(doc!(
+ // text_field => "g h",
+ // score_field => 7_000u64,
+ // bytes_score_field => vec![0u8, 0, 27, 88],
+ // ))?;
+ // index_writer.commit()?;
+ // reader.reload()?;
+ // let searcher = reader.searcher();
+
+ // assert_eq!(searcher.segment_readers().len(), 2);
+ // assert_eq!(searcher.num_docs(), 3);
+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 4);
+ // assert_eq!(searcher.segment_readers()[1].num_docs(), 1);
+ // assert_eq!(searcher.segment_readers()[1].max_doc(), 3);
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,
+ // vec![3]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,
+ // vec![3]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,
+ // vec![6_000]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,
+ // vec![6_000, 7_000]
+ // );
+
+ // let score_field_reader = searcher
+ // .segment_reader(0)
+ // .fast_fields()
+ // .u64("score")
+ // .unwrap();
+ // assert_eq!(score_field_reader.min_value(), 4000);
+ // assert_eq!(score_field_reader.max_value(), 7000);
+
+ // let score_field_reader = searcher
+ // .segment_reader(1)
+ // .fast_fields()
+ // .u64("score")
+ // .unwrap();
+ // assert_eq!(score_field_reader.min_value(), 1);
+ // assert_eq!(score_field_reader.max_value(), 3);
+ // }
+ // {
+ // // merging the segments
+ // let segment_ids = index.searchable_segment_ids()?;
+ // index_writer.merge(&segment_ids).wait()?;
+ // reader.reload()?;
+ // let searcher = reader.searcher();
+ // assert_eq!(searcher.segment_readers().len(), 1);
+ // assert_eq!(searcher.num_docs(), 3);
+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,
+ // vec![3]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,
+ // vec![3]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,
+ // vec![6_000]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,
+ // vec![6_000, 7_000]
+ // );
+ // let score_field_reader = searcher
+ // .segment_reader(0)
+ // .fast_fields()
+ // .u64("score")
+ // .unwrap();
+ // assert_eq!(score_field_reader.min_value(), 3);
+ // assert_eq!(score_field_reader.max_value(), 7000);
+ // }
+ // {
+ // // test a commit with only deletes
+ // index_writer.delete_term(Term::from_field_text(text_field, "c"));
+ // index_writer.commit()?;
+
+ // reader.reload()?;
+ // let searcher = reader.searcher();
+ // assert_eq!(searcher.segment_readers().len(), 1);
+ // assert_eq!(searcher.num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,
+ // vec![6_000]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,
+ // vec![6_000, 7_000]
+ // );
+ // let score_field_reader = searcher
+ // .segment_reader(0)
+ // .fast_fields()
+ // .u64("score")
+ // .unwrap();
+ // assert_eq!(score_field_reader.min_value(), 3);
+ // assert_eq!(score_field_reader.max_value(), 7000);
+ // }
+ // {
+ // // Test merging a single segment in order to remove deletes.
+ // let segment_ids = index.searchable_segment_ids()?;
+ // index_writer.merge(&segment_ids).wait()?;
+ // reader.reload()?;
+
+ // let searcher = reader.searcher();
+ // assert_eq!(searcher.segment_readers().len(), 1);
+ // assert_eq!(searcher.num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+ // assert_eq!(searcher.segment_readers()[0].max_doc(), 2);
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "a"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "b"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "c"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "d"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "e"))?,
+ // empty_vec
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "f"))?,
+ // vec![6_000]
+ // );
+ // assert_eq!(
+ // search_term(&searcher, Term::from_field_text(text_field, "g"))?,
+ // vec![6_000, 7_000]
+ // );
+ // let score_field_reader = searcher
+ // .segment_reader(0)
+ // .fast_fields()
+ // .u64("score")
+ // .unwrap();
+ // assert_eq!(score_field_reader.min_value(), 6000);
+ // assert_eq!(score_field_reader.max_value(), 7000);
+ // }
+
+ // {
+ // // Test removing all docs
+ // index_writer.delete_term(Term::from_field_text(text_field, "g"));
+ // index_writer.commit()?;
+ // let segment_ids = index.searchable_segment_ids()?;
+ // reader.reload()?;
+
+ // let searcher = reader.searcher();
+ // assert!(segment_ids.is_empty());
+ // assert!(searcher.segment_readers().is_empty());
+ // assert_eq!(searcher.num_docs(), 0);
+ // }
+ // Ok(())
+ // }
+
+ // #[test]
+ // fn test_bug_merge() -> crate::Result<()> {
+ // let mut schema_builder = schema::Schema::builder();
+ // let int_field = schema_builder.add_u64_field("intvals", INDEXED);
+ // let index = Index::create_in_ram(schema_builder.build());
+ // let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+ // index_writer.add_document(doc!(int_field => 1u64))?;
+ // index_writer.commit().expect("commit failed");
+ // index_writer.add_document(doc!(int_field => 1u64))?;
+ // index_writer.commit().expect("commit failed");
+ // let reader = index.reader()?;
+ // let searcher = reader.searcher();
+ // assert_eq!(searcher.num_docs(), 2);
+ // index_writer.delete_term(Term::from_field_u64(int_field, 1));
+ // let segment_ids = index
+ // .searchable_segment_ids()
+ // .expect("Searchable segments failed.");
+ // index_writer.merge(&segment_ids).wait()?;
+ // reader.reload()?;
+ // // commit has not been called yet. The document should still be
+ // // there.
+ // assert_eq!(reader.searcher().num_docs(), 2);
+ // Ok(())
+ // }
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
enum IndexingOp {
- ZeroVal,
OneVal { val: u64 },
- TwoVal { val: u64 },
Commit,
}
fn balanced_operation_strategy() -> impl Strategy<Value = IndexingOp> {
prop_oneof![
- (0u64..1u64).prop_map(|_| IndexingOp::ZeroVal),
(0u64..1u64).prop_map(|val| IndexingOp::OneVal { val }),
- (0u64..1u64).prop_map(|val| IndexingOp::TwoVal { val }),
(0u64..1u64).prop_map(|_| IndexingOp::Commit),
]
}
use proptest::prelude::*;
- proptest! {
- #[test]
- fn test_merge_columnar_int_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..20)) {
- assert!(test_merge_int_fields(&ops[..]).is_ok());
- }
- }
+ // proptest! {
+ // #[test]
+ // fn test_merge_columnar_int_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..20)) {
+ // assert!(test_merge_int_fields(&ops[..]).is_ok());
+ // }
+ // }
fn test_merge_int_fields(ops: &[IndexingOp]) -> crate::Result<()> {
if ops.iter().all(|op| *op == IndexingOp::Commit) {
return Ok(());
@@ -1566,9 +1324,7 @@ mod tests {
.iter()
.filter(|op| *op != &IndexingOp::Commit)
.map(|op| match op {
- IndexingOp::ZeroVal => vec![],
IndexingOp::OneVal { val } => vec![*val],
- IndexingOp::TwoVal { val } => vec![*val, *val],
IndexingOp::Commit => unreachable!(),
})
.enumerate()
@@ -1592,9 +1348,7 @@ mod tests {
for op in ops {
match op {
- IndexingOp::ZeroVal => index_doc(&mut index_writer, &[]),
IndexingOp::OneVal { val } => index_doc(&mut index_writer, &[*val]),
- IndexingOp::TwoVal { val } => index_doc(&mut index_writer, &[*val, *val]),
IndexingOp::Commit => {
index_writer.commit().expect("commit failed");
}
@@ -1615,7 +1369,7 @@ mod tests {
let mut vals: Vec<u64> = Vec::new();
let mut test_vals = move |col: &Column<u64>, doc: DocId, expected: &[u64]| {
vals.clear();
- vals.extend(col.values_for_doc(doc));
+ vals.extend(col.first(doc));
assert_eq!(&vals[..], expected);
};
@@ -1640,122 +1394,11 @@ mod tests {
Ok(())
}
- #[test]
- fn test_merge_multivalued_int_fields_simple() -> crate::Result<()> {
- let mut schema_builder = schema::Schema::builder();
- let int_options = NumericOptions::default().set_fast().set_indexed();
- let int_field = schema_builder.add_u64_field("intvals", int_options);
- let index = Index::create_in_ram(schema_builder.build());
-
- let mut vals: Vec<u64> = Vec::new();
- let mut test_vals = move |col: &Column<u64>, doc: DocId, expected: &[u64]| {
- vals.clear();
- vals.extend(col.values_for_doc(doc));
- assert_eq!(&vals[..], expected);
- };
-
- {
- let mut index_writer = index.writer_for_tests()?;
- let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {
- let mut doc = TantivyDocument::default();
- for &val in int_vals {
- doc.add_u64(int_field, val);
- }
- index_writer.add_document(doc).unwrap();
- };
- index_doc(&mut index_writer, &[1, 2]);
- index_doc(&mut index_writer, &[1, 2, 3]);
- index_doc(&mut index_writer, &[4, 5]);
- index_doc(&mut index_writer, &[1, 2]);
- index_doc(&mut index_writer, &[1, 5]);
- index_doc(&mut index_writer, &[3]);
- index_doc(&mut index_writer, &[17]);
- assert!(index_writer.commit().is_ok());
- index_doc(&mut index_writer, &[20]);
- assert!(index_writer.commit().is_ok());
- index_doc(&mut index_writer, &[28, 27]);
- index_doc(&mut index_writer, &[1_000]);
- assert!(index_writer.commit().is_ok());
- }
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- {
- let segment = searcher.segment_reader(0u32);
- let column = segment
- .fast_fields()
- .column_opt::<u64>("intvals")
- .unwrap()
- .unwrap();
- test_vals(&column, 0, &[1, 2]);
- test_vals(&column, 1, &[1, 2, 3]);
- test_vals(&column, 2, &[4, 5]);
- test_vals(&column, 3, &[1, 2]);
- test_vals(&column, 4, &[1, 5]);
- test_vals(&column, 5, &[3]);
- test_vals(&column, 6, &[17]);
- }
-
- {
- let segment = searcher.segment_reader(1u32);
- let col = segment
- .fast_fields()
- .column_opt::<u64>("intvals")
- .unwrap()
- .unwrap();
- test_vals(&col, 0, &[28, 27]);
- test_vals(&col, 1, &[1000]);
- }
-
- {
- let segment = searcher.segment_reader(2u32);
- let col = segment
- .fast_fields()
- .column_opt::<u64>("intvals")
- .unwrap()
- .unwrap();
- test_vals(&col, 0, &[20]);
- }
-
- // Merging the segments
- {
- let segment_ids = index.searchable_segment_ids()?;
- let mut index_writer: IndexWriter = index.writer_for_tests()?;
- index_writer.merge(&segment_ids).wait()?;
- index_writer.wait_merging_threads()?;
- }
- reader.reload()?;
-
- {
- let searcher = reader.searcher();
- let segment = searcher.segment_reader(0u32);
- let col = segment
- .fast_fields()
- .column_opt::<u64>("intvals")
- .unwrap()
- .unwrap();
- test_vals(&col, 0, &[1, 2]);
- test_vals(&col, 1, &[1, 2, 3]);
- test_vals(&col, 2, &[4, 5]);
- test_vals(&col, 3, &[1, 2]);
- test_vals(&col, 4, &[1, 5]);
- test_vals(&col, 5, &[3]);
- test_vals(&col, 6, &[17]);
- test_vals(&col, 7, &[28, 27]);
- test_vals(&col, 8, &[1000]);
- test_vals(&col, 9, &[20]);
- }
- Ok(())
- }
-
#[test]
fn merges_f64_fast_fields_correctly() -> crate::Result<()> {
let mut builder = schema::SchemaBuilder::new();
- let fast_multi = NumericOptions::default().set_fast();
-
let field = builder.add_f64_field("f64", schema::FAST);
- let multi_field = builder.add_f64_field("f64s", fast_multi);
let index = Index::create_in_ram(builder.build());
@@ -1769,8 +1412,6 @@ mod tests {
for i in 0..100 {
let mut doc = TantivyDocument::new();
doc.add_f64(field, 42.0);
- doc.add_f64(multi_field, 0.24);
- doc.add_f64(multi_field, 0.27);
writer.add_document(doc)?;
if i % 5 == 0 {
writer.commit()?;
diff --git a/crates/tantivy/src/indexer/merger_sorted_index_test.rs b/crates/tantivy/src/indexer/merger_sorted_index_test.rs
index 04c7d217..5db63559 100644
--- a/crates/tantivy/src/indexer/merger_sorted_index_test.rs
+++ b/crates/tantivy/src/indexer/merger_sorted_index_test.rs
@@ -6,8 +6,8 @@ mod tests {
use crate::postings::Postings;
use crate::query::QueryParser;
use crate::schema::{
- self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,
- TextFieldIndexing, TextOptions, Value,
+ self, BytesOptions, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
+ Value,
};
use crate::{
DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, TantivyDocument,
@@ -19,8 +19,6 @@ mod tests {
let int_options = NumericOptions::default().set_fast().set_indexed();
let int_field = schema_builder.add_u64_field("intval", int_options);
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
-
let schema = schema_builder.build();
let mut index_builder = Index::builder().schema(schema);
@@ -31,16 +29,10 @@ mod tests {
{
let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(int_field=>3_u64, facet_field=> Facet::from("/crime")))
- .unwrap();
- index_writer
- .add_document(doc!(int_field=>6_u64, facet_field=> Facet::from("/crime")))
- .unwrap();
+ index_writer.add_document(doc!(int_field=>3_u64)).unwrap();
+ index_writer.add_document(doc!(int_field=>6_u64)).unwrap();
index_writer.commit().unwrap();
- index_writer
- .add_document(doc!(int_field=>5_u64, facet_field=> Facet::from("/fanta")))
- .unwrap();
+ index_writer.add_document(doc!(int_field=>5_u64)).unwrap();
index_writer.commit().unwrap();
}
@@ -69,12 +61,6 @@ mod tests {
.set_indexed();
let int_field = schema_builder.add_u64_field("intval", int_options);
- let bytes_options = BytesOptions::default().set_fast().set_indexed();
- let bytes_field = schema_builder.add_bytes_field("bytes", bytes_options);
- let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
-
- let multi_numbers =
- schema_builder.add_u64_field("multi_numbers", NumericOptions::default().set_fast());
let text_field_options = TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
@@ -94,27 +80,24 @@ mod tests {
let mut index_writer = index.writer_for_tests()?;
// segment 1 - range 1-3
- index_writer.add_document(doc!(int_field=>1_u64))?;
- index_writer.add_document(
- doc!(int_field=>3_u64, multi_numbers => 3_u64, multi_numbers => 4_u64, bytes_field => vec![1, 2, 3], text_field => "some text", facet_field=> Facet::from("/book/crime")),
- )?;
+ index_writer.add_document(doc!(int_field=>1_u64, text_field => "text"))?;
+ index_writer.add_document(doc!(int_field=>3_u64, text_field => "some text"))?;
index_writer.add_document(
doc!(int_field=>1_u64, text_field=> "deleteme", text_field => "ok text more text"),
)?;
- index_writer.add_document(
- doc!(int_field=>2_u64, multi_numbers => 2_u64, multi_numbers => 3_u64, text_field => "ok text more text"),
- )?;
+ index_writer.add_document(doc!(int_field=>2_u64, text_field => "ok text more text"))?;
index_writer.commit()?;
// segment 2 - range 1-20 , with force_disjunct_segment_sort_values 10-20
- index_writer.add_document(doc!(int_field=>20_u64, multi_numbers => 20_u64))?;
+ index_writer
+ .add_document(doc!(int_field=>20_u64, text_field => "ok text more text"))?;
let in_val = if force_disjunct_segment_sort_values {
10_u64
} else {
1
};
- index_writer.add_document(doc!(int_field=>in_val, text_field=> "deleteme" , text_field => "ok text more text", facet_field=> Facet::from("/book/crime")))?;
+ index_writer.add_document(doc!(int_field=>in_val, text_field=> "deleteme" , text_field => "ok text more text"))?;
index_writer.commit()?;
// segment 3 - range 5-1000, with force_disjunct_segment_sort_values 50-1000
let int_vals = if force_disjunct_segment_sort_values {
@@ -122,13 +105,13 @@ mod tests {
} else {
[10, 5]
};
- index_writer.add_document( // position of this doc after delete in desc sorting = [2], in disjunct case [1]
- doc!(int_field=>int_vals[0], multi_numbers => 10_u64, multi_numbers => 11_u64, text_field=> "blubber", facet_field=> Facet::from("/book/fantasy")),
- )?;
- index_writer.add_document(doc!(int_field=>int_vals[1], text_field=> "deleteme"))?;
index_writer.add_document(
- doc!(int_field=>1_000u64, multi_numbers => 1001_u64, multi_numbers => 1002_u64, bytes_field => vec![5, 5],text_field => "the biggest num")
+ // position of this doc after delete in desc sorting = [2], in disjunct case [1]
+ doc!(int_field=>int_vals[0], text_field=> "blubber"),
)?;
+ index_writer.add_document(doc!(int_field=>int_vals[1], text_field=> "deleteme"))?;
+ index_writer
+ .add_document(doc!(int_field=>1_000u64, text_field => "the biggest num"))?;
index_writer.delete_term(Term::from_field_text(text_field, "deleteme"));
index_writer.commit()?;
@@ -203,16 +186,16 @@ mod tests {
{
let my_text_field = index.schema().get_field("text_field").unwrap();
let fieldnorm_reader = segment_reader.get_fieldnorms_reader(my_text_field).unwrap();
- assert_eq!(fieldnorm_reader.fieldnorm(0), 3); // the biggest num
+ assert_eq!(fieldnorm_reader.fieldnorm(0), 3);
if force_disjunct_segment_sort_values {
assert_eq!(fieldnorm_reader.fieldnorm(1), 1); // blubber
- assert_eq!(fieldnorm_reader.fieldnorm(2), 0);
+ assert_eq!(fieldnorm_reader.fieldnorm(2), 4);
} else {
- assert_eq!(fieldnorm_reader.fieldnorm(1), 0);
+ assert_eq!(fieldnorm_reader.fieldnorm(1), 4);
assert_eq!(fieldnorm_reader.fieldnorm(2), 1); // blubber
}
assert_eq!(fieldnorm_reader.fieldnorm(3), 2); // some text
- assert_eq!(fieldnorm_reader.fieldnorm(5), 0);
+ assert_eq!(fieldnorm_reader.fieldnorm(5), 1);
}
let my_text_field = index.schema().get_field("text_field").unwrap();
@@ -249,24 +232,24 @@ mod tests {
.unwrap()
.unwrap();
- assert_eq!(postings.doc_freq(), 2);
+ assert_eq!(postings.doc_freq(), 4);
let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);
assert_eq!(
postings.doc_freq_given_deletes(
segment_reader.alive_bitset().unwrap_or(&fallback_bitset)
),
- 2
+ 4
);
- assert_eq!(postings.term_freq(), 1);
+ assert_eq!(postings.term_freq(), 2);
let mut output = vec![];
postings.positions(&mut output);
- assert_eq!(output, vec![1]);
+ assert_eq!(output, vec![1, 3]);
postings.advance();
- assert_eq!(postings.term_freq(), 2);
+ assert_eq!(postings.term_freq(), 1);
postings.positions(&mut output);
- assert_eq!(output, vec![1, 3]);
+ assert_eq!(output, vec![1]);
}
// access doc store
@@ -336,24 +319,24 @@ mod tests {
.read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
.unwrap()
.unwrap();
- assert_eq!(postings.doc_freq(), 2);
+ assert_eq!(postings.doc_freq(), 4);
let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);
assert_eq!(
postings.doc_freq_given_deletes(
segment_reader.alive_bitset().unwrap_or(&fallback_bitset)
),
- 2
+ 3
);
assert_eq!(postings.term_freq(), 1);
let mut output = vec![];
postings.positions(&mut output);
- assert_eq!(output, vec![1]);
+ assert_eq!(output, vec![0]);
postings.advance();
- assert_eq!(postings.term_freq(), 2);
+ assert_eq!(postings.term_freq(), 1);
postings.positions(&mut output);
- assert_eq!(output, vec![1, 3]);
+ assert_eq!(output, vec![1]);
}
}
diff --git a/crates/tantivy/src/indexer/mod.rs b/crates/tantivy/src/indexer/mod.rs
index 60bed2cc..7c78a61f 100644
--- a/crates/tantivy/src/indexer/mod.rs
+++ b/crates/tantivy/src/indexer/mod.rs
@@ -83,414 +83,4 @@ mod tests_mmap {
index_writer.commit()?;
Ok(())
}
-
- #[test]
- fn test_json_field_expand_dots_disabled_dot_escaped_required() {
- let mut schema_builder = Schema::builder();
- let json_field = schema_builder.add_json_field("json", TEXT);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 1);
- let parse_query = QueryParser::for_index(&index, Vec::new());
- {
- let query = parse_query
- .parse_query(r"json.k8s\.container\.name:prometheus")
- .unwrap();
- let num_docs = searcher.search(&query, &Count).unwrap();
- assert_eq!(num_docs, 1);
- }
- {
- let query = parse_query
- .parse_query(r#"json.k8s.container.name:prometheus"#)
- .unwrap();
- let num_docs = searcher.search(&query, &Count).unwrap();
- assert_eq!(num_docs, 0);
- }
- }
-
- #[test]
- fn test_json_field_number() {
- // this test was added specifically to reach some cases related to using json fields, with
- // frequency enabled, to store integers, with enough documents containing a single integer
- // that the posting list can be bitpacked.
- let mut schema_builder = Schema::builder();
-
- let json_field = schema_builder.add_json_field("json", TEXT);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer = index.writer_for_tests().unwrap();
- for _ in 0..256 {
- let json = serde_json::json!({"somekey": 1u64, "otherkey": -2i64});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
-
- let json = serde_json::json!({"somekey": "1str", "otherkey": "2str"});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- }
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 512);
- let parse_query = QueryParser::for_index(&index, Vec::new());
- {
- let query = parse_query.parse_query(r"json.somekey:1").unwrap();
- let num_docs = searcher.search(&query, &Count).unwrap();
- assert_eq!(num_docs, 256);
- }
- }
- #[test]
- fn test_json_field_null_byte() {
- // Test when field name contains a zero byte, which has special meaning in tantivy.
- // As a workaround, we convert the zero byte to the ASCII character '0'.
- // https://github.com/quickwit-oss/tantivy/issues/2340
- // https://github.com/quickwit-oss/tantivy/issues/2193
- let field_name_in = "\u{0000}";
- let field_name_out = "0";
- test_json_field_name(field_name_in, field_name_out);
- }
- #[test]
- fn test_json_field_1byte() {
- // Test when field name contains a '1' byte, which has special meaning in tantivy.
- // The 1 byte can be addressed as '1' byte or '.'.
- let field_name_in = "\u{0001}";
- let field_name_out = "\u{0001}";
- test_json_field_name(field_name_in, field_name_out);
-
- // Test when field name contains a '1' byte, which has special meaning in tantivy.
- let field_name_in = "\u{0001}";
- let field_name_out = ".";
- test_json_field_name(field_name_in, field_name_out);
- }
- #[test]
- fn test_json_field_dot() {
- // Test when field name contains a '.'
- let field_name_in = ".";
- let field_name_out = ".";
- test_json_field_name(field_name_in, field_name_out);
- }
- fn test_json_field_name(field_name_in: &str, field_name_out: &str) {
- let mut schema_builder = Schema::builder();
-
- let options = JsonObjectOptions::from(TEXT | FAST).set_expand_dots_enabled();
- let field = schema_builder.add_json_field("json", options);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer = index.writer_for_tests().unwrap();
- index_writer
- .add_document(doc!(field=>json!({format!("{field_name_in}"): "test1", format!("num{field_name_in}"): 10})))
- .unwrap();
- index_writer
- .add_document(doc!(field=>json!({format!("a{field_name_in}"): "test2"})))
- .unwrap();
- index_writer
- .add_document(doc!(field=>json!({format!("a{field_name_in}a"): "test3"})))
- .unwrap();
- index_writer
- .add_document(
- doc!(field=>json!({format!("a{field_name_in}a{field_name_in}"): "test4"})),
- )
- .unwrap();
- index_writer
- .add_document(
- doc!(field=>json!({format!("a{field_name_in}.ab{field_name_in}"): "test5"})),
- )
- .unwrap();
- index_writer
- .add_document(
- doc!(field=>json!({format!("a{field_name_in}"): json!({format!("a{field_name_in}"): "test6"}) })),
- )
- .unwrap();
- index_writer
- .add_document(doc!(field=>json!({format!("{field_name_in}a" ): "test7"})))
- .unwrap();
-
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- let parse_query = QueryParser::for_index(&index, Vec::new());
- let test_query = |query_str: &str| {
- let query = parse_query.parse_query(query_str).unwrap();
- let num_docs = searcher.search(&query, &Count).unwrap();
- assert_eq!(num_docs, 1, "{query_str}");
- };
- test_query(format!("json.{field_name_out}:test1").as_str());
- test_query(format!("json.a{field_name_out}:test2").as_str());
- test_query(format!("json.a{field_name_out}a:test3").as_str());
- test_query(format!("json.a{field_name_out}a{field_name_out}:test4").as_str());
- test_query(format!("json.a{field_name_out}.ab{field_name_out}:test5").as_str());
- test_query(format!("json.a{field_name_out}.a{field_name_out}:test6").as_str());
- test_query(format!("json.{field_name_out}a:test7").as_str());
-
- // `.` is stored as `\u{0001}` internally in tantivy
- let field_name_out_internal = if field_name_out == "." {
- "\u{0001}"
- } else {
- field_name_out
- };
-
- let mut fields = reader.searcher().segment_readers()[0]
- .inverted_index(field)
- .unwrap()
- .list_encoded_fields()
- .unwrap();
- assert_eq!(fields.len(), 8);
- fields.sort();
- let mut expected_fields = vec![
- (format!("a{field_name_out_internal}"), Type::Str),
- (format!("a{field_name_out_internal}a"), Type::Str),
- (
- format!("a{field_name_out_internal}a{field_name_out_internal}"),
- Type::Str,
- ),
- (
- format!("a{field_name_out_internal}\u{1}ab{field_name_out_internal}"),
- Type::Str,
- ),
- (
- format!("a{field_name_out_internal}\u{1}a{field_name_out_internal}"),
- Type::Str,
- ),
- (format!("{field_name_out_internal}a"), Type::Str),
- (field_name_out_internal.to_string(), Type::Str),
- (format!("num{field_name_out_internal}"), Type::I64),
- ];
- expected_fields.sort();
- assert_eq!(fields, expected_fields);
- // Check columnar reader
- let mut columns = reader.searcher().segment_readers()[0]
- .fast_fields()
- .columnar()
- .list_columns()
- .unwrap()
- .into_iter()
- .map(|(name, _)| name)
- .collect::<Vec<_>>();
- let mut expected_columns = vec![
- format!("json\u{1}{field_name_out_internal}"),
- format!("json\u{1}{field_name_out_internal}a"),
- format!("json\u{1}a{field_name_out_internal}"),
- format!("json\u{1}a{field_name_out_internal}a"),
- format!("json\u{1}a{field_name_out_internal}a{field_name_out_internal}"),
- format!("json\u{1}a{field_name_out_internal}\u{1}ab{field_name_out_internal}"),
- format!("json\u{1}a{field_name_out_internal}\u{1}a{field_name_out_internal}"),
- format!("json\u{1}num{field_name_out_internal}"),
- ];
- columns.sort();
- expected_columns.sort();
- assert_eq!(columns, expected_columns);
- }
-
- #[test]
- fn test_json_field_expand_dots_enabled_dot_escape_not_required() {
- let mut schema_builder = Schema::builder();
- let json_options: JsonObjectOptions =
- JsonObjectOptions::from(TEXT).set_expand_dots_enabled();
- let json_field = schema_builder.add_json_field("json", json_options);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 1);
- let parse_query = QueryParser::for_index(&index, Vec::new());
- {
- let query = parse_query
- .parse_query(r#"json.k8s.container.name:prometheus"#)
- .unwrap();
- let num_docs = searcher.search(&query, &Count).unwrap();
- assert_eq!(num_docs, 1);
- }
- {
- let query = parse_query
- .parse_query(r"json.k8s\.container\.name:prometheus")
- .unwrap();
- let num_docs = searcher.search(&query, &Count).unwrap();
- assert_eq!(num_docs, 1);
- }
- }
-
- #[test]
- fn test_json_field_list_fields() {
- let mut schema_builder = Schema::builder();
- let json_options: JsonObjectOptions = JsonObjectOptions::from(TEXT);
- let json_field = schema_builder.add_json_field("json", json_options);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer = index.writer_for_tests().unwrap();
- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "sub": {"a": 1, "b": 2}});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": 1, "b": 2}});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": "mixed", "b": 2}});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
-
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 3);
-
- let reader = &searcher.segment_readers()[0];
- let inverted_index = reader.inverted_index(json_field).unwrap();
- assert_eq!(
- inverted_index.list_encoded_fields().unwrap(),
- [
- ("k8s.container.name".to_string(), Type::Str),
- ("sub\u{1}a".to_string(), Type::I64),
- ("sub\u{1}b".to_string(), Type::I64),
- ("suber\u{1}a".to_string(), Type::I64),
- ("suber\u{1}a".to_string(), Type::Str),
- ("suber\u{1}b".to_string(), Type::I64),
- ("val".to_string(), Type::Str),
- ]
- );
- }
-
- #[test]
- fn test_json_fields_metadata_expanded_dots_one_segment() {
- test_json_fields_metadata(true, true);
- }
- #[test]
- fn test_json_fields_metadata_expanded_dots_multi_segment() {
- test_json_fields_metadata(true, false);
- }
- #[test]
- fn test_json_fields_metadata_no_expanded_dots_one_segment() {
- test_json_fields_metadata(false, true);
- }
- #[test]
- fn test_json_fields_metadata_no_expanded_dots_multi_segment() {
- test_json_fields_metadata(false, false);
- }
-
- fn test_json_fields_metadata(expanded_dots: bool, one_segment: bool) {
- use pretty_assertions::assert_eq;
- let mut schema_builder = Schema::builder();
- let json_options: JsonObjectOptions =
- JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
- let json_options = if expanded_dots {
- json_options.set_expand_dots_enabled()
- } else {
- json_options
- };
- schema_builder.add_json_field("json.confusing", json_options.clone());
- let json_field = schema_builder.add_json_field("json.shadow", json_options.clone());
- let json_field2 = schema_builder.add_json_field("json", json_options.clone());
- schema_builder.add_json_field("empty_json", json_options);
- let number_field = schema_builder.add_u64_field("numbers", FAST);
- schema_builder.add_u64_field("empty", FAST | INDEXED | STORED);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer = index.writer_for_tests().unwrap();
- let json =
- serde_json::json!({"k8s.container.name": "a", "val": "a", "sub": {"a": 1, "b": 1}});
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- let json =
- serde_json::json!({"k8s.container.name": "a", "val": "a", "suber": {"a": 1, "b": 1}});
- if !one_segment {
- index_writer.commit().unwrap();
- }
- index_writer.add_document(doc!(json_field=>json)).unwrap();
- let json = serde_json::json!({"k8s.container.name": "a", "k8s.container.name": "a", "val": "a", "suber": {"a": "a", "b": 1}});
- index_writer
- .add_document(doc!(number_field => 50u64, json_field=>json, json_field2=>json!({"shadow": {"val": "a"}})))
- .unwrap();
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
-
- let searcher = reader.searcher();
- assert_eq!(searcher.num_docs(), 3);
-
- let fields_metadata = index.fields_metadata().unwrap();
- assert_eq!(
- fields_metadata,
- [
- FieldMetadata {
- field_name: "empty".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::U64
- },
- FieldMetadata {
- field_name: if expanded_dots {
- "json.shadow.k8s.container.name".to_string()
- } else {
- "json.shadow.k8s\\.container\\.name".to_string()
- },
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::Str
- },
- FieldMetadata {
- field_name: "json.shadow.sub.a".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::I64
- },
- FieldMetadata {
- field_name: "json.shadow.sub.b".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::I64
- },
- FieldMetadata {
- field_name: "json.shadow.suber.a".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::I64
- },
- FieldMetadata {
- field_name: "json.shadow.suber.a".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::Str
- },
- FieldMetadata {
- field_name: "json.shadow.suber.b".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::I64
- },
- FieldMetadata {
- field_name: "json.shadow.val".to_string(),
- indexed: true,
- stored: true,
- fast: true,
- typ: Type::Str
- },
- FieldMetadata {
- field_name: "numbers".to_string(),
- indexed: false,
- stored: false,
- fast: true,
- typ: Type::U64
- }
- ]
- );
- let query_parser = QueryParser::for_index(&index, vec![]);
- // Test if returned field name can be queried
- for indexed_field in fields_metadata.iter().filter(|meta| meta.indexed) {
- let val = if indexed_field.typ == Type::Str {
- "a"
- } else {
- "1"
- };
- let query_str = &format!("{}:{}", indexed_field.field_name, val);
- let query = query_parser.parse_query(query_str).unwrap();
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
- if indexed_field.field_name.contains("empty") || indexed_field.typ == Type::Json {
- assert_eq!(count_docs.len(), 0);
- } else {
- assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);
- }
- }
- }
}
diff --git a/crates/tantivy/src/indexer/segment_writer.rs b/crates/tantivy/src/indexer/segment_writer.rs
index d16dd613..4d716391 100644
--- a/crates/tantivy/src/indexer/segment_writer.rs
+++ b/crates/tantivy/src/indexer/segment_writer.rs
@@ -17,7 +17,7 @@ use crate::postings::{
use crate::schema::document::{Document, Value};
use crate::schema::{FieldEntry, FieldType, Schema, Term, DATE_TIME_PRECISION_INDEXED};
use crate::store::{StoreReader, StoreWriter};
-use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
+use crate::tokenizer::{PreTokenizedStream, TextAnalyzer};
use crate::{DocId, Opstamp, TantivyError};
/// Computes the initial size of the hash table.
@@ -200,23 +200,6 @@ impl SegmentWriter {
term_buffer.clear_with_field_and_type(field_entry.field_type().value_type(), field);
match field_entry.field_type() {
- FieldType::Facet(_) => {
- let mut facet_tokenizer = FacetTokenizer::default(); // this can be global
- for value in values {
- let value = value.as_value();
-
- let facet_str = value.as_facet().ok_or_else(make_schema_error)?;
- let mut facet_tokenizer = facet_tokenizer.token_stream(facet_str);
- let mut indexing_position = IndexingPosition::default();
- postings_writer.index_text(
- doc_id,
- &mut facet_tokenizer,
- term_buffer,
- ctx,
- &mut indexing_position,
- );
- }
- }
FieldType::Str(_) => {
let mut indexing_position = IndexingPosition::default();
for value in values {
diff --git a/crates/tantivy/src/postings/per_field_postings_writer.rs b/crates/tantivy/src/postings/per_field_postings_writer.rs
index f3d6d653..b283b3d9 100644
--- a/crates/tantivy/src/postings/per_field_postings_writer.rs
+++ b/crates/tantivy/src/postings/per_field_postings_writer.rs
@@ -50,8 +50,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
| FieldType::Bool(_)
| FieldType::Date(_)
| FieldType::Bytes(_)
- | FieldType::IpAddr(_)
- | FieldType::Facet(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),
+ | FieldType::IpAddr(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),
FieldType::JsonObject(ref json_object_options) => {
if let Some(text_indexing_option) = json_object_options.get_text_indexing_options() {
match text_indexing_option.index_option() {
diff --git a/crates/tantivy/src/query/exist_query.rs b/crates/tantivy/src/query/exist_query.rs
deleted file mode 100644
index ca1213e5..00000000
--- a/crates/tantivy/src/query/exist_query.rs
+++ /dev/null
@@ -1,345 +0,0 @@
-use core::fmt::Debug;
-
-use crate::columnar::{ColumnIndex, DynamicColumn};
-
-use super::{ConstScorer, EmptyScorer};
-use crate::docset::{DocSet, TERMINATED};
-use crate::index::SegmentReader;
-use crate::query::explanation::does_not_match;
-use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
-use crate::{DocId, Score, TantivyError};
-
-/// Query that matches all documents with a non-null value in the specified field.
-///
-/// All of the matched documents get the score 1.0.
-#[derive(Clone, Debug)]
-pub struct ExistsQuery {
- field_name: String,
-}
-
-impl ExistsQuery {
- /// Creates a new `ExistQuery` from the given field.
- ///
- /// This query matches all documents with at least one non-null value in the specified field.
- /// This constructor never fails, but executing the search with this query will return an
- /// error if the specified field doesn't exists or is not a fast field.
- pub fn new_exists_query(field: String) -> ExistsQuery {
- ExistsQuery { field_name: field }
- }
-}
-
-impl Query for ExistsQuery {
- fn weight(&self, enable_scoring: EnableScoring) -> crate::Result<Box<dyn Weight>> {
- let schema = enable_scoring.schema();
- let Some((field, _path)) = schema.find_field(&self.field_name) else {
- return Err(TantivyError::FieldNotFound(self.field_name.clone()));
- };
- let field_type = schema.get_field_entry(field).field_type();
- if !field_type.is_fast() {
- return Err(TantivyError::SchemaError(format!(
- "Field {} is not a fast field.",
- self.field_name
- )));
- }
- Ok(Box::new(ExistsWeight {
- field_name: self.field_name.clone(),
- }))
- }
-}
-
-/// Weight associated with the `ExistsQuery` query.
-pub struct ExistsWeight {
- field_name: String,
-}
-
-impl Weight for ExistsWeight {
- fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
- let fast_field_reader = reader.fast_fields();
- let dynamic_columns: crate::Result<Vec<DynamicColumn>> = fast_field_reader
- .dynamic_column_handles(&self.field_name)?
- .into_iter()
- .map(|handle| handle.open().map_err(|io_error| io_error.into()))
- .collect();
- let mut non_empty_columns = Vec::new();
- for column in dynamic_columns? {
- if !matches!(column.column_index(), ColumnIndex::Empty { .. }) {
- non_empty_columns.push(column)
- }
- }
- // TODO: we can optimizer more here since in most cases we will have only one index
- if !non_empty_columns.is_empty() {
- let docset = ExistsDocSet::new(non_empty_columns, reader.max_doc());
- Ok(Box::new(ConstScorer::new(docset, boost)))
- } else {
- Ok(Box::new(EmptyScorer))
- }
- }
-
- fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
- let mut scorer = self.scorer(reader, 1.0)?;
- if scorer.seek(doc) != doc {
- return Err(does_not_match(doc));
- }
- Ok(Explanation::new("ExistsQuery", 1.0))
- }
-}
-
-pub(crate) struct ExistsDocSet {
- columns: Vec<DynamicColumn>,
- doc: DocId,
- max_doc: DocId,
-}
-
-impl ExistsDocSet {
- pub(crate) fn new(columns: Vec<DynamicColumn>, max_doc: DocId) -> Self {
- let mut set = Self {
- columns,
- doc: 0u32,
- max_doc,
- };
- set.find_next();
- set
- }
-
- fn find_next(&mut self) -> DocId {
- while self.doc < self.max_doc {
- if self
- .columns
- .iter()
- .any(|col| col.column_index().has_value(self.doc))
- {
- return self.doc;
- }
- self.doc += 1;
- }
- self.doc = TERMINATED;
- TERMINATED
- }
-}
-
-impl DocSet for ExistsDocSet {
- fn advance(&mut self) -> DocId {
- self.seek(self.doc + 1)
- }
-
- fn size_hint(&self) -> u32 {
- 0
- }
-
- fn doc(&self) -> DocId {
- self.doc
- }
-
- #[inline(always)]
- fn seek(&mut self, target: DocId) -> DocId {
- self.doc = target;
- self.find_next()
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::net::Ipv6Addr;
- use std::ops::Bound;
-
- use crate::common::DateTime;
- use time::OffsetDateTime;
-
- use crate::collector::Count;
- use crate::query::exist_query::ExistsQuery;
- use crate::query::{BooleanQuery, RangeQuery};
- use crate::schema::{Facet, FacetOptions, Schema, FAST, INDEXED, STRING, TEXT};
- use crate::{Index, Searcher};
-
- #[test]
- fn test_exists_query_simple() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let all_field = schema_builder.add_u64_field("all", INDEXED | FAST);
- let even_field = schema_builder.add_u64_field("even", INDEXED | FAST);
- let odd_field = schema_builder.add_text_field("odd", STRING | FAST);
- let multi_field = schema_builder.add_text_field("multi", FAST);
- let _never_field = schema_builder.add_u64_field("never", INDEXED | FAST);
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema);
- {
- let mut index_writer = index.writer_for_tests()?;
- for i in 0u64..100u64 {
- if i % 2 == 0 {
- if i % 10 == 0 {
- index_writer.add_document(doc!(all_field => i, even_field => i, multi_field => i.to_string(), multi_field => (i + 1).to_string()))?;
- } else {
- index_writer.add_document(doc!(all_field => i, even_field => i))?;
- }
- } else {
- index_writer.add_document(doc!(all_field => i, odd_field => i.to_string()))?;
- }
- }
- index_writer.commit()?;
- }
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- assert_eq!(count_existing_fields(&searcher, "all")?, 100);
- assert_eq!(count_existing_fields(&searcher, "odd")?, 50);
- assert_eq!(count_existing_fields(&searcher, "even")?, 50);
- assert_eq!(count_existing_fields(&searcher, "multi")?, 10);
- assert_eq!(count_existing_fields(&searcher, "never")?, 0);
-
- // exercise seek
- let query = BooleanQuery::intersection(vec![
- Box::new(RangeQuery::new_u64_bounds(
- "all".to_string(),
- Bound::Included(50),
- Bound::Unbounded,
- )),
- Box::new(ExistsQuery::new_exists_query("even".to_string())),
- ]);
- assert_eq!(searcher.search(&query, &Count)?, 25);
-
- let query = BooleanQuery::intersection(vec![
- Box::new(RangeQuery::new_u64_bounds(
- "all".to_string(),
- Bound::Included(0),
- Bound::Excluded(50),
- )),
- Box::new(ExistsQuery::new_exists_query("odd".to_string())),
- ]);
- assert_eq!(searcher.search(&query, &Count)?, 25);
-
- Ok(())
- }
-
- #[test]
- fn test_exists_query_json() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", TEXT | FAST);
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema);
- {
- let mut index_writer = index.writer_for_tests()?;
- for i in 0u64..100u64 {
- if i % 2 == 0 {
- index_writer.add_document(doc!(json => json!({"all": i, "even": true})))?;
- } else {
- index_writer
- .add_document(doc!(json => json!({"all": i.to_string(), "odd": true})))?;
- }
- }
- index_writer.commit()?;
- }
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- assert_eq!(count_existing_fields(&searcher, "json.all")?, 100);
- assert_eq!(count_existing_fields(&searcher, "json.even")?, 50);
- assert_eq!(count_existing_fields(&searcher, "json.odd")?, 50);
-
- // Handling of non-existing fields:
- assert_eq!(count_existing_fields(&searcher, "json.absent")?, 0);
- assert_eq!(
- searcher
- .search(
- &ExistsQuery::new_exists_query("does_not_exists.absent".to_string()),
- &Count
- )
- .unwrap_err()
- .to_string(),
- "The field does not exist: 'does_not_exists.absent'"
- );
-
- Ok(())
- }
-
- #[test]
- fn test_exists_query_misc_supported_types() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let bool = schema_builder.add_bool_field("bool", FAST);
- let bytes = schema_builder.add_bytes_field("bytes", FAST);
- let date = schema_builder.add_date_field("date", FAST);
- let f64 = schema_builder.add_f64_field("f64", FAST);
- let ip_addr = schema_builder.add_ip_addr_field("ip_addr", FAST);
- let facet = schema_builder.add_facet_field("facet", FacetOptions::default());
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema);
- {
- let mut index_writer = index.writer_for_tests()?;
- let now = OffsetDateTime::now_utc().unix_timestamp();
- for i in 0u8..100u8 {
- if i % 2 == 0 {
- let date_val = DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
- now + i as i64 * 100,
- )?);
- index_writer.add_document(
- doc!(bool => i % 3 == 0, bytes => vec![i, i + 1, i + 2], date => date_val),
- )?;
- } else {
- let ip_addr_v6 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, i.into());
- index_writer
- .add_document(doc!(f64 => i as f64 * 0.5, ip_addr => ip_addr_v6, facet => Facet::from("/facet/foo"), facet => Facet::from("/facet/bar")))?;
- }
- }
- index_writer.commit()?;
- }
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- assert_eq!(count_existing_fields(&searcher, "bool")?, 50);
- assert_eq!(count_existing_fields(&searcher, "bytes")?, 50);
- assert_eq!(count_existing_fields(&searcher, "date")?, 50);
- assert_eq!(count_existing_fields(&searcher, "f64")?, 50);
- assert_eq!(count_existing_fields(&searcher, "ip_addr")?, 50);
- assert_eq!(count_existing_fields(&searcher, "facet")?, 50);
-
- Ok(())
- }
-
- #[test]
- fn test_exists_query_unsupported_types() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let not_fast = schema_builder.add_text_field("not_fast", TEXT);
- let schema = schema_builder.build();
-
- let index = Index::create_in_ram(schema);
- {
- let mut index_writer = index.writer_for_tests()?;
- index_writer.add_document(doc!(
- not_fast => "slow",
- ))?;
- index_writer.commit()?;
- }
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- assert_eq!(
- searcher
- .search(
- &ExistsQuery::new_exists_query("not_fast".to_string()),
- &Count
- )
- .unwrap_err()
- .to_string(),
- "Schema error: 'Field not_fast is not a fast field.'"
- );
-
- assert_eq!(
- searcher
- .search(
- &ExistsQuery::new_exists_query("does_not_exists".to_string()),
- &Count
- )
- .unwrap_err()
- .to_string(),
- "The field does not exist: 'does_not_exists'"
- );
-
- Ok(())
- }
-
- fn count_existing_fields(searcher: &Searcher, field: &str) -> crate::Result<usize> {
- let query = ExistsQuery::new_exists_query(field.to_string());
- searcher.search(&query, &Count)
- }
-}
diff --git a/crates/tantivy/src/query/mod.rs b/crates/tantivy/src/query/mod.rs
index 4083627a..5ab71842 100644
--- a/crates/tantivy/src/query/mod.rs
+++ b/crates/tantivy/src/query/mod.rs
@@ -8,7 +8,6 @@ mod const_score_query;
mod disjunction_max_query;
mod empty_query;
mod exclude;
-mod exist_query;
mod explanation;
mod fuzzy_query;
mod intersection;
@@ -44,7 +43,6 @@ pub use self::const_score_query::{ConstScoreQuery, ConstScorer};
pub use self::disjunction_max_query::DisjunctionMaxQuery;
pub use self::empty_query::{EmptyQuery, EmptyScorer, EmptyWeight};
pub use self::exclude::Exclude;
-pub use self::exist_query::ExistsQuery;
pub use self::explanation::Explanation;
#[cfg(test)]
pub(crate) use self::fuzzy_query::DfaWrapper;
@@ -56,7 +54,7 @@ pub use self::phrase_query::PhraseQuery;
pub use self::query::{EnableScoring, Query, QueryClone};
#[cfg(test)]
pub use self::query_parser::{QueryParser, QueryParserError};
-pub use self::range_query::{FastFieldRangeWeight, IPFastFieldRangeWeight, RangeQuery};
+pub use self::range_query::{FastFieldRangeWeight, RangeQuery};
pub use self::reqopt_scorer::RequiredOptionalScorer;
pub use self::score_combiner::{
DisjunctionMaxCombiner, ScoreCombiner, SumCombiner, SumWithCoordsCombiner,
diff --git a/crates/tantivy/src/query/more_like_this/more_like_this.rs b/crates/tantivy/src/query/more_like_this/more_like_this.rs
index 1240be53..9ae5d307 100644
--- a/crates/tantivy/src/query/more_like_this/more_like_this.rs
+++ b/crates/tantivy/src/query/more_like_this/more_like_this.rs
@@ -7,7 +7,7 @@ use crate::query::bm25::idf;
use crate::query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery};
use crate::schema::document::{Document, Value};
use crate::schema::{Field, FieldType, IndexRecordOption, Term};
-use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer};
+use crate::tokenizer::{PreTokenizedStream, TokenStream};
use crate::{DocAddress, Result, Searcher, TantivyDocument, TantivyError};
#[derive(Debug, PartialEq)]
@@ -176,26 +176,6 @@ impl MoreLikeThis {
// extract the raw value, possibly tokenizing & filtering to update the term frequency map
match field_entry.field_type() {
- FieldType::Facet(_) => {
- let facets: Vec<&str> = values
- .iter()
- .map(|value| {
- value.as_facet().ok_or_else(|| {
- TantivyError::InvalidArgument("invalid field value".to_string())
- })
- })
- .collect::<Result<Vec<_>>>()?;
- for fake_str in facets {
- FacetTokenizer::default()
- .token_stream(fake_str)
- .process(&mut |token| {
- if self.is_noise_word(token.text.clone()) {
- let term = Term::from_field_text(field, &token.text);
- *term_frequencies.entry(term).or_insert(0) += 1;
- }
- });
- }
- }
FieldType::Str(text_options) => {
let mut tokenizer_opt = text_options
.get_indexing_options()
diff --git a/crates/tantivy/src/query/query_parser/query_parser.rs b/crates/tantivy/src/query/query_parser/query_parser.rs
index ef681603..f37ab2d8 100644
--- a/crates/tantivy/src/query/query_parser/query_parser.rs
+++ b/crates/tantivy/src/query/query_parser/query_parser.rs
@@ -18,8 +18,8 @@ use crate::query::{
PhraseQuery, Query, TermQuery, TermSetQuery,
};
use crate::schema::{
- Facet, FacetParseError, Field, FieldType, IndexRecordOption, IntoIpv6Addr, JsonObjectOptions,
- Schema, Term, TextFieldIndexing, Type,
+ Field, FieldType, IndexRecordOption, IntoIpv6Addr, JsonObjectOptions, Schema, Term,
+ TextFieldIndexing, Type,
};
use crate::time::format_description::well_known::Rfc3339;
use crate::time::OffsetDateTime;
@@ -96,9 +96,6 @@ pub enum QueryParserError {
/// The format for the date field is not RFC 3339 compliant.
#[error("The date field has an invalid format")]
DateFormatError(#[from] time::error::Parse),
- /// The format for the facet field is invalid.
- #[error("The facet field is malformed: {0}")]
- FacetFormatError(#[from] FacetParseError),
/// The format for the ip field is invalid.
#[error("The ip field is malformed: {0}")]
IpFormatError(#[from] AddrParseError),
@@ -493,10 +490,6 @@ impl QueryParser {
"Range query are not supported on json field.".to_string(),
))
}
- FieldType::Facet(_) => match Facet::from_text(phrase) {
- Ok(facet) => Ok(Term::from_facet(field, &facet)),
- Err(e) => Err(QueryParserError::from(e)),
- },
FieldType::Bytes(_) => {
let bytes = BASE64
.decode(phrase)
@@ -588,13 +581,6 @@ impl QueryParser {
&self.tokenizer_manager,
json_options,
),
- FieldType::Facet(_) => match Facet::from_text(phrase) {
- Ok(facet) => {
- let facet_term = Term::from_facet(field, &facet);
- Ok(vec![LogicalLiteral::Term(facet_term)])
- }
- Err(e) => Err(QueryParserError::from(e)),
- },
FieldType::Bytes(_) => {
let bytes = BASE64
.decode(phrase)
@@ -1019,886 +1005,3 @@ fn convert_to_query(fuzzy: &FxHashMap<Field, Fuzzy>, logical_ast: LogicalAst) ->
None => Box::new(EmptyQuery),
}
}
-
-#[cfg(test)]
-mod test {
- use matches::assert_matches;
-
- use super::super::logical_ast::*;
- use super::{QueryParser, QueryParserError};
- use crate::query::Query;
- use crate::schema::{
- FacetOptions, Field, IndexRecordOption, Schema, Term, TextFieldIndexing, TextOptions, FAST,
- INDEXED, STORED, STRING, TEXT,
- };
- use crate::tokenizer::{
- LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
- };
- use crate::Index;
-
- fn make_schema() -> Schema {
- let mut schema_builder = Schema::builder();
- let text_field_indexing = TextFieldIndexing::default()
- .set_tokenizer("en_with_stop_words")
- .set_index_option(IndexRecordOption::WithFreqsAndPositions);
- let text_options = TextOptions::default()
- .set_indexing_options(text_field_indexing)
- .set_stored();
- schema_builder.add_text_field("title", TEXT);
- schema_builder.add_text_field("text", TEXT);
- schema_builder.add_i64_field("signed", INDEXED);
- schema_builder.add_u64_field("unsigned", INDEXED);
- schema_builder.add_text_field("notindexed_text", STORED);
- schema_builder.add_text_field("notindexed_u64", STORED);
- schema_builder.add_text_field("notindexed_i64", STORED);
- schema_builder.add_text_field("nottokenized", STRING);
- schema_builder.add_text_field("with_stop_words", text_options);
- schema_builder.add_date_field("date", INDEXED);
- schema_builder.add_f64_field("float", INDEXED);
- schema_builder.add_facet_field("facet", FacetOptions::default());
- schema_builder.add_bytes_field("bytes", INDEXED);
- schema_builder.add_bytes_field("bytes_not_indexed", STORED);
- schema_builder.add_json_field("json", TEXT);
- schema_builder.add_json_field("json_not_indexed", STORED);
- schema_builder.add_bool_field("bool", INDEXED);
- schema_builder.add_bool_field("notindexed_bool", STORED);
- schema_builder.add_u64_field("u64_ff", FAST);
- schema_builder.build()
- }
-
- fn make_query_parser_with_default_fields(default_fields: &[&'static str]) -> QueryParser {
- let schema = make_schema();
- let default_fields: Vec<Field> = default_fields
- .iter()
- .flat_map(|field_name| schema.get_field(field_name))
- .collect();
- let tokenizer_manager = TokenizerManager::default();
- tokenizer_manager.register(
- "en_with_stop_words",
- TextAnalyzer::builder(SimpleTokenizer::default())
- .filter(LowerCaser)
- .filter(StopWordFilter::remove(vec!["the".to_string()]))
- .build(),
- );
- QueryParser::new(schema, default_fields, tokenizer_manager)
- }
-
- fn make_query_parser() -> QueryParser {
- make_query_parser_with_default_fields(&["title", "text"])
- }
-
- fn parse_query_to_logical_ast(
- query: &str,
- default_conjunction: bool,
- ) -> Result<LogicalAst, QueryParserError> {
- let mut query_parser = make_query_parser();
- if default_conjunction {
- query_parser.set_conjunction_by_default();
- }
- query_parser.parse_query_to_logical_ast(query)
- }
-
- fn test_parse_query_to_logical_ast_helper(
- query: &str,
- expected: &str,
- default_conjunction: bool,
- ) {
- let query = parse_query_to_logical_ast(query, default_conjunction).unwrap();
- let query_str = format!("{query:?}");
- assert_eq!(query_str, expected);
- }
-
- #[test]
- pub fn test_parse_query_facet() {
- let query_parser = make_query_parser();
- let query = query_parser.parse_query("facet:/root/branch/leaf").unwrap();
- assert_eq!(
- format!("{query:?}"),
- r#"TermQuery(Term(field=11, type=Facet, Facet(/root/branch/leaf)))"#
- );
- }
-
- #[test]
- pub fn test_parse_query_with_boost() {
- let mut query_parser = make_query_parser();
- let schema = make_schema();
- let text_field = schema.get_field("text").unwrap();
- query_parser.set_field_boost(text_field, 2.0);
- let query = query_parser.parse_query("text:hello").unwrap();
- assert_eq!(
- format!("{query:?}"),
- r#"Boost(query=TermQuery(Term(field=1, type=Str, "hello")), boost=2)"#
- );
- }
-
- #[test]
- pub fn test_parse_query_range_with_boost() {
- let query = make_query_parser().parse_query("title:[A TO B]").unwrap();
- assert_eq!(
- format!("{query:?}"),
- "RangeQuery { field: \"title\", value_type: Str, lower_bound: Included([97]), \
- upper_bound: Included([98]), limit: None }"
- );
- }
-
- #[test]
- pub fn test_parse_query_with_default_boost_and_custom_boost() {
- let mut query_parser = make_query_parser();
- let schema = make_schema();
- let text_field = schema.get_field("text").unwrap();
- query_parser.set_field_boost(text_field, 2.0);
- let query = query_parser.parse_query("text:hello^2").unwrap();
- assert_eq!(
- format!("{query:?}"),
- r#"Boost(query=Boost(query=TermQuery(Term(field=1, type=Str, "hello")), boost=2), boost=2)"#
- );
- }
-
- #[test]
- pub fn test_parse_nonindexed_field_yields_error() {
- let query_parser = make_query_parser();
-
- let is_not_indexed_err = |query: &str| {
- let result: Result<Box<dyn Query>, QueryParserError> = query_parser.parse_query(query);
- if let Err(QueryParserError::FieldNotIndexed(field_name)) = result {
- Some(field_name)
- } else {
- None
- }
- };
-
- assert_eq!(
- is_not_indexed_err("notindexed_text:titi"),
- Some(String::from("notindexed_text"))
- );
- assert_eq!(
- is_not_indexed_err("notindexed_u64:23424"),
- Some(String::from("notindexed_u64"))
- );
- assert_eq!(
- is_not_indexed_err("notindexed_i64:-234324"),
- Some(String::from("notindexed_i64"))
- );
- assert_eq!(
- is_not_indexed_err("notindexed_bool:true"),
- Some(String::from("notindexed_bool"))
- );
- }
-
- #[test]
- pub fn test_parse_query_untokenized() {
- test_parse_query_to_logical_ast_helper(
- "nottokenized:\"wordone wordtwo\"",
- r#"Term(field=7, type=Str, "wordone wordtwo")"#,
- false,
- );
- }
-
- #[test]
- pub fn test_parse_query_empty() {
- test_parse_query_to_logical_ast_helper("", "<emptyclause>", false);
- test_parse_query_to_logical_ast_helper(" ", "<emptyclause>", false);
- let query_parser = make_query_parser();
- let query_result = query_parser.parse_query("");
- let query = query_result.unwrap();
- assert_eq!(format!("{query:?}"), "EmptyQuery");
- }
-
- #[test]
- pub fn test_parse_query_ints() {
- let query_parser = make_query_parser();
- assert!(query_parser.parse_query("signed:2324").is_ok());
- assert!(query_parser.parse_query("signed:\"22\"").is_ok());
- assert!(query_parser.parse_query("signed:\"-2234\"").is_ok());
- assert!(query_parser
- .parse_query("signed:\"-9999999999999\"")
- .is_ok());
- assert!(query_parser.parse_query("signed:\"a\"").is_err());
- assert!(query_parser.parse_query("signed:\"2a\"").is_err());
- assert!(query_parser
- .parse_query("signed:\"18446744073709551615\"")
- .is_err());
- assert!(query_parser.parse_query("unsigned:\"2\"").is_ok());
- assert!(query_parser.parse_query("unsigned:\"-2\"").is_err());
- assert!(query_parser
- .parse_query("unsigned:\"18446744073709551615\"")
- .is_ok());
- assert!(query_parser.parse_query("float:\"3.1\"").is_ok());
- assert!(query_parser.parse_query("float:\"-2.4\"").is_ok());
- assert!(query_parser.parse_query("float:\"2.1.2\"").is_err());
- assert!(query_parser.parse_query("float:\"2.1a\"").is_err());
- assert!(query_parser
- .parse_query("float:\"18446744073709551615.0\"")
- .is_ok());
- test_parse_query_to_logical_ast_helper(
- "unsigned:2324",
- "Term(field=3, type=U64, 2324)",
- false,
- );
-
- test_parse_query_to_logical_ast_helper(
- "signed:-2324",
- &format!(
- "{:?}",
- Term::from_field_i64(Field::from_field_id(2u32), -2324)
- ),
- false,
- );
-
- test_parse_query_to_logical_ast_helper(
- "float:2.5",
- &format!(
- "{:?}",
- Term::from_field_f64(Field::from_field_id(10u32), 2.5)
- ),
- false,
- );
- }
-
- #[test]
- fn test_parse_bytes() {
- test_parse_query_to_logical_ast_helper(
- "bytes:YnVidQ==",
- "Term(field=12, type=Bytes, [98, 117, 98, 117])",
- false,
- );
- }
-
- #[test]
- fn test_parse_bool() {
- test_parse_query_to_logical_ast_helper(
- "bool:true",
- &format!(
- "{:?}",
- Term::from_field_bool(Field::from_field_id(16u32), true),
- ),
- false,
- );
- }
-
- #[test]
- fn test_parse_bytes_not_indexed() {
- let error = parse_query_to_logical_ast("bytes_not_indexed:aaa", false).unwrap_err();
- assert!(matches!(error, QueryParserError::FieldNotIndexed(_)));
- }
-
- #[test]
- fn test_json_field() {
- test_parse_query_to_logical_ast_helper(
- "json.titi:hello",
- "Term(field=14, type=Json, path=titi, type=Str, \"hello\")",
- false,
- );
- }
-
- fn extract_query_term_json_path(query: &str) -> String {
- let LogicalAst::Leaf(literal) = parse_query_to_logical_ast(query, false).unwrap() else {
- panic!();
- };
- let LogicalLiteral::Term(term) = *literal else {
- panic!();
- };
- std::str::from_utf8(term.serialized_value_bytes())
- .unwrap()
- .to_string()
- }
-
- #[test]
- fn test_json_field_query_with_espaced_dot() {
- assert_eq!(
- extract_query_term_json_path(r#"json.k8s.node.name:hello"#),
- "k8s\u{1}node\u{1}name\0shello"
- );
- assert_eq!(
- extract_query_term_json_path(r"json.k8s\.node\.name:hello"),
- "k8s.node.name\0shello"
- );
- }
-
- #[test]
- fn test_json_field_possibly_a_number() {
- test_parse_query_to_logical_ast_helper(
- "json.titi:5",
- r#"(Term(field=14, type=Json, path=titi, type=I64, 5) Term(field=14, type=Json, path=titi, type=Str, "5"))"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "json.titi:-5",
- r#"(Term(field=14, type=Json, path=titi, type=I64, -5) Term(field=14, type=Json, path=titi, type=Str, "5"))"#, //< Yes this is a bit weird after going through the tokenizer we lose the "-".
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "json.titi:10000000000000000000",
- r#"(Term(field=14, type=Json, path=titi, type=U64, 10000000000000000000) Term(field=14, type=Json, path=titi, type=Str, "10000000000000000000"))"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "json.titi:-5.2",
- r#"(Term(field=14, type=Json, path=titi, type=F64, -5.2) "[(0, Term(field=14, type=Json, path=titi, type=Str, "5")), (1, Term(field=14, type=Json, path=titi, type=Str, "2"))]")"#,
- true,
- );
- }
-
- #[test]
- fn test_json_field_possibly_a_date() {
- test_parse_query_to_logical_ast_helper(
- r#"json.date:"2019-10-12T07:20:50.52Z""#,
- r#"(Term(field=14, type=Json, path=date, type=Date, 2019-10-12T07:20:50Z) "[(0, Term(field=14, type=Json, path=date, type=Str, "2019")), (1, Term(field=14, type=Json, path=date, type=Str, "10")), (2, Term(field=14, type=Json, path=date, type=Str, "12t07")), (3, Term(field=14, type=Json, path=date, type=Str, "20")), (4, Term(field=14, type=Json, path=date, type=Str, "50")), (5, Term(field=14, type=Json, path=date, type=Str, "52z"))]")"#,
- true,
- );
- }
-
- #[test]
- fn test_json_field_possibly_a_bool() {
- test_parse_query_to_logical_ast_helper(
- "json.titi:true",
- r#"(Term(field=14, type=Json, path=titi, type=Bool, true) Term(field=14, type=Json, path=titi, type=Str, "true"))"#,
- true,
- );
- }
-
- #[test]
- fn test_json_field_not_indexed() {
- let error = parse_query_to_logical_ast("json_not_indexed.titi:hello", false).unwrap_err();
- assert!(matches!(error, QueryParserError::FieldNotIndexed(_)));
- }
-
- fn test_query_to_logical_ast_with_default_json(
- query: &str,
- expected: &str,
- default_conjunction: bool,
- ) {
- let mut query_parser = make_query_parser_with_default_fields(&["json"]);
- if default_conjunction {
- query_parser.set_conjunction_by_default();
- }
- let ast = query_parser.parse_query_to_logical_ast(query).unwrap();
- let ast_str = format!("{ast:?}");
- assert_eq!(ast_str, expected);
- }
-
- #[test]
- fn test_json_default() {
- test_query_to_logical_ast_with_default_json(
- "titi:4",
- "(Term(field=14, type=Json, path=titi, type=I64, 4) Term(field=14, type=Json, \
- path=titi, type=Str, \"4\"))",
- false,
- );
- }
-
- #[test]
- fn test_json_default_with_different_field() {
- for conjunction in [false, true] {
- test_query_to_logical_ast_with_default_json(
- "text:4",
- r#"Term(field=1, type=Str, "4")"#,
- conjunction,
- );
- }
- }
-
- #[test]
- fn test_json_default_with_same_field() {
- for conjunction in [false, true] {
- test_query_to_logical_ast_with_default_json(
- "json:4",
- r#"(Term(field=14, type=Json, path=, type=I64, 4) Term(field=14, type=Json, path=, type=Str, "4"))"#,
- conjunction,
- );
- }
- }
-
- #[test]
- fn test_parse_bytes_phrase() {
- test_parse_query_to_logical_ast_helper(
- "bytes:\"YnVidQ==\"",
- "Term(field=12, type=Bytes, [98, 117, 98, 117])",
- false,
- );
- }
-
- #[test]
- fn test_parse_bytes_invalid_base64() {
- let base64_err: QueryParserError =
- parse_query_to_logical_ast("bytes:aa", false).unwrap_err();
- assert!(matches!(base64_err, QueryParserError::ExpectedBase64(_)));
- }
-
- #[test]
- fn test_parse_query_to_ast_ab_c() {
- test_parse_query_to_logical_ast_helper(
- "(+title:a +title:b) title:c",
- r#"((+Term(field=0, type=Str, "a") +Term(field=0, type=Str, "b")) Term(field=0, type=Str, "c"))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "(+title:a +title:b) title:c",
- r#"(+(+Term(field=0, type=Str, "a") +Term(field=0, type=Str, "b")) +Term(field=0, type=Str, "c"))"#,
- true,
- );
- }
-
- #[test]
- pub fn test_parse_query_to_ast_single_term() {
- test_parse_query_to_logical_ast_helper(
- "title:toto",
- r#"Term(field=0, type=Str, "toto")"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "+title:toto",
- r#"Term(field=0, type=Str, "toto")"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "+title:toto -titi",
- r#"(+Term(field=0, type=Str, "toto") -(Term(field=0, type=Str, "titi") Term(field=1, type=Str, "titi")))"#,
- false,
- );
- }
-
- #[test]
- fn test_single_negative_term() {
- assert_matches!(
- parse_query_to_logical_ast("-title:toto", false),
- Err(QueryParserError::AllButQueryForbidden)
- );
- }
-
- #[test]
- pub fn test_parse_query_to_ast_two_terms() {
- test_parse_query_to_logical_ast_helper(
- "title:a b",
- r#"(Term(field=0, type=Str, "a") (Term(field=0, type=Str, "b") Term(field=1, type=Str, "b")))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- r#"title:"a b""#,
- r#""[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]""#,
- false,
- );
- }
- #[test]
- pub fn test_parse_query_all_query() {
- let logical_ast = parse_query_to_logical_ast("*", false).unwrap();
- assert_eq!(format!("{logical_ast:?}"), "*");
- }
-
- #[test]
- pub fn test_parse_query_range_require_a_target_field() {
- let query_parser_error = parse_query_to_logical_ast("[A TO B]", false).err().unwrap();
- assert_eq!(
- query_parser_error.to_string(),
- "Unsupported query: Range query need to target a specific field."
- );
- }
-
- #[test]
- pub fn test_parse_query_to_ast_ranges() {
- test_parse_query_to_logical_ast_helper(
- "title:[a TO b]",
- r#"(Included(Term(field=0, type=Str, "a")) TO Included(Term(field=0, type=Str, "b")))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "title:{titi TO toto}",
- r#"(Excluded(Term(field=0, type=Str, "titi")) TO Excluded(Term(field=0, type=Str, "toto")))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "title:{* TO toto}",
- r#"(Unbounded TO Excluded(Term(field=0, type=Str, "toto")))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "title:{titi TO *}",
- r#"(Excluded(Term(field=0, type=Str, "titi")) TO Unbounded)"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "signed:{-5 TO 3}",
- r#"(Excluded(Term(field=2, type=I64, -5)) TO Excluded(Term(field=2, type=I64, 3)))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "float:{-1.5 TO 1.5}",
- r#"(Excluded(Term(field=10, type=F64, -1.5)) TO Excluded(Term(field=10, type=F64, 1.5)))"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "u64_ff:[7 TO 77]",
- r#"(Included(Term(field=18, type=U64, 7)) TO Included(Term(field=18, type=U64, 77)))"#,
- false,
- );
- }
-
- #[test]
- pub fn test_query_parser_field_does_not_exist() {
- let query_parser = make_query_parser();
- assert_eq!(
- query_parser
- .parse_query("boujou:\"18446744073709551615\"")
- .unwrap_err(),
- QueryParserError::FieldDoesNotExist("boujou".to_string())
- );
- }
-
- #[test]
- pub fn test_query_parser_field_not_indexed() {
- let query_parser = make_query_parser();
- assert_matches!(
- query_parser.parse_query("notindexed_text:\"18446744073709551615\""),
- Err(QueryParserError::FieldNotIndexed(_))
- );
- }
-
- #[test]
- pub fn test_unknown_tokenizer() {
- let mut schema_builder = Schema::builder();
- let text_field_indexing = TextFieldIndexing::default()
- .set_tokenizer("nonexistingtokenizer")
- .set_index_option(IndexRecordOption::Basic);
- let text_options = TextOptions::default().set_indexing_options(text_field_indexing);
- let title = schema_builder.add_text_field("title", text_options);
- let schema = schema_builder.build();
- let default_fields = vec![title];
- let tokenizer_manager = TokenizerManager::default();
- let query_parser = QueryParser::new(schema, default_fields, tokenizer_manager);
-
- assert_matches!(
- query_parser.parse_query("title:\"happy tax payer\""),
- Err(QueryParserError::UnknownTokenizer { .. })
- );
- }
-
- #[test]
- pub fn test_query_parser_no_positions() {
- let mut schema_builder = Schema::builder();
- let text_field_indexing = TextFieldIndexing::default()
- .set_tokenizer("customtokenizer")
- .set_index_option(IndexRecordOption::Basic);
- let text_options = TextOptions::default().set_indexing_options(text_field_indexing);
- let title = schema_builder.add_text_field("title", text_options);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- index
- .tokenizers()
- .register("customtokenizer", SimpleTokenizer::default());
- let query_parser = QueryParser::for_index(&index, vec![title]);
- assert_eq!(
- query_parser.parse_query("title:\"happy tax\"").unwrap_err(),
- QueryParserError::FieldDoesNotHavePositionsIndexed("title".to_string())
- );
- }
-
- #[test]
- pub fn test_query_parser_expected_int() {
- let query_parser = make_query_parser();
- assert_matches!(
- query_parser.parse_query("unsigned:18a"),
- Err(QueryParserError::ExpectedInt(_))
- );
- assert!(query_parser.parse_query("unsigned:\"18\"").is_ok());
- assert_matches!(
- query_parser.parse_query("signed:18b"),
- Err(QueryParserError::ExpectedInt(_))
- );
- assert!(query_parser.parse_query("float:\"1.8\"").is_ok());
- assert_matches!(
- query_parser.parse_query("float:1.8a"),
- Err(QueryParserError::ExpectedFloat(_))
- );
- }
-
- #[test]
- pub fn test_query_parser_expected_bool() {
- let query_parser = make_query_parser();
- assert_matches!(
- query_parser.parse_query("bool:brie"),
- Err(QueryParserError::ExpectedBool(_))
- );
- assert!(query_parser.parse_query("bool:\"true\"").is_ok());
- assert!(query_parser.parse_query("bool:\"false\"").is_ok());
- }
-
- #[test]
- pub fn test_query_parser_expected_date() {
- let query_parser = make_query_parser();
- assert_matches!(
- query_parser.parse_query("date:18a"),
- Err(QueryParserError::DateFormatError(_))
- );
- test_parse_query_to_logical_ast_helper(
- r#"date:"2010-11-21T09:55:06.000000000+02:00""#,
- r#"Term(field=9, type=Date, 2010-11-21T07:55:06Z)"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- r#"date:"1985-04-12T23:20:50.52Z""#,
- r#"Term(field=9, type=Date, 1985-04-12T23:20:50Z)"#,
- true,
- );
- }
-
- #[test]
- pub fn test_query_parser_expected_facet() {
- let query_parser = make_query_parser();
- match query_parser.parse_query("facet:INVALID") {
- Ok(_) => panic!("should never succeed"),
- Err(e) => assert_eq!(
- "The facet field is malformed: Failed to parse the facet string: 'INVALID'",
- format!("{e}")
- ),
- }
- assert!(query_parser.parse_query("facet:\"/foo/bar\"").is_ok());
- }
-
- #[test]
- pub fn test_query_parser_not_empty_but_no_tokens() {
- let query_parser = make_query_parser();
- assert!(query_parser.parse_query(" !, ").is_ok());
- assert!(query_parser.parse_query("with_stop_words:the").is_ok());
- }
-
- #[test]
- pub fn test_parse_query_single_negative_term_through_error() {
- assert_matches!(
- parse_query_to_logical_ast("-title:toto", true),
- Err(QueryParserError::AllButQueryForbidden)
- );
- assert_matches!(
- parse_query_to_logical_ast("-title:toto", false),
- Err(QueryParserError::AllButQueryForbidden)
- );
- }
-
- #[test]
- pub fn test_parse_query_to_ast_conjunction() {
- test_parse_query_to_logical_ast_helper(
- "title:toto",
- r#"Term(field=0, type=Str, "toto")"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "+title:toto",
- r#"Term(field=0, type=Str, "toto")"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "+title:toto -titi",
- r#"(+Term(field=0, type=Str, "toto") -(Term(field=0, type=Str, "titi") Term(field=1, type=Str, "titi")))"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "title:a b",
- r#"(+Term(field=0, type=Str, "a") +(Term(field=0, type=Str, "b") Term(field=1, type=Str, "b")))"#,
- true,
- );
- test_parse_query_to_logical_ast_helper(
- "title:\"a b\"",
- r#""[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]""#,
- true,
- );
- }
-
- #[test]
- pub fn test_query_parser_hyphen() {
- test_parse_query_to_logical_ast_helper(
- "title:www-form-encoded",
- r#""[(0, Term(field=0, type=Str, "www")), (1, Term(field=0, type=Str, "form")), (2, Term(field=0, type=Str, "encoded"))]""#,
- false,
- );
- }
-
- #[test]
- fn test_and_default_regardless_of_default_conjunctive() {
- for &default_conjunction in &[false, true] {
- test_parse_query_to_logical_ast_helper(
- "title:a AND title:b",
- r#"(+Term(field=0, type=Str, "a") +Term(field=0, type=Str, "b"))"#,
- default_conjunction,
- );
- }
- }
-
- #[test]
- fn test_or_default_conjunctive() {
- for &default_conjunction in &[false, true] {
- test_parse_query_to_logical_ast_helper(
- "title:a OR title:b",
- r#"(Term(field=0, type=Str, "a") Term(field=0, type=Str, "b"))"#,
- default_conjunction,
- );
- }
- }
-
- #[test]
- fn test_escaped_field() {
- let mut schema_builder = Schema::builder();
- schema_builder.add_text_field(r"a\.b", STRING);
- let schema = schema_builder.build();
- let query_parser = QueryParser::new(schema, Vec::new(), TokenizerManager::default());
- let query = query_parser.parse_query(r"a\.b:hello").unwrap();
- assert_eq!(
- format!("{query:?}"),
- "TermQuery(Term(field=0, type=Str, \"hello\"))"
- );
- }
-
- #[test]
- fn test_split_full_path() {
- let mut schema_builder = Schema::builder();
- schema_builder.add_text_field("second", STRING);
- schema_builder.add_text_field("first", STRING);
- schema_builder.add_text_field("first.toto", STRING);
- schema_builder.add_text_field("first.toto.titi", STRING);
- schema_builder.add_text_field("third.a.b.c", STRING);
- let schema = schema_builder.build();
- let query_parser =
- QueryParser::new(schema.clone(), Vec::new(), TokenizerManager::default());
- assert_eq!(
- query_parser.split_full_path("first.toto"),
- Some((schema.get_field("first.toto").unwrap(), ""))
- );
- assert_eq!(
- query_parser.split_full_path("first.toto.bubu"),
- Some((schema.get_field("first.toto").unwrap(), "bubu"))
- );
- assert_eq!(
- query_parser.split_full_path("first.toto.titi"),
- Some((schema.get_field("first.toto.titi").unwrap(), ""))
- );
- assert_eq!(
- query_parser.split_full_path("first.titi"),
- Some((schema.get_field("first").unwrap(), "titi"))
- );
- assert_eq!(query_parser.split_full_path("third"), None);
- assert_eq!(query_parser.split_full_path("hello.toto"), None);
- assert_eq!(query_parser.split_full_path(""), None);
- assert_eq!(query_parser.split_full_path("firsty"), None);
- }
-
- #[test]
- pub fn test_phrase_slop() {
- test_parse_query_to_logical_ast_helper(
- "\"a b\"~0",
- r#"("[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]" "[(0, Term(field=1, type=Str, "a")), (1, Term(field=1, type=Str, "b"))]")"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "\"a b\"~2",
- r#"("[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b"))]"~2 "[(0, Term(field=1, type=Str, "a")), (1, Term(field=1, type=Str, "b"))]"~2)"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "title:\"a b~4\"~2",
- r#""[(0, Term(field=0, type=Str, "a")), (1, Term(field=0, type=Str, "b")), (2, Term(field=0, type=Str, "4"))]"~2"#,
- false,
- );
- }
-
- #[test]
- pub fn test_phrase_prefix() {
- test_parse_query_to_logical_ast_helper(
- "\"big bad wo\"*",
- r#"("[(0, Term(field=0, type=Str, "big")), (1, Term(field=0, type=Str, "bad")), (2, Term(field=0, type=Str, "wo"))]"* "[(0, Term(field=1, type=Str, "big")), (1, Term(field=1, type=Str, "bad")), (2, Term(field=1, type=Str, "wo"))]"*)"#,
- false,
- );
-
- let query_parser = make_query_parser();
- let query = query_parser.parse_query("\"big bad wo\"*").unwrap();
- assert_eq!(
- format!("{query:?}"),
- "BooleanQuery { subqueries: [(Should, PhrasePrefixQuery { field: Field(0), \
- phrase_terms: [(0, Term(field=0, type=Str, \"big\")), (1, Term(field=0, type=Str, \
- \"bad\"))], prefix: (2, Term(field=0, type=Str, \"wo\")), max_expansions: 50 }), \
- (Should, PhrasePrefixQuery { field: Field(1), phrase_terms: [(0, Term(field=1, \
- type=Str, \"big\")), (1, Term(field=1, type=Str, \"bad\"))], prefix: (2, \
- Term(field=1, type=Str, \"wo\")), max_expansions: 50 })] }"
- );
- }
-
- #[test]
- pub fn test_phrase_prefix_too_short() {
- let err = parse_query_to_logical_ast("\"wo\"*", true).unwrap_err();
- assert_eq!(
- err,
- QueryParserError::PhrasePrefixRequiresAtLeastTwoTerms {
- phrase: "wo".to_owned(),
- tokenizer: "default".to_owned()
- }
- );
-
- let err = parse_query_to_logical_ast("\"\"*", true).unwrap_err();
- assert_eq!(
- err,
- QueryParserError::PhrasePrefixRequiresAtLeastTwoTerms {
- phrase: "".to_owned(),
- tokenizer: "default".to_owned()
- }
- );
- }
-
- #[test]
- pub fn test_term_set_query() {
- test_parse_query_to_logical_ast_helper(
- "title: IN [a b cd]",
- r#"IN [Term(field=0, type=Str, "a"), Term(field=0, type=Str, "b"), Term(field=0, type=Str, "cd")]"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "bytes: IN [AA== ABA= ABCD]",
- r#"IN [Term(field=12, type=Bytes, [0]), Term(field=12, type=Bytes, [0, 16]), Term(field=12, type=Bytes, [0, 16, 131])]"#,
- false,
- );
- test_parse_query_to_logical_ast_helper(
- "signed: IN [1 2 -3]",
- r#"IN [Term(field=2, type=I64, 1), Term(field=2, type=I64, 2), Term(field=2, type=I64, -3)]"#,
- false,
- );
-
- test_parse_query_to_logical_ast_helper(
- "float: IN [1.1 2.2 -3.3]",
- r#"IN [Term(field=10, type=F64, 1.1), Term(field=10, type=F64, 2.2), Term(field=10, type=F64, -3.3)]"#,
- false,
- );
- }
-
- #[test]
- pub fn test_set_field_fuzzy() {
- {
- let mut query_parser = make_query_parser();
- query_parser.set_field_fuzzy(
- query_parser.schema.get_field("title").unwrap(),
- false,
- 1,
- true,
- );
- let query = query_parser.parse_query("abc").unwrap();
- assert_eq!(
- format!("{query:?}"),
- "BooleanQuery { subqueries: [(Should, FuzzyTermQuery { term: Term(field=0, \
- type=Str, \"abc\"), distance: 1, transposition_cost_one: true, prefix: false }), \
- (Should, TermQuery(Term(field=1, type=Str, \"abc\")))] }"
- );
- }
-
- {
- let mut query_parser = make_query_parser();
- query_parser.set_field_fuzzy(
- query_parser.schema.get_field("text").unwrap(),
- true,
- 2,
- false,
- );
- let query = query_parser.parse_query("abc").unwrap();
- assert_eq!(
- format!("{query:?}"),
- "BooleanQuery { subqueries: [(Should, TermQuery(Term(field=0, type=Str, \
- \"abc\"))), (Should, FuzzyTermQuery { term: Term(field=1, type=Str, \"abc\"), \
- distance: 2, transposition_cost_one: false, prefix: true })] }"
- );
- }
- }
-}
diff --git a/crates/tantivy/src/query/range_query/mod.rs b/crates/tantivy/src/query/range_query/mod.rs
index 75b47626..6214c1e5 100644
--- a/crates/tantivy/src/query/range_query/mod.rs
+++ b/crates/tantivy/src/query/range_query/mod.rs
@@ -4,11 +4,9 @@ use crate::schema::Type;
mod fast_field_range_query;
mod range_query;
-mod range_query_ip_fastfield;
mod range_query_u64_fastfield;
pub use self::range_query::RangeQuery;
-pub use self::range_query_ip_fastfield::IPFastFieldRangeWeight;
pub use self::range_query_u64_fastfield::FastFieldRangeWeight;
// TODO is this correct?
@@ -16,7 +14,7 @@ pub(crate) fn is_type_valid_for_fastfield_range_query(typ: Type) -> bool {
match typ {
Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,
Type::IpAddr => true,
- Type::Str | Type::Facet | Type::Bytes | Type::Json => false,
+ Type::Str | Type::Bytes | Type::Json => false,
}
}
@@ -28,15 +26,3 @@ fn map_bound<TFrom, TTo>(bound: &Bound<TFrom>, transform: impl Fn(&TFrom) -> TTo
Unbounded => Unbounded,
}
}
-
-fn map_bound_res<TFrom, TTo, Err>(
- bound: &Bound<TFrom>,
- transform: impl Fn(&TFrom) -> Result<TTo, Err>,
-) -> Result<Bound<TTo>, Err> {
- use self::Bound::*;
- Ok(match bound {
- Excluded(ref from_val) => Excluded(transform(from_val)?),
- Included(ref from_val) => Included(transform(from_val)?),
- Unbounded => Unbounded,
- })
-}
diff --git a/crates/tantivy/src/query/range_query/range_query.rs b/crates/tantivy/src/query/range_query/range_query.rs
index 2b46a30e..351c1888 100644
--- a/crates/tantivy/src/query/range_query/range_query.rs
+++ b/crates/tantivy/src/query/range_query/range_query.rs
@@ -2,7 +2,6 @@ use std::io;
use std::net::Ipv6Addr;
use std::ops::{Bound, Range};
-use crate::columnar::MonotonicallyMappableToU128;
use crate::common::{BinarySerializable, BitSet};
use super::map_bound;
@@ -10,8 +9,7 @@ use super::range_query_u64_fastfield::FastFieldRangeWeight;
use crate::error::TantivyError;
use crate::index::SegmentReader;
use crate::query::explanation::does_not_match;
-use crate::query::range_query::range_query_ip_fastfield::IPFastFieldRangeWeight;
-use crate::query::range_query::{is_type_valid_for_fastfield_range_query, map_bound_res};
+use crate::query::range_query::is_type_valid_for_fastfield_range_query;
use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};
use crate::schema::{Field, IndexRecordOption, Term, Type};
use crate::termdict::{TermDictionary, TermStreamer};
@@ -324,7 +322,7 @@ pub(crate) fn maps_to_u64_fastfield(typ: Type) -> bool {
match typ {
Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,
Type::IpAddr => false,
- Type::Str | Type::Facet | Type::Bytes | Type::Json => false,
+ Type::Str | Type::Bytes | Type::Json => false,
}
}
@@ -345,39 +343,20 @@ impl Query for RangeQuery {
}
if field_type.is_fast() && is_type_valid_for_fastfield_range_query(self.value_type) {
- if field_type.is_ip_addr() {
- let parse_ip_from_bytes = |data: &Vec<u8>| {
- let ip_u128_bytes: [u8; 16] = data.as_slice().try_into().map_err(|_| {
- crate::TantivyError::InvalidArgument(
- "Expected 8 bytes for ip address".to_string(),
- )
- })?;
- let ip_u128 = u128::from_be_bytes(ip_u128_bytes);
- crate::Result::<Ipv6Addr>::Ok(Ipv6Addr::from_u128(ip_u128))
- };
- let lower_bound = map_bound_res(&self.lower_bound, parse_ip_from_bytes)?;
- let upper_bound = map_bound_res(&self.upper_bound, parse_ip_from_bytes)?;
- Ok(Box::new(IPFastFieldRangeWeight::new(
- self.field.to_string(),
- lower_bound,
- upper_bound,
- )))
- } else {
- // We run the range query on u64 value space for performance reasons and simpicity
- // assert the type maps to u64
- assert!(maps_to_u64_fastfield(self.value_type));
- let parse_from_bytes = |data: &Vec<u8>| {
- u64::from_be(BinarySerializable::deserialize(&mut &data[..]).unwrap())
- };
-
- let lower_bound = map_bound(&self.lower_bound, parse_from_bytes);
- let upper_bound = map_bound(&self.upper_bound, parse_from_bytes);
- Ok(Box::new(FastFieldRangeWeight::new_u64_lenient(
- self.field.to_string(),
- lower_bound,
- upper_bound,
- )))
- }
+ // We run the range query on u64 value space for performance reasons and simpicity
+ // assert the type maps to u64
+ assert!(maps_to_u64_fastfield(self.value_type));
+ let parse_from_bytes = |data: &Vec<u8>| {
+ u64::from_be(BinarySerializable::deserialize(&mut &data[..]).unwrap())
+ };
+
+ let lower_bound = map_bound(&self.lower_bound, parse_from_bytes);
+ let upper_bound = map_bound(&self.upper_bound, parse_from_bytes);
+ Ok(Box::new(FastFieldRangeWeight::new_u64_lenient(
+ self.field.to_string(),
+ lower_bound,
+ upper_bound,
+ )))
} else {
Ok(Box::new(RangeWeight {
field: self.field.to_string(),
@@ -695,165 +674,4 @@ mod tests {
assert_eq!(top_docs.len(), 1);
Ok(())
}
-
- #[test]
- fn search_ip_range_test_posting_list() {
- search_ip_range_test_opt(false);
- }
-
- #[test]
- fn search_ip_range_test() {
- search_ip_range_test_opt(true);
- }
-
- fn search_ip_range_test_opt(with_fast_field: bool) {
- let mut schema_builder = Schema::builder();
- let ip_field = if with_fast_field {
- schema_builder.add_ip_addr_field("ip", INDEXED | STORED | FAST)
- } else {
- schema_builder.add_ip_addr_field("ip", INDEXED | STORED)
- };
- let text_field = schema_builder.add_text_field("text", TEXT | STORED);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let ip_addr_1 = IpAddr::from_str("127.0.0.10").unwrap().into_ipv6_addr();
- let ip_addr_2 = IpAddr::from_str("127.0.0.20").unwrap().into_ipv6_addr();
-
- {
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- for _ in 0..1_000 {
- index_writer
- .add_document(doc!(
- ip_field => ip_addr_1,
- text_field => "BLUBBER"
- ))
- .unwrap();
- }
- for _ in 0..1_000 {
- index_writer
- .add_document(doc!(
- ip_field => ip_addr_2,
- text_field => "BLOBBER"
- ))
- .unwrap();
- }
- index_writer.commit().unwrap();
- }
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- assert_eq!(searcher.segment_readers().len(), 1);
-
- let get_num_hits = |query| {
- let (_top_docs, count) = searcher
- .search(&query, &(TopDocs::with_limit(10), Count))
- .unwrap();
- count
- };
- let query_from_text = |text: &str| {
- QueryParser::for_index(&index, vec![])
- .parse_query(text)
- .unwrap()
- };
-
- // Inclusive range
- assert_eq!(
- get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20]")),
- 2000
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:[127.0.0.10 TO 127.0.0.20]")),
- 2000
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.20]")),
- 1000
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.19]")),
- 0
- );
-
- assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.11 TO *]")), 1000);
- assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.21 TO *]")), 0);
- assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.9]")), 0);
- assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.10]")), 1000);
-
- // Exclusive range
- assert_eq!(
- get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20}")),
- 1000
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.21}")),
- 2000
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:{127.0.0.10 TO 127.0.0.20}")),
- 0
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.20}")),
- 0
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.19}")),
- 0
- );
-
- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.11 TO *}")), 1000);
- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.10 TO *}")), 1000);
- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.21 TO *}")), 0);
- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.20 TO *}")), 0);
- assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.19 TO *}")), 1000);
- assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.9}")), 0);
- assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.10}")), 0);
- assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.11}")), 1000);
-
- // Inclusive/Exclusive range
- assert_eq!(
- get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20}")),
- 1000
- );
-
- assert_eq!(
- get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20]")),
- 2000
- );
-
- // Intersection
- assert_eq!(
- get_num_hits(query_from_text(
- "text:BLUBBER AND ip:[127.0.0.10 TO 127.0.0.10]"
- )),
- 1000
- );
-
- assert_eq!(
- get_num_hits(query_from_text(
- "text:BLOBBER AND ip:[127.0.0.10 TO 127.0.0.10]"
- )),
- 0
- );
-
- assert_eq!(
- get_num_hits(query_from_text(
- "text:BLOBBER AND ip:[127.0.0.20 TO 127.0.0.20]"
- )),
- 1000
- );
-
- assert_eq!(
- get_num_hits(query_from_text(
- "text:BLUBBER AND ip:[127.0.0.20 TO 127.0.0.20]"
- )),
- 0
- );
- }
}
diff --git a/crates/tantivy/src/query/range_query/range_query_ip_fastfield.rs b/crates/tantivy/src/query/range_query/range_query_ip_fastfield.rs
deleted file mode 100644
index 60b8bcfa..00000000
--- a/crates/tantivy/src/query/range_query/range_query_ip_fastfield.rs
+++ /dev/null
@@ -1,273 +0,0 @@
-//! IP Fastfields support efficient scanning for range queries.
-//! We use this variant only if the fastfield exists, otherwise the default in `range_query` is
-//! used, which uses the term dictionary + postings.
-
-use std::net::Ipv6Addr;
-use std::ops::{Bound, RangeInclusive};
-
-use crate::columnar::{Column, MonotonicallyMappableToU128};
-
-use crate::query::range_query::fast_field_range_query::RangeDocSet;
-use crate::query::{ConstScorer, EmptyScorer, Explanation, Scorer, Weight};
-use crate::{DocId, DocSet, Score, SegmentReader, TantivyError};
-
-/// `IPFastFieldRangeWeight` uses the ip address fast field to execute range queries.
-pub struct IPFastFieldRangeWeight {
- field: String,
- lower_bound: Bound<Ipv6Addr>,
- upper_bound: Bound<Ipv6Addr>,
-}
-
-impl IPFastFieldRangeWeight {
- /// Creates a new IPFastFieldRangeWeight.
- pub fn new(field: String, lower_bound: Bound<Ipv6Addr>, upper_bound: Bound<Ipv6Addr>) -> Self {
- Self {
- field,
- lower_bound,
- upper_bound,
- }
- }
-}
-
-impl Weight for IPFastFieldRangeWeight {
- fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
- let Some(ip_addr_column): Option<Column<Ipv6Addr>> =
- reader.fast_fields().column_opt(&self.field)?
- else {
- return Ok(Box::new(EmptyScorer));
- };
- let value_range = bound_to_value_range(
- &self.lower_bound,
- &self.upper_bound,
- ip_addr_column.min_value(),
- ip_addr_column.max_value(),
- );
- let docset = RangeDocSet::new(value_range, ip_addr_column);
- Ok(Box::new(ConstScorer::new(docset, boost)))
- }
-
- fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
- let mut scorer = self.scorer(reader, 1.0)?;
- if scorer.seek(doc) != doc {
- return Err(TantivyError::InvalidArgument(format!(
- "Document #({doc}) does not match"
- )));
- }
- let explanation = Explanation::new("Const", scorer.score());
- Ok(explanation)
- }
-}
-
-fn bound_to_value_range(
- lower_bound: &Bound<Ipv6Addr>,
- upper_bound: &Bound<Ipv6Addr>,
- min_value: Ipv6Addr,
- max_value: Ipv6Addr,
-) -> RangeInclusive<Ipv6Addr> {
- let start_value = match lower_bound {
- Bound::Included(ip_addr) => *ip_addr,
- Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() + 1),
- Bound::Unbounded => min_value,
- };
-
- let end_value = match upper_bound {
- Bound::Included(ip_addr) => *ip_addr,
- Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() - 1),
- Bound::Unbounded => max_value,
- };
- start_value..=end_value
-}
-
-#[cfg(test)]
-pub mod tests {
- use proptest::prelude::ProptestConfig;
- use proptest::strategy::Strategy;
- use proptest::{prop_oneof, proptest};
-
- use super::*;
- use crate::collector::Count;
- use crate::query::QueryParser;
- use crate::schema::{Schema, FAST, INDEXED, STORED, STRING};
- use crate::{Index, IndexWriter};
-
- #[derive(Clone, Debug)]
- pub struct Doc {
- pub id: String,
- pub ip: Ipv6Addr,
- }
-
- fn operation_strategy() -> impl Strategy<Value = Doc> {
- prop_oneof![
- (0u64..10_000u64).prop_map(doc_from_id_1),
- (1u64..10_000u64).prop_map(doc_from_id_2),
- ]
- }
-
- pub fn doc_from_id_1(id: u64) -> Doc {
- let id = id * 1000;
- Doc {
- // ip != id
- id: id.to_string(),
- ip: Ipv6Addr::from_u128(id as u128),
- }
- }
- fn doc_from_id_2(id: u64) -> Doc {
- let id = id * 1000;
- Doc {
- // ip != id
- id: (id - 1).to_string(),
- ip: Ipv6Addr::from_u128(id as u128),
- }
- }
-
- proptest! {
- #![proptest_config(ProptestConfig::with_cases(10))]
- #[test]
- fn test_ip_range_for_docs_prop(ops in proptest::collection::vec(operation_strategy(), 1..1000)) {
- assert!(test_ip_range_for_docs(&ops).is_ok());
- }
- }
-
- #[test]
- fn test_ip_range_regression1() {
- let ops = &[doc_from_id_1(0)];
- assert!(test_ip_range_for_docs(ops).is_ok());
- }
-
- #[test]
- fn test_ip_range_regression2() {
- let ops = &[
- doc_from_id_1(52),
- doc_from_id_1(63),
- doc_from_id_1(12),
- doc_from_id_2(91),
- doc_from_id_2(33),
- ];
- assert!(test_ip_range_for_docs(ops).is_ok());
- }
-
- #[test]
- fn test_ip_range_regression3() {
- let ops = &[doc_from_id_1(1), doc_from_id_1(2), doc_from_id_1(3)];
- assert!(test_ip_range_for_docs(ops).is_ok());
- }
-
- #[test]
- fn test_ip_range_regression3_simple() {
- let mut schema_builder = Schema::builder();
- let ips_field = schema_builder.add_ip_addr_field("ips", FAST | INDEXED);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut writer: IndexWriter = index.writer_for_tests().unwrap();
- let ip_addrs: Vec<Ipv6Addr> = [1000, 2000, 3000]
- .into_iter()
- .map(Ipv6Addr::from_u128)
- .collect();
- for &ip_addr in &ip_addrs {
- writer
- .add_document(doc!(ips_field=>ip_addr, ips_field=>ip_addr))
- .unwrap();
- }
- writer.commit().unwrap();
- let searcher = index.reader().unwrap().searcher();
- let range_weight = IPFastFieldRangeWeight {
- field: "ips".to_string(),
- lower_bound: Bound::Included(ip_addrs[1]),
- upper_bound: Bound::Included(ip_addrs[2]),
- };
- let count = range_weight.count(searcher.segment_reader(0)).unwrap();
- assert_eq!(count, 2);
- }
-
- pub fn create_index_from_docs(docs: &[Doc]) -> Index {
- let mut schema_builder = Schema::builder();
- let ip_field = schema_builder.add_ip_addr_field("ip", STORED | FAST);
- let ips_field = schema_builder.add_ip_addr_field("ips", FAST | INDEXED);
- let text_field = schema_builder.add_text_field("id", STRING | STORED);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- {
- let mut index_writer = index.writer_with_num_threads(2, 60_000_000).unwrap();
- for doc in docs.iter() {
- index_writer
- .add_document(doc!(
- ips_field => doc.ip,
- ips_field => doc.ip,
- ip_field => doc.ip,
- text_field => doc.id.to_string(),
- ))
- .unwrap();
- }
-
- index_writer.commit().unwrap();
- }
- index
- }
-
- fn test_ip_range_for_docs(docs: &[Doc]) -> crate::Result<()> {
- let index = create_index_from_docs(docs);
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
-
- let get_num_hits = |query| searcher.search(&query, &Count).unwrap();
- let query_from_text = |text: &str| {
- QueryParser::for_index(&index, vec![])
- .parse_query(text)
- .unwrap()
- };
-
- let gen_query_inclusive = |field: &str, ip_range: &RangeInclusive<Ipv6Addr>| {
- format!("{field}:[{} TO {}]", ip_range.start(), ip_range.end())
- };
-
- let test_sample = |sample_docs: &[Doc]| {
- let mut ips: Vec<Ipv6Addr> = sample_docs.iter().map(|doc| doc.ip).collect();
- ips.sort();
- let ip_range = ips[0]..=ips[1];
- let expected_num_hits = docs
- .iter()
- .filter(|doc| (ips[0]..=ips[1]).contains(&doc.ip))
- .count();
-
- let query = gen_query_inclusive("ip", &ip_range);
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
-
- let query = gen_query_inclusive("ips", &ip_range);
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
-
- // Intersection search
- let id_filter = sample_docs[0].id.to_string();
- let expected_num_hits = docs
- .iter()
- .filter(|doc| ip_range.contains(&doc.ip) && doc.id == id_filter)
- .count();
- let query = format!(
- "{} AND id:{}",
- gen_query_inclusive("ip", &ip_range),
- &id_filter
- );
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
-
- // Intersection search on multivalue ip field
- let id_filter = sample_docs[0].id.to_string();
- let query = format!(
- "{} AND id:{}",
- gen_query_inclusive("ips", &ip_range),
- &id_filter
- );
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
- };
-
- test_sample(&[docs[0].clone(), docs[0].clone()]);
- if docs.len() > 1 {
- test_sample(&[docs[0].clone(), docs[1].clone()]);
- test_sample(&[docs[1].clone(), docs[1].clone()]);
- }
- if docs.len() > 2 {
- test_sample(&[docs[1].clone(), docs[2].clone()]);
- }
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs b/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs
index 2e9981bb..13c8ca4a 100644
--- a/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs
+++ b/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs
@@ -140,7 +140,7 @@ pub mod tests {
use crate::collector::Count;
use crate::query::range_query::range_query_u64_fastfield::FastFieldRangeWeight;
use crate::query::{QueryParser, Weight};
- use crate::schema::{NumericOptions, Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING};
+ use crate::schema::{Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING};
use crate::{Index, IndexWriter, TERMINATED};
#[derive(Clone, Debug)]
@@ -179,12 +179,6 @@ pub mod tests {
}
}
- #[test]
- fn range_regression1_test() {
- let ops = vec![doc_from_id_1(0)];
- assert!(test_id_range_for_docs(ops).is_ok());
- }
-
#[test]
fn test_range_regression2() {
let ops = vec![
@@ -197,12 +191,6 @@ pub mod tests {
assert!(test_id_range_for_docs(ops).is_ok());
}
- #[test]
- fn test_range_regression3() {
- let ops = vec![doc_from_id_1(9), doc_from_id_1(0), doc_from_id_1(13)];
- assert!(test_id_range_for_docs(ops).is_ok());
- }
-
#[test]
fn test_range_regression_simplified() {
let mut schema_builder = SchemaBuilder::new();
@@ -225,34 +213,16 @@ pub mod tests {
}
#[test]
- fn range_regression3_test() {
+ fn test_range_regression3() {
let ops = vec![doc_from_id_1(1), doc_from_id_1(2), doc_from_id_1(3)];
assert!(test_id_range_for_docs(ops).is_ok());
}
- #[test]
- fn range_regression4_test() {
- let ops = vec![doc_from_id_2(100)];
- assert!(test_id_range_for_docs(ops).is_ok());
- }
-
pub fn create_index_from_docs(docs: &[Doc]) -> Index {
let mut schema_builder = Schema::builder();
let id_u64_field = schema_builder.add_u64_field("id", INDEXED | STORED | FAST);
- let ids_u64_field =
- schema_builder.add_u64_field("ids", NumericOptions::default().set_fast().set_indexed());
-
let id_f64_field = schema_builder.add_f64_field("id_f64", INDEXED | STORED | FAST);
- let ids_f64_field = schema_builder.add_f64_field(
- "ids_f64",
- NumericOptions::default().set_fast().set_indexed(),
- );
-
let id_i64_field = schema_builder.add_i64_field("id_i64", INDEXED | STORED | FAST);
- let ids_i64_field = schema_builder.add_i64_field(
- "ids_i64",
- NumericOptions::default().set_fast().set_indexed(),
- );
let text_field = schema_builder.add_text_field("id_name", STRING | STORED);
let schema = schema_builder.build();
@@ -263,12 +233,6 @@ pub mod tests {
for doc in docs.iter() {
index_writer
.add_document(doc!(
- ids_i64_field => doc.id as i64,
- ids_i64_field => doc.id as i64,
- ids_f64_field => doc.id as f64,
- ids_f64_field => doc.id as f64,
- ids_u64_field => doc.id,
- ids_u64_field => doc.id,
id_u64_field => doc.id,
id_f64_field => doc.id as f64,
id_i64_field => doc.id as i64,
@@ -314,9 +278,6 @@ pub mod tests {
let query = gen_query_inclusive("id", ids[0]..=ids[1]);
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
- let query = gen_query_inclusive("ids", ids[0]..=ids[1]);
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
-
// Exclusive range
let expected_num_hits = docs
.iter()
@@ -328,9 +289,6 @@ pub mod tests {
let query = gen_query_exclusive("id", ids[0]..=ids[1]);
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
- let query = gen_query_exclusive("ids", ids[0]..=ids[1]);
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
-
// Intersection search
let id_filter = sample_docs[0].id_name.to_string();
let expected_num_hits = docs
@@ -355,27 +313,6 @@ pub mod tests {
&id_filter
);
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
-
- // Intersection search on multivalue id field
- let id_filter = sample_docs[0].id_name.to_string();
- let query = format!(
- "{} AND id_name:{}",
- gen_query_inclusive("ids", ids[0]..=ids[1]),
- &id_filter
- );
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
- let query = format!(
- "{} AND id_name:{}",
- gen_query_inclusive("ids_f64", ids[0]..=ids[1]),
- &id_filter
- );
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
- let query = format!(
- "{} AND id_name:{}",
- gen_query_inclusive("ids_i64", ids[0]..=ids[1]),
- &id_filter
- );
- assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
};
test_sample(vec![docs[0].clone(), docs[0].clone()]);
diff --git a/crates/tantivy/src/schema/document/de.rs b/crates/tantivy/src/schema/document/de.rs
index a7d5ef8c..22c1c8b1 100644
--- a/crates/tantivy/src/schema/document/de.rs
+++ b/crates/tantivy/src/schema/document/de.rs
@@ -21,7 +21,7 @@ use crate::common::{u64_to_f64, BinarySerializable, DateTime, VInt};
use super::se::BinaryObjectSerializer;
use super::{OwnedValue, Value};
use crate::schema::document::type_codes;
-use crate::schema::{Facet, Field};
+use crate::schema::Field;
use crate::tokenizer::PreTokenizedString;
#[derive(Debug, thiserror::Error, Clone)]
@@ -112,9 +112,6 @@ pub trait ValueDeserializer<'de> {
/// Attempts to deserialize a datetime value from the deserializer.
fn deserialize_datetime(self) -> Result<DateTime, DeserializeError>;
- /// Attempts to deserialize a facet value from the deserializer.
- fn deserialize_facet(self) -> Result<Facet, DeserializeError>;
-
/// Attempts to deserialize a bytes value from the deserializer.
fn deserialize_bytes(self) -> Result<Vec<u8>, DeserializeError>;
@@ -148,8 +145,6 @@ pub enum ValueType {
F64,
/// A datetime value.
DateTime,
- /// A facet value.
- Facet,
/// A bytes value.
Bytes,
/// A IP address value.
@@ -224,12 +219,6 @@ pub trait ValueVisitor {
Err(DeserializeError::UnsupportedType(ValueType::IpAddr))
}
- #[inline]
- /// Called when the deserializer visits a facet value.
- fn visit_facet(&self, _val: Facet) -> Result<Self::Value, DeserializeError> {
- Err(DeserializeError::UnsupportedType(ValueType::Facet))
- }
-
#[inline]
/// Called when the deserializer visits a bytes value.
fn visit_bytes(&self, _val: Vec<u8>) -> Result<Self::Value, DeserializeError> {
@@ -370,7 +359,6 @@ where
type_codes::F64_CODE => ValueType::F64,
type_codes::BOOL_CODE => ValueType::Bool,
type_codes::DATE_CODE => ValueType::DateTime,
- type_codes::HIERARCHICAL_FACET_CODE => ValueType::Facet,
type_codes::BYTES_CODE => ValueType::Bytes,
type_codes::EXT_CODE => {
let ext_type_code = <u8 as BinarySerializable>::deserialize(reader)?;
@@ -452,11 +440,6 @@ where
<DateTime as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
}
- fn deserialize_facet(self) -> Result<Facet, DeserializeError> {
- self.validate_type(ValueType::Facet)?;
- <Facet as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
- }
-
fn deserialize_bytes(self) -> Result<Vec<u8>, DeserializeError> {
self.validate_type(ValueType::Bytes)?;
<Vec<u8> as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
@@ -506,10 +489,6 @@ where
let val = self.deserialize_datetime()?;
visitor.visit_datetime(val)
}
- ValueType::Facet => {
- let val = self.deserialize_facet()?;
- visitor.visit_facet(val)
- }
ValueType::Bytes => {
let val = self.deserialize_bytes()?;
visitor.visit_bytes(val)
@@ -726,16 +705,6 @@ impl ValueDeserialize for Ipv6Addr {
}
}
-impl ValueDeserialize for Facet {
- #[inline]
- fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
- where
- D: ValueDeserializer<'de>,
- {
- deserializer.deserialize_facet()
- }
-}
-
impl ValueDeserialize for Vec<u8> {
#[inline]
fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
@@ -923,11 +892,6 @@ mod tests {
crate::schema::OwnedValue::Date(DateTime::from_timestamp_micros(100))
);
- let facet = Facet::from_text("/hello/world").unwrap();
- let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());
- let value = deserialize_value(result);
- assert_eq!(value, crate::schema::OwnedValue::Facet(facet));
-
let pre_tok_str = PreTokenizedString {
text: "hello, world".to_string(),
tokens: vec![Token::default(), Token::default()],
diff --git a/crates/tantivy/src/schema/document/default_document.rs b/crates/tantivy/src/schema/document/default_document.rs
index 40df237f..119c50a2 100644
--- a/crates/tantivy/src/schema/document/default_document.rs
+++ b/crates/tantivy/src/schema/document/default_document.rs
@@ -14,7 +14,7 @@ use crate::schema::document::{
DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
};
use crate::schema::field_type::ValueParsingError;
-use crate::schema::{Facet, Field, NamedFieldDocument, OwnedValue, Schema};
+use crate::schema::{Field, NamedFieldDocument, OwnedValue, Schema};
use crate::tokenizer::PreTokenizedString;
#[repr(packed)]
@@ -68,15 +68,6 @@ impl CompactDoc {
self.field_values.len()
}
- /// Adding a facet to the document.
- pub fn add_facet<F>(&mut self, field: Field, path: F)
- where
- Facet: From<F>,
- {
- let facet = Facet::from(path);
- self.add_leaf_field_value(field, ReferenceValueLeaf::Facet(facet.encoded_str()));
- }
-
/// Add a text field.
pub fn add_text<S: AsRef<str>>(&mut self, field: Field, text: S) {
self.add_leaf_field_value(field, ReferenceValueLeaf::Str(text.as_ref()));
@@ -245,9 +236,6 @@ impl CompactDoc {
ReferenceValueLeaf::Str(bytes) => {
write_bytes_into(&mut self.node_data, bytes.as_bytes())
}
- ReferenceValueLeaf::Facet(bytes) => {
- write_bytes_into(&mut self.node_data, bytes.as_bytes())
- }
ReferenceValueLeaf::Bytes(bytes) => write_bytes_into(&mut self.node_data, bytes),
ReferenceValueLeaf::U64(num) => write_into(&mut self.node_data, num),
ReferenceValueLeaf::I64(num) => write_into(&mut self.node_data, num),
@@ -430,10 +418,6 @@ impl<'a> CompactDocValue<'a> {
let str_ref = self.container.extract_str(addr);
Ok(ReferenceValueLeaf::Str(str_ref).into())
}
- ValueType::Facet => {
- let str_ref = self.container.extract_str(addr);
- Ok(ReferenceValueLeaf::Facet(str_ref).into())
- }
ValueType::Bytes => {
let data = self.container.extract_bytes(addr);
Ok(ReferenceValueLeaf::Bytes(data).into())
@@ -534,8 +518,6 @@ pub enum ValueType {
F64 = 4,
/// Date/time with nanoseconds precision
Date = 5,
- /// Facet
- Facet = 6,
/// Arbitrarily sized byte array
Bytes = 7,
/// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
@@ -591,7 +573,6 @@ impl<'a> From<&ReferenceValueLeaf<'a>> for ValueType {
ReferenceValueLeaf::Date(_) => ValueType::Date,
ReferenceValueLeaf::IpAddr(_) => ValueType::IpAddr,
ReferenceValueLeaf::PreTokStr(_) => ValueType::PreTokStr,
- ReferenceValueLeaf::Facet(_) => ValueType::Facet,
ReferenceValueLeaf::Bytes(_) => ValueType::Bytes,
}
}
diff --git a/crates/tantivy/src/schema/document/existing_type_impls.rs b/crates/tantivy/src/schema/document/existing_type_impls.rs
index d3187f6d..45d00974 100644
--- a/crates/tantivy/src/schema/document/existing_type_impls.rs
+++ b/crates/tantivy/src/schema/document/existing_type_impls.rs
@@ -13,7 +13,6 @@ use serde_json::Number;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime;
-use super::facet::Facet;
use super::ReferenceValueLeaf;
use crate::schema::document::{
ArrayAccess, DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
@@ -83,15 +82,6 @@ impl<'a> Value<'a> for &'a String {
}
}
-impl<'a> Value<'a> for &'a Facet {
- type ArrayIter = Empty<&'a Facet>;
- type ObjectIter = Empty<(&'a str, &'a Facet)>;
- #[inline]
- fn as_value(&self) -> ReferenceValue<'a, Self> {
- ReferenceValue::Leaf(ReferenceValueLeaf::Facet(self.encoded_str()))
- }
-}
-
impl<'a> Value<'a> for &'a u64 {
type ArrayIter = Empty<&'a u64>;
type ObjectIter = Empty<(&'a str, &'a u64)>;
diff --git a/crates/tantivy/src/schema/document/mod.rs b/crates/tantivy/src/schema/document/mod.rs
index 91ce894c..71275bad 100644
--- a/crates/tantivy/src/schema/document/mod.rs
+++ b/crates/tantivy/src/schema/document/mod.rs
@@ -184,11 +184,13 @@ use super::*;
pub trait Document: Send + Sync + 'static {
/// The value of the field.
type Value<'a>: Value<'a> + Clone
- where Self: 'a;
+ where
+ Self: 'a;
/// The iterator over all of the fields and values within the doc.
type FieldsValuesIter<'a>: Iterator<Item = (Field, Self::Value<'a>)>
- where Self: 'a;
+ where
+ Self: 'a;
/// Get an iterator iterating over all fields and values in a document.
fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_>;
@@ -255,7 +257,6 @@ pub(crate) mod type_codes {
pub const TEXT_CODE: u8 = 0;
pub const U64_CODE: u8 = 1;
pub const I64_CODE: u8 = 2;
- pub const HIERARCHICAL_FACET_CODE: u8 = 3;
pub const BYTES_CODE: u8 = 4;
pub const DATE_CODE: u8 = 5;
pub const F64_CODE: u8 = 6;
diff --git a/crates/tantivy/src/schema/document/owned_value.rs b/crates/tantivy/src/schema/document/owned_value.rs
index 9fbf1f8c..95240af2 100644
--- a/crates/tantivy/src/schema/document/owned_value.rs
+++ b/crates/tantivy/src/schema/document/owned_value.rs
@@ -14,7 +14,6 @@ use crate::schema::document::{
ArrayAccess, DeserializeError, ObjectAccess, ReferenceValue, Value, ValueDeserialize,
ValueDeserializer, ValueVisitor,
};
-use crate::schema::Facet;
use crate::tokenizer::PreTokenizedString;
use crate::DateTime;
@@ -39,8 +38,6 @@ pub enum OwnedValue {
Bool(bool),
/// Date/time with nanoseconds precision
Date(DateTime),
- /// Facet
- Facet(Facet),
/// Arbitrarily sized byte array
Bytes(Vec<u8>),
/// A set of values.
@@ -72,7 +69,6 @@ impl<'a> Value<'a> for &'a OwnedValue {
OwnedValue::F64(val) => ReferenceValueLeaf::F64(*val).into(),
OwnedValue::Bool(val) => ReferenceValueLeaf::Bool(*val).into(),
OwnedValue::Date(val) => ReferenceValueLeaf::Date(*val).into(),
- OwnedValue::Facet(val) => ReferenceValueLeaf::Facet(val.encoded_str()).into(),
OwnedValue::Bytes(val) => ReferenceValueLeaf::Bytes(val).into(),
OwnedValue::IpAddr(val) => ReferenceValueLeaf::IpAddr(*val).into(),
OwnedValue::Array(array) => ReferenceValue::Array(array.iter()),
@@ -83,7 +79,9 @@ impl<'a> Value<'a> for &'a OwnedValue {
impl ValueDeserialize for OwnedValue {
fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
- where D: ValueDeserializer<'de> {
+ where
+ D: ValueDeserializer<'de>,
+ {
struct Visitor;
impl ValueVisitor for Visitor {
@@ -121,10 +119,6 @@ impl ValueDeserialize for OwnedValue {
Ok(OwnedValue::IpAddr(val))
}
- fn visit_facet(&self, val: Facet) -> Result<Self::Value, DeserializeError> {
- Ok(OwnedValue::Facet(val))
- }
-
fn visit_bytes(&self, val: Vec<u8>) -> Result<Self::Value, DeserializeError> {
Ok(OwnedValue::Bytes(val))
}
@@ -137,7 +131,9 @@ impl ValueDeserialize for OwnedValue {
}
fn visit_array<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
- where A: ArrayAccess<'de> {
+ where
+ A: ArrayAccess<'de>,
+ {
let mut elements = Vec::with_capacity(access.size_hint());
while let Some(value) = access.next_element()? {
@@ -148,7 +144,9 @@ impl ValueDeserialize for OwnedValue {
}
fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
- where A: ObjectAccess<'de> {
+ where
+ A: ObjectAccess<'de>,
+ {
let mut elements = Vec::with_capacity(access.size_hint());
while let Some((key, value)) = access.next_entry()? {
@@ -167,7 +165,9 @@ impl Eq for OwnedValue {}
impl serde::Serialize for OwnedValue {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
- where S: serde::Serializer {
+ where
+ S: serde::Serializer,
+ {
use serde::ser::SerializeMap;
match *self {
OwnedValue::Null => serializer.serialize_unit(),
@@ -180,7 +180,6 @@ impl serde::Serialize for OwnedValue {
OwnedValue::Date(ref date) => {
time::serde::rfc3339::serialize(&date.into_utc(), serializer)
}
- OwnedValue::Facet(ref facet) => facet.serialize(serializer),
OwnedValue::Bytes(ref bytes) => serializer.serialize_str(&BASE64.encode(bytes)),
OwnedValue::Object(ref obj) => {
let mut map = serializer.serialize_map(Some(obj.len()))?;
@@ -204,7 +203,9 @@ impl serde::Serialize for OwnedValue {
impl<'de> serde::Deserialize<'de> for OwnedValue {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where D: serde::Deserializer<'de> {
+ where
+ D: serde::Deserializer<'de>,
+ {
struct ValueVisitor;
impl<'de> serde::de::Visitor<'de> for ValueVisitor {
@@ -239,12 +240,16 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
}
fn visit_unit<E>(self) -> Result<Self::Value, E>
- where E: serde::de::Error {
+ where
+ E: serde::de::Error,
+ {
Ok(OwnedValue::Null)
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
- where A: SeqAccess<'de> {
+ where
+ A: SeqAccess<'de>,
+ {
let mut elements = Vec::with_capacity(seq.size_hint().unwrap_or_default());
while let Some(value) = seq.next_element()? {
@@ -255,7 +260,9 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
- where A: MapAccess<'de> {
+ where
+ A: MapAccess<'de>,
+ {
let mut object = map.size_hint().map(Vec::with_capacity).unwrap_or_default();
while let Some((key, value)) = map.next_entry()? {
object.push((key, value));
@@ -278,9 +285,6 @@ impl<'a, V: Value<'a>> From<ReferenceValue<'a, V>> for OwnedValue {
ReferenceValueLeaf::I64(val) => OwnedValue::I64(val),
ReferenceValueLeaf::F64(val) => OwnedValue::F64(val),
ReferenceValueLeaf::Date(val) => OwnedValue::Date(val),
- ReferenceValueLeaf::Facet(val) => {
- OwnedValue::Facet(Facet::from_encoded_string(val.to_string()))
- }
ReferenceValueLeaf::Bytes(val) => OwnedValue::Bytes(val.to_vec()),
ReferenceValueLeaf::IpAddr(val) => OwnedValue::IpAddr(val),
ReferenceValueLeaf::Bool(val) => OwnedValue::Bool(val),
@@ -351,12 +355,6 @@ impl<'a> From<&'a [u8]> for OwnedValue {
}
}
-impl From<Facet> for OwnedValue {
- fn from(facet: Facet) -> OwnedValue {
- OwnedValue::Facet(facet)
- }
-}
-
impl From<Vec<u8>> for OwnedValue {
fn from(bytes: Vec<u8>) -> OwnedValue {
OwnedValue::Bytes(bytes)
diff --git a/crates/tantivy/src/schema/document/se.rs b/crates/tantivy/src/schema/document/se.rs
index 18d49467..a0bb2661 100644
--- a/crates/tantivy/src/schema/document/se.rs
+++ b/crates/tantivy/src/schema/document/se.rs
@@ -119,11 +119,6 @@ where
self.write_type_code(type_codes::DATE_CODE)?;
val.serialize(self.writer)
}
- ReferenceValueLeaf::Facet(val) => {
- self.write_type_code(type_codes::HIERARCHICAL_FACET_CODE)?;
-
- Cow::Borrowed(val).serialize(self.writer)
- }
ReferenceValueLeaf::Bytes(val) => {
self.write_type_code(type_codes::BYTES_CODE)?;
@@ -312,7 +307,7 @@ mod tests {
use super::*;
use crate::schema::document::existing_type_impls::JsonObjectIter;
- use crate::schema::{Facet, Field, FAST, STORED, TEXT};
+ use crate::schema::{Field, FAST, STORED, TEXT};
use crate::tokenizer::PreTokenizedString;
fn serialize_value<'a>(value: ReferenceValue<'a, &'a serde_json::Value>) -> Vec<u8> {
@@ -427,16 +422,6 @@ mod tests {
"Expected serialized value to match the binary representation"
);
- let facet = Facet::from_text("/hello/world").unwrap();
- let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());
- let expected = binary_repr!(
- type_codes::HIERARCHICAL_FACET_CODE => Facet::from_text("/hello/world").unwrap(),
- );
- assert_eq!(
- result, expected,
- "Expected serialized value to match the binary representation"
- );
-
let pre_tok_str = PreTokenizedString {
text: "hello, world".to_string(),
tokens: vec![Token::default(), Token::default()],
diff --git a/crates/tantivy/src/schema/document/value.rs b/crates/tantivy/src/schema/document/value.rs
index fc3023da..086ccefa 100644
--- a/crates/tantivy/src/schema/document/value.rs
+++ b/crates/tantivy/src/schema/document/value.rs
@@ -83,12 +83,6 @@ pub trait Value<'a>: Send + Sync + Debug {
self.as_leaf().and_then(|leaf| leaf.as_bytes())
}
- #[inline]
- /// If the Value is a facet, returns the associated facet. Returns None otherwise.
- fn as_facet(&self) -> Option<&'a str> {
- self.as_leaf().and_then(|leaf| leaf.as_facet())
- }
-
#[inline]
/// Returns the iterator over the array if the Value is an array.
fn as_array(&self) -> Option<Self::ArrayIter> {
@@ -125,9 +119,6 @@ pub enum ReferenceValueLeaf<'a> {
F64(f64),
/// Date/time with nanoseconds precision
Date(DateTime),
- /// Facet string needs to match the format of
- /// [Facet::encoded_str](crate::schema::Facet::encoded_str).
- Facet(&'a str),
/// Arbitrarily sized byte array
Bytes(&'a [u8]),
/// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
@@ -211,7 +202,6 @@ impl<'a, T: Value<'a> + ?Sized> From<ReferenceValueLeaf<'a>> for ReferenceValue<
ReferenceValueLeaf::I64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::I64(val)),
ReferenceValueLeaf::F64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::F64(val)),
ReferenceValueLeaf::Date(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Date(val)),
- ReferenceValueLeaf::Facet(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Facet(val)),
ReferenceValueLeaf::Bytes(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(val)),
ReferenceValueLeaf::IpAddr(val) => {
ReferenceValue::Leaf(ReferenceValueLeaf::IpAddr(val))
@@ -321,16 +311,6 @@ impl<'a> ReferenceValueLeaf<'a> {
None
}
}
-
- #[inline]
- /// If the Value is a facet, returns the associated facet. Returns None otherwise.
- pub fn as_facet(&self) -> Option<&'a str> {
- if let Self::Facet(val) = self {
- Some(val)
- } else {
- None
- }
- }
}
/// A enum representing a value for tantivy to index.
@@ -433,12 +413,6 @@ where
self.as_leaf().and_then(|leaf| leaf.as_bytes())
}
- #[inline]
- /// If the Value is a facet, returns the associated facet. Returns None otherwise.
- pub fn as_facet(&self) -> Option<&'a str> {
- self.as_leaf().and_then(|leaf| leaf.as_facet())
- }
-
#[inline]
/// Returns true if the Value is an array.
pub fn is_array(&self) -> bool {
diff --git a/crates/tantivy/src/schema/facet.rs b/crates/tantivy/src/schema/facet.rs
deleted file mode 100644
index 515c41df..00000000
--- a/crates/tantivy/src/schema/facet.rs
+++ /dev/null
@@ -1,354 +0,0 @@
-use std::borrow::{Borrow, Cow};
-use std::fmt::{self, Debug, Display, Formatter};
-use std::io::{self, Read, Write};
-use std::str;
-use std::string::FromUtf8Error;
-
-use crate::common::BinarySerializable;
-use once_cell::sync::Lazy;
-use regex::Regex;
-use serde::de::Error as _;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
-
-const SLASH_BYTE: u8 = b'/';
-const ESCAPE_BYTE: u8 = b'\\';
-
-/// BYTE used as a level separation in the binary
-/// representation of facets.
-pub const FACET_SEP_BYTE: u8 = 0u8;
-
-/// `char` used as a level separation in the binary
-/// representation of facets. (It is the null codepoint.)
-pub const FACET_SEP_CHAR: char = '\u{0}';
-
-/// An error enum for facet parser.
-#[derive(Debug, PartialEq, Eq, Error)]
-pub enum FacetParseError {
- /// The facet text representation is unparsable.
- #[error("Failed to parse the facet string: '{0}'")]
- FacetParseError(String),
-}
-
-/// A Facet represent a point in a given hierarchy.
-///
-/// They are typically represented similarly to a filepath.
-/// For instance, an e-commerce website could
-/// have a `Facet` for `/electronics/tv_and_video/led_tv`.
-///
-/// A document can be associated with any number of facets.
-/// The hierarchy implicitly imply that a document
-/// belonging to a facet also belongs to the ancestor of
-/// its facet. In the example above, `/electronics/tv_and_video/`
-/// and `/electronics`.
-#[derive(Clone, Default, Eq, Hash, PartialEq, Ord, PartialOrd)]
-pub struct Facet(pub(crate) String);
-
-impl Facet {
- /// Returns a new instance of the "root facet"
- /// Equivalent to `/`.
- pub fn root() -> Facet {
- Facet("".to_string())
- }
-
- /// Returns true if the facet is the root facet `/`.
- pub fn is_root(&self) -> bool {
- self.encoded_str().is_empty()
- }
-
- /// Returns a binary representation of the facet.
- ///
- /// In this representation, `0u8` is used as a separator
- /// and the string parts of the facet are unescaped.
- /// (The first `/` is not encoded at all).
- ///
- /// This representation has the benefit of making it possible to
- /// express "being a child of a given facet" as a range over
- /// the term ordinals.
- pub fn encoded_str(&self) -> &str {
- &self.0
- }
-
- pub(crate) fn from_encoded_string(facet_string: String) -> Facet {
- Facet(facet_string)
- }
-
- /// Creates a `Facet` from its binary representation.
- pub fn from_encoded(encoded_bytes: Vec<u8>) -> Result<Facet, FromUtf8Error> {
- // facet bytes validation. `0u8` is used a separator but that is still legal utf-8
- String::from_utf8(encoded_bytes).map(Facet)
- }
-
- /// Parse a text representation of a facet.
- ///
- /// If one of the segments of this path
- /// contains a `/`, it should be escaped
- /// using an anti-slash `\`.
- pub fn from_text<T>(path: &T) -> Result<Facet, FacetParseError>
- where
- T: ?Sized + AsRef<str>,
- {
- #[derive(Copy, Clone)]
- enum State {
- Escaped,
- Idle,
- }
- let path_ref = path.as_ref();
- if path_ref.is_empty() {
- return Err(FacetParseError::FacetParseError(path_ref.to_string()));
- }
- if !path_ref.starts_with('/') {
- return Err(FacetParseError::FacetParseError(path_ref.to_string()));
- }
- let mut facet_encoded = String::new();
- let mut state = State::Idle;
- let path_bytes = path_ref.as_bytes();
- let mut last_offset = 1;
- for i in 1..path_bytes.len() {
- let c = path_bytes[i];
- match (state, c) {
- (State::Idle, ESCAPE_BYTE) => {
- facet_encoded.push_str(&path_ref[last_offset..i]);
- last_offset = i + 1;
- state = State::Escaped
- }
- (State::Idle, SLASH_BYTE) => {
- facet_encoded.push_str(&path_ref[last_offset..i]);
- facet_encoded.push(FACET_SEP_CHAR);
- last_offset = i + 1;
- }
- (State::Escaped, _escaped_char) => {
- state = State::Idle;
- }
- (State::Idle, _any_char) => {}
- }
- }
- facet_encoded.push_str(&path_ref[last_offset..]);
- Ok(Facet(facet_encoded))
- }
-
- /// Returns a `Facet` from an iterator over the different
- /// steps of the facet path.
- ///
- /// The steps are expected to be unescaped.
- pub fn from_path<Path>(path: Path) -> Facet
- where
- Path: IntoIterator,
- Path::Item: AsRef<str>,
- {
- let mut facet_string: String = String::with_capacity(100);
- let mut step_it = path.into_iter();
- if let Some(step) = step_it.next() {
- facet_string.push_str(step.as_ref());
- }
- for step in step_it {
- facet_string.push(FACET_SEP_CHAR);
- facet_string.push_str(step.as_ref());
- }
- Facet(facet_string)
- }
-
- /// Returns `true` if other is a `strict` subfacet of `self`.
- ///
- /// Disclaimer: By strict we mean that the relation is not reflexive.
- /// `/happy` is not a prefix of `/happy`.
- pub fn is_prefix_of(&self, other: &Facet) -> bool {
- let self_str = self.encoded_str();
- let other_str = other.encoded_str();
-
- // Fast path, but also required to ensure that / is not a prefix of /.
- if other_str.len() <= self_str.len() {
- return false;
- }
-
- // Root is a prefix of every other path.
- // This is not just an optimisation. It is necessary for correctness.
- if self.is_root() {
- return true;
- }
-
- other_str.starts_with(self_str) && other_str.as_bytes()[self_str.len()] == FACET_SEP_BYTE
- }
-
- /// Extract path from the `Facet`.
- pub fn to_path(&self) -> Vec<&str> {
- self.encoded_str().split(|c| c == FACET_SEP_CHAR).collect()
- }
-
- /// This function is the inverse of Facet::from(&str).
- pub fn to_path_string(&self) -> String {
- format!("{self}")
- }
-}
-
-impl Borrow<str> for Facet {
- fn borrow(&self) -> &str {
- self.encoded_str()
- }
-}
-
-impl<'a, T: ?Sized + AsRef<str>> From<&'a T> for Facet {
- fn from(path_asref: &'a T) -> Facet {
- Facet::from_text(path_asref).unwrap()
- }
-}
-
-impl BinarySerializable for Facet {
- fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
- <String as BinarySerializable>::serialize(&self.0, writer)
- }
-
- fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
- Ok(Facet(<String as BinarySerializable>::deserialize(reader)?))
- }
-}
-
-impl Display for Facet {
- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
- for step in self.0.split(FACET_SEP_CHAR) {
- write!(f, "/")?;
- write!(f, "{}", escape_slashes(step))?;
- }
- Ok(())
- }
-}
-
-fn escape_slashes(s: &str) -> Cow<'_, str> {
- static SLASH_PTN: Lazy<Regex> = Lazy::new(|| Regex::new(r"[\\/]").unwrap());
- SLASH_PTN.replace_all(s, "\\/")
-}
-
-impl Serialize for Facet {
- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
- where
- S: Serializer,
- {
- serializer.serialize_str(&self.to_string())
- }
-}
-
-impl<'de> Deserialize<'de> for Facet {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: Deserializer<'de>,
- {
- <Cow<'de, str> as Deserialize<'de>>::deserialize(deserializer).and_then(|path| {
- Facet::from_text(&*path).map_err(|err| D::Error::custom(err.to_string()))
- })
- }
-}
-
-impl Debug for Facet {
- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
- write!(f, "Facet({self})")?;
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
-
- use super::{Facet, FacetParseError};
-
- #[test]
- fn test_root() {
- assert_eq!(Facet::root(), Facet::from("/"));
- assert_eq!(format!("{}", Facet::root()), "/");
- assert!(Facet::root().is_root());
- assert_eq!(Facet::root().encoded_str(), "");
- }
-
- #[test]
- fn test_from_path() {
- assert_eq!(
- Facet::from_path(vec!["top", "a", "firstdoc"]),
- Facet::from("/top/a/firstdoc")
- );
- }
-
- #[test]
- fn test_facet_display() {
- {
- let v = ["first", "second", "third"];
- let facet = Facet::from_path(v.iter());
- assert_eq!(format!("{facet}"), "/first/second/third");
- }
- {
- let v = ["first", "sec/ond", "third"];
- let facet = Facet::from_path(v.iter());
- assert_eq!(format!("{facet}"), "/first/sec\\/ond/third");
- }
- }
-
- #[test]
- fn test_facet_debug() {
- let v = ["first", "second", "third"];
- let facet = Facet::from_path(v.iter());
- assert_eq!(format!("{facet:?}"), "Facet(/first/second/third)");
- }
-
- #[test]
- fn test_to_path() {
- let v = ["first", "second", "third\\/not_fourth"];
- let facet = Facet::from_path(v.iter());
- assert_eq!(facet.to_path(), v);
- }
-
- #[test]
- fn test_to_path_string() {
- let v = ["first", "second", "third/not_fourth"];
- let facet = Facet::from_path(v.iter());
- assert_eq!(
- facet.to_path_string(),
- String::from("/first/second/third\\/not_fourth")
- );
- }
-
- #[test]
- fn test_to_path_string_empty() {
- let v: Vec<&str> = vec![];
- let facet = Facet::from_path(v.iter());
- assert_eq!(facet.to_path_string(), "/");
- }
-
- #[test]
- fn test_from_text() {
- assert_eq!(
- Err(FacetParseError::FacetParseError("INVALID".to_string())),
- Facet::from_text("INVALID")
- );
- }
-
- #[test]
- fn only_proper_prefixes() {
- assert!(Facet::from("/foo").is_prefix_of(&Facet::from("/foo/bar")));
-
- assert!(!Facet::from("/foo/bar").is_prefix_of(&Facet::from("/foo/bar")));
- }
-
- #[test]
- fn root_is_a_prefix() {
- assert!(Facet::from("/").is_prefix_of(&Facet::from("/foobar")));
- assert!(!Facet::from("/").is_prefix_of(&Facet::from("/")));
- }
-
- #[test]
- fn deserialize_from_borrowed_string() {
- let facet = serde_json::from_str::<Facet>(r#""/foo/bar""#).unwrap();
- assert_eq!(facet, Facet::from_path(["foo", "bar"]));
- }
-
- #[test]
- fn deserialize_from_owned_string() {
- let facet = serde_json::from_str::<Facet>(r#""/foo/\u263A""#).unwrap();
- assert_eq!(facet, Facet::from_path(["foo", "☺"]));
- }
-
- #[test]
- fn deserialize_from_invalid_string() {
- let error = serde_json::from_str::<Facet>(r#""foo/bar""#).unwrap_err();
- assert_eq!(
- error.to_string(),
- "Failed to parse the facet string: 'foo/bar'"
- );
- }
-}
diff --git a/crates/tantivy/src/schema/facet_options.rs b/crates/tantivy/src/schema/facet_options.rs
deleted file mode 100644
index 77a8714e..00000000
--- a/crates/tantivy/src/schema/facet_options.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-use std::ops::BitOr;
-
-use serde::{Deserialize, Serialize};
-
-use crate::schema::flags::{IndexedFlag, SchemaFlagList, StoredFlag};
-
-/// Define how a facet field should be handled by tantivy.
-///
-/// Note that a Facet is always indexed and stored as a fastfield.
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
-pub struct FacetOptions {
- stored: bool,
-}
-
-impl FacetOptions {
- /// Returns true if the value is stored.
- #[inline]
- pub fn is_stored(&self) -> bool {
- self.stored
- }
-
- /// Set the field as stored.
- ///
- /// Only the fields that are set as *stored* are
- /// persisted into the Tantivy's store.
- #[must_use]
- pub fn set_stored(mut self) -> FacetOptions {
- self.stored = true;
- self
- }
-}
-
-impl From<()> for FacetOptions {
- fn from(_: ()) -> FacetOptions {
- FacetOptions::default()
- }
-}
-
-impl From<StoredFlag> for FacetOptions {
- fn from(_: StoredFlag) -> Self {
- FacetOptions { stored: true }
- }
-}
-
-impl<T: Into<FacetOptions>> BitOr<T> for FacetOptions {
- type Output = FacetOptions;
-
- fn bitor(self, other: T) -> FacetOptions {
- let other = other.into();
- FacetOptions {
- stored: self.stored | other.stored,
- }
- }
-}
-
-impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for FacetOptions
-where
- Head: Clone,
- Tail: Clone,
- Self: BitOr<Output = Self> + From<Head> + From<Tail>,
-{
- fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
- Self::from(head_tail.head) | Self::from(head_tail.tail)
- }
-}
-
-impl From<IndexedFlag> for FacetOptions {
- fn from(_: IndexedFlag) -> Self {
- FacetOptions { stored: false }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::schema::{FacetOptions, INDEXED};
-
- #[test]
- fn test_from_index_flag() {
- let facet_option = FacetOptions::from(INDEXED);
- assert_eq!(facet_option, FacetOptions::default());
- }
-}
diff --git a/crates/tantivy/src/schema/field_entry.rs b/crates/tantivy/src/schema/field_entry.rs
index 932a2972..67cc894d 100644
--- a/crates/tantivy/src/schema/field_entry.rs
+++ b/crates/tantivy/src/schema/field_entry.rs
@@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize};
use super::ip_options::IpAddrOptions;
use crate::schema::bytes_options::BytesOptions;
use crate::schema::{
- is_valid_field_name, DateOptions, FacetOptions, FieldType, JsonObjectOptions, NumericOptions,
- TextOptions,
+ is_valid_field_name, DateOptions, FieldType, JsonObjectOptions, NumericOptions, TextOptions,
};
/// A `FieldEntry` represents a field and its configuration.
@@ -66,11 +65,6 @@ impl FieldEntry {
Self::new(field_name, FieldType::IpAddr(ip_options))
}
- /// Creates a field entry for a facet.
- pub fn new_facet(field_name: String, facet_options: FacetOptions) -> FieldEntry {
- Self::new(field_name, FieldType::Facet(facet_options))
- }
-
/// Creates a field entry for a bytes field
pub fn new_bytes(field_name: String, bytes_options: BytesOptions) -> FieldEntry {
Self::new(field_name, FieldType::Bytes(bytes_options))
@@ -126,7 +120,6 @@ impl FieldEntry {
| FieldType::Bool(ref options) => options.is_stored(),
FieldType::Date(ref options) => options.is_stored(),
FieldType::Str(ref options) => options.is_stored(),
- FieldType::Facet(ref options) => options.is_stored(),
FieldType::Bytes(ref options) => options.is_stored(),
FieldType::JsonObject(ref options) => options.is_stored(),
FieldType::IpAddr(ref options) => options.is_stored(),
diff --git a/crates/tantivy/src/schema/field_type.rs b/crates/tantivy/src/schema/field_type.rs
index d057a499..472ab4aa 100644
--- a/crates/tantivy/src/schema/field_type.rs
+++ b/crates/tantivy/src/schema/field_type.rs
@@ -11,9 +11,8 @@ use thiserror::Error;
use super::ip_options::IpAddrOptions;
use super::IntoIpv6Addr;
use crate::schema::bytes_options::BytesOptions;
-use crate::schema::facet_options::FacetOptions;
use crate::schema::{
- DateOptions, Facet, IndexRecordOption, JsonObjectOptions, NumericOptions, OwnedValue,
+ DateOptions, IndexRecordOption, JsonObjectOptions, NumericOptions, OwnedValue,
TextFieldIndexing, TextOptions,
};
use crate::time::format_description::well_known::Rfc3339;
@@ -63,8 +62,6 @@ pub enum Type {
Bool = b'o',
/// `date(i64) timestamp`
Date = b'd',
- /// `tantivy::schema::Facet`. Passed as a string in JSON.
- Facet = b'h',
/// `Vec<u8>`
Bytes = b'b',
/// Leaf in a Json object.
@@ -76,26 +73,23 @@ pub enum Type {
impl From<ColumnType> for Type {
fn from(value: ColumnType) -> Self {
match value {
- ColumnType::Str => Type::Str,
ColumnType::U64 => Type::U64,
ColumnType::I64 => Type::I64,
ColumnType::F64 => Type::F64,
ColumnType::Bool => Type::Bool,
ColumnType::DateTime => Type::Date,
ColumnType::Bytes => Type::Bytes,
- ColumnType::IpAddr => Type::IpAddr,
}
}
}
-const ALL_TYPES: [Type; 10] = [
+const ALL_TYPES: [Type; 9] = [
Type::Str,
Type::U64,
Type::I64,
Type::F64,
Type::Bool,
Type::Date,
- Type::Facet,
Type::Bytes,
Type::Json,
Type::IpAddr,
@@ -123,7 +117,6 @@ impl Type {
Type::F64 => "F64",
Type::Bool => "Bool",
Type::Date => "Date",
- Type::Facet => "Facet",
Type::Bytes => "Bytes",
Type::Json => "Json",
Type::IpAddr => "IpAddr",
@@ -141,7 +134,6 @@ impl Type {
b'f' => Some(Type::F64),
b'o' => Some(Type::Bool),
b'd' => Some(Type::Date),
- b'h' => Some(Type::Facet),
b'b' => Some(Type::Bytes),
b'j' => Some(Type::Json),
b'p' => Some(Type::IpAddr),
@@ -169,8 +161,6 @@ pub enum FieldType {
Bool(NumericOptions),
/// Signed 64-bits Date 64 field type configuration,
Date(DateOptions),
- /// Hierarchical Facet
- Facet(FacetOptions),
/// Bytes (one per document)
Bytes(BytesOptions),
/// Json object
@@ -189,7 +179,6 @@ impl FieldType {
FieldType::F64(_) => Type::F64,
FieldType::Bool(_) => Type::Bool,
FieldType::Date(_) => Type::Date,
- FieldType::Facet(_) => Type::Facet,
FieldType::Bytes(_) => Type::Bytes,
FieldType::JsonObject(_) => Type::Json,
FieldType::IpAddr(_) => Type::IpAddr,
@@ -215,7 +204,6 @@ impl FieldType {
| FieldType::F64(ref int_options)
| FieldType::Bool(ref int_options) => int_options.is_indexed(),
FieldType::Date(ref date_options) => date_options.is_indexed(),
- FieldType::Facet(ref _facet_options) => true,
FieldType::Bytes(ref bytes_options) => bytes_options.is_indexed(),
FieldType::JsonObject(ref json_object_options) => json_object_options.is_indexed(),
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_indexed(),
@@ -254,7 +242,6 @@ impl FieldType {
| FieldType::Bool(ref int_options) => int_options.is_fast(),
FieldType::Date(ref date_options) => date_options.is_fast(),
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(),
- FieldType::Facet(_) => true,
FieldType::JsonObject(ref json_object_options) => json_object_options.is_fast(),
}
}
@@ -271,7 +258,6 @@ impl FieldType {
| FieldType::F64(ref int_options)
| FieldType::Bool(ref int_options) => int_options.fieldnorms(),
FieldType::Date(ref date_options) => date_options.fieldnorms(),
- FieldType::Facet(_) => false,
FieldType::Bytes(ref bytes_options) => bytes_options.fieldnorms(),
FieldType::JsonObject(ref _json_object_options) => false,
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.fieldnorms(),
@@ -308,7 +294,6 @@ impl FieldType {
None
}
}
- FieldType::Facet(ref _facet_options) => Some(IndexRecordOption::Basic),
FieldType::Bytes(ref bytes_options) => {
if bytes_options.is_indexed() {
Some(IndexRecordOption::Basic)
@@ -408,7 +393,6 @@ impl FieldType {
})
}
}
- FieldType::Facet(_) => Ok(OwnedValue::Facet(Facet::from(&field_text))),
FieldType::Bytes(_) => BASE64
.decode(&field_text)
.map(OwnedValue::Bytes)
@@ -474,7 +458,7 @@ impl FieldType {
})
}
}
- FieldType::Facet(_) | FieldType::Bytes(_) => Err(ValueParsingError::TypeError {
+ FieldType::Bytes(_) => Err(ValueParsingError::TypeError {
expected: "a string",
json: JsonValue::Number(field_val_num),
}),
diff --git a/crates/tantivy/src/schema/mod.rs b/crates/tantivy/src/schema/mod.rs
index 0d4b4b1a..2c6e98d8 100644
--- a/crates/tantivy/src/schema/mod.rs
+++ b/crates/tantivy/src/schema/mod.rs
@@ -8,7 +8,7 @@
//! - the field name (may contain any characted, can't start with a `-` and can't be empty. Some
//! characters may require escaping when using the query parser).
//! - the type of the field (currently `text`, `u64`, `i64`, `f64`, `bool`, `date`, `IpAddr`,
-//! facets, bytes and json are supported)
+//! bytes and json are supported)
//! - how the field should be indexed / stored.
//!
//! This very last point is critical as it will enable / disable some of the functionality
@@ -107,8 +107,6 @@
//! ```
pub mod document;
-mod facet;
-mod facet_options;
mod schema;
pub(crate) mod term;
@@ -131,9 +129,6 @@ use crate::columnar::ColumnType;
pub use self::bytes_options::BytesOptions;
pub use self::date_time_options::{DateOptions, DateTimePrecision, DATE_TIME_PRECISION_INDEXED};
pub use self::document::{DocParsingError, Document, OwnedValue, TantivyDocument, Value};
-pub(crate) use self::facet::FACET_SEP_BYTE;
-pub use self::facet::{Facet, FacetParseError};
-pub use self::facet_options::FacetOptions;
pub use self::field::Field;
pub use self::field_entry::FieldEntry;
pub use self::field_type::{FieldType, Type};
@@ -158,16 +153,13 @@ pub fn is_valid_field_name(field_name: &str) -> bool {
pub(crate) fn value_type_to_column_type(typ: Type) -> Option<ColumnType> {
match typ {
- Type::Str => Some(ColumnType::Str),
Type::U64 => Some(ColumnType::U64),
Type::I64 => Some(ColumnType::I64),
Type::F64 => Some(ColumnType::F64),
Type::Bool => Some(ColumnType::Bool),
Type::Date => Some(ColumnType::DateTime),
- Type::Facet => Some(ColumnType::Str),
Type::Bytes => Some(ColumnType::Bytes),
- Type::IpAddr => Some(ColumnType::IpAddr),
- Type::Json => None,
+ Type::Json | Type::Str | Type::IpAddr => None,
}
}
diff --git a/crates/tantivy/src/schema/schema.rs b/crates/tantivy/src/schema/schema.rs
index a91fb671..1c377dc8 100644
--- a/crates/tantivy/src/schema/schema.rs
+++ b/crates/tantivy/src/schema/schema.rs
@@ -155,16 +155,6 @@ impl SchemaBuilder {
self.add_field(field_entry)
}
- /// Adds a facet field to the schema.
- pub fn add_facet_field(
- &mut self,
- field_name: &str,
- facet_options: impl Into<FacetOptions>,
- ) -> Field {
- let field_entry = FieldEntry::new_facet(field_name.to_string(), facet_options.into());
- self.add_field(field_entry)
- }
-
/// Adds a fast bytes field to the schema.
///
/// Bytes field are not searchable and are only used
@@ -372,7 +362,9 @@ impl Schema {
impl Serialize for Schema {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
- where S: Serializer {
+ where
+ S: Serializer,
+ {
let mut seq = serializer.serialize_seq(Some(self.0.fields.len()))?;
for e in &self.0.fields {
seq.serialize_element(e)?;
@@ -383,7 +375,9 @@ impl Serialize for Schema {
impl<'de> Deserialize<'de> for Schema {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where D: Deserializer<'de> {
+ where
+ D: Deserializer<'de>,
+ {
struct SchemaVisitor;
impl<'de> Visitor<'de> for SchemaVisitor {
@@ -394,7 +388,9 @@ impl<'de> Deserialize<'de> for Schema {
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
- where A: SeqAccess<'de> {
+ where
+ A: SeqAccess<'de>,
+ {
let mut schema = SchemaBuilder {
fields: Vec::with_capacity(seq.size_hint().unwrap_or(0)),
fields_map: HashMap::with_capacity(seq.size_hint().unwrap_or(0)),
diff --git a/crates/tantivy/src/schema/term.rs b/crates/tantivy/src/schema/term.rs
index 62261f53..21c1b1e4 100644
--- a/crates/tantivy/src/schema/term.rs
+++ b/crates/tantivy/src/schema/term.rs
@@ -10,7 +10,7 @@ use super::date_time_options::DATE_TIME_PRECISION_INDEXED;
use super::Field;
use crate::fastfield::FastValue;
use crate::json_utils::split_json_path;
-use crate::schema::{Facet, Type};
+use crate::schema::Type;
use crate::DateTime;
/// Term represents the value that the token can take.
@@ -124,12 +124,6 @@ impl Term {
Term::from_fast_value(field, &val.truncate(DATE_TIME_PRECISION_INDEXED))
}
- /// Creates a `Term` given a facet.
- pub fn from_facet(field: Field, facet: &Facet) -> Term {
- let facet_encoded_str = facet.encoded_str();
- Term::with_bytes_and_field_and_payload(Type::Facet, field, facet_encoded_str.as_bytes())
- }
-
/// Builds a term given a field, and a string value
pub fn from_field_text(field: Field, text: &str) -> Term {
Term::with_bytes_and_field_and_payload(Type::Str, field, text.as_bytes())
@@ -401,18 +395,6 @@ where
str::from_utf8(self.value_bytes()).ok()
}
- /// Returns the facet associated with the term.
- ///
- /// Returns `None` if the field is not of facet type
- /// or if the bytes are not valid utf-8.
- pub fn as_facet(&self) -> Option<Facet> {
- if self.typ() != Type::Facet {
- return None;
- }
- let facet_encode_str = str::from_utf8(self.value_bytes()).ok()?;
- Some(Facet::from_encoded_string(facet_encode_str.to_string()))
- }
-
/// Returns the bytes associated with the term.
///
/// Returns `None` if the field is not of bytes type.
@@ -506,9 +488,6 @@ where
Type::Date => {
write_opt(f, self.as_date())?;
}
- Type::Facet => {
- write_opt(f, self.as_facet())?;
- }
Type::Bytes => {
write_opt(f, self.as_bytes())?;
}
diff --git a/crates/tantivy/src/tokenizer/facet_tokenizer.rs b/crates/tantivy/src/tokenizer/facet_tokenizer.rs
deleted file mode 100644
index db0df294..00000000
--- a/crates/tantivy/src/tokenizer/facet_tokenizer.rs
+++ /dev/null
@@ -1,128 +0,0 @@
-use super::{Token, TokenStream, Tokenizer};
-use crate::schema::FACET_SEP_BYTE;
-
-/// The `FacetTokenizer` process a `Facet` binary representation
-/// and emits a token for all of its parent.
-///
-/// For instance, `/america/north_america/canada`
-/// will emit the three following tokens
-/// - `/america/north_america/canada`
-/// - `/america/north_america`
-/// - `/america`
-#[derive(Clone, Default)]
-pub struct FacetTokenizer {
- token: Token,
-}
-
-#[derive(Debug)]
-enum State {
- RootFacetNotEmitted,
- UpToPosition(usize), //< we already emitted facet prefix up to &text[..cursor]
- Terminated,
-}
-
-pub struct FacetTokenStream<'a> {
- text: &'a str,
- state: State,
- token: &'a mut Token,
-}
-
-impl Tokenizer for FacetTokenizer {
- type TokenStream<'a> = FacetTokenStream<'a>;
- fn token_stream<'a>(&'a mut self, text: &'a str) -> FacetTokenStream<'a> {
- self.token.reset();
- self.token.position = 0;
- FacetTokenStream {
- text,
- state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
- token: &mut self.token,
- }
- }
-}
-
-impl<'a> TokenStream for FacetTokenStream<'a> {
- fn advance(&mut self) -> bool {
- match self.state {
- State::RootFacetNotEmitted => {
- self.state = if self.text.is_empty() {
- State::Terminated
- } else {
- State::UpToPosition(0)
- };
- true
- }
- State::UpToPosition(cursor) => {
- let bytes: &[u8] = self.text.as_bytes();
- if let Some(next_sep_pos) = bytes[cursor + 1..]
- .iter()
- .cloned()
- .position(|b| b == FACET_SEP_BYTE)
- .map(|pos| cursor + 1 + pos)
- {
- let facet_part = &self.text[cursor..next_sep_pos];
- self.token.text.push_str(facet_part);
- self.state = State::UpToPosition(next_sep_pos);
- } else {
- let facet_part = &self.text[cursor..];
- self.token.text.push_str(facet_part);
- self.state = State::Terminated;
- }
- true
- }
- State::Terminated => false,
- }
- }
-
- fn token(&self) -> &Token {
- self.token
- }
-
- fn token_mut(&mut self) -> &mut Token {
- self.token
- }
-}
-
-#[cfg(test)]
-mod tests {
-
- use super::FacetTokenizer;
- use crate::schema::Facet;
- use crate::tokenizer::{Token, TokenStream, Tokenizer};
-
- #[test]
- fn test_facet_tokenizer() {
- let facet = Facet::from_path(vec!["top", "a", "b"]);
- let mut tokens = vec![];
- {
- let mut add_token = |token: &Token| {
- let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap();
- tokens.push(format!("{facet}"));
- };
- FacetTokenizer::default()
- .token_stream(facet.encoded_str())
- .process(&mut add_token);
- }
- assert_eq!(tokens.len(), 4);
- assert_eq!(tokens[0], "/");
- assert_eq!(tokens[1], "/top");
- assert_eq!(tokens[2], "/top/a");
- assert_eq!(tokens[3], "/top/a/b");
- }
-
- #[test]
- fn test_facet_tokenizer_root_facets() {
- let facet = Facet::root();
- let mut tokens = vec![];
- {
- let mut add_token = |token: &Token| {
- let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap(); // ok test
- tokens.push(format!("{facet}"));
- };
- FacetTokenizer::default()
- .token_stream(facet.encoded_str()) // ok test
- .process(&mut add_token);
- }
- assert_eq!(tokens.len(), 1);
- assert_eq!(tokens[0], "/");
- }
-}
diff --git a/crates/tantivy/src/tokenizer/mod.rs b/crates/tantivy/src/tokenizer/mod.rs
index dd79ac10..b2d30319 100644
--- a/crates/tantivy/src/tokenizer/mod.rs
+++ b/crates/tantivy/src/tokenizer/mod.rs
@@ -124,7 +124,6 @@
mod alphanum_only;
mod ascii_folding_filter;
mod empty_tokenizer;
-mod facet_tokenizer;
mod lower_caser;
mod ngram_tokenizer;
mod raw_tokenizer;
@@ -143,7 +142,6 @@ pub use crate::tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream,
pub use self::alphanum_only::AlphaNumOnlyFilter;
pub use self::ascii_folding_filter::AsciiFoldingFilter;
-pub use self::facet_tokenizer::FacetTokenizer;
pub use self::lower_caser::LowerCaser;
pub use self::ngram_tokenizer::NgramTokenizer;
pub use self::raw_tokenizer::RawTokenizer;