web search done right
rename fast fields to columnar fields
Mikkel Denker5b8f03c89021
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex 0bbe6595..46b3f028 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -25,12 +25,12 @@ use tantivy::{};use crate::{+ columnfield_reader,config::CollectorConfig,- fastfield_reader,inverted_index::{DocAddress, WebpagePointer},prehashed::Prehashed,ranking::initial::{InitialScoreTweaker, Score},- schema::{fast_field, FastFieldEnum},+ schema::{column_field, ColumnFieldEnum},simhash,};@@ -40,19 +40,22 @@ pub struct TopDocs {top_n: usize,offset: usize,max_docs: Option<MaxDocsConsidered>,- fastfield_reader: fastfield_reader::FastFieldReader,+ columnfield_reader: columnfield_reader::ColumnFieldReader,de_rank_similar: bool,collector_config: CollectorConfig,}impl TopDocs {- pub fn with_limit(top_n: usize, fastfield_reader: fastfield_reader::FastFieldReader) -> Self {+ pub fn with_limit(+ top_n: usize,+ columnfield_reader: columnfield_reader::ColumnFieldReader,+ ) -> Self {Self {top_n,offset: 0,max_docs: None,de_rank_similar: false,- fastfield_reader,+ columnfield_reader,collector_config: CollectorConfig::default(),}}@@ -98,7 +101,7 @@ impl TopDocs {.map(|max_docs| max_docs.total_docs / max_docs.segments);Ok(TopSegmentCollector {- fastfield_segment_reader: self.fastfield_reader.get_segment(&segment.segment_id()),+ columnfield_segment_reader: self.columnfield_reader.get_segment(&segment.segment_id()),max_docs,num_docs_taken: 0,segment_ord: segment_local_id,@@ -111,7 +114,7 @@ impl TopDocs {}pub struct TopSegmentCollector {- fastfield_segment_reader: Arc<fastfield_reader::SegmentReader>,+ columnfield_segment_reader: Arc<columnfield_reader::SegmentReader>,max_docs: Option<usize>,num_docs_taken: usize,segment_ord: SegmentOrdinal,@@ -119,8 +122,8 @@ pub struct TopSegmentCollector {}impl TopSegmentCollector {- fn get_hash(&self, doc: DocId, field1: FastFieldEnum, field2: FastFieldEnum) -> Prehashed {- let field_reader = self.fastfield_segment_reader.get_field_reader(doc);+ fn get_hash(&self, doc: DocId, field1: ColumnFieldEnum, field2: ColumnFieldEnum) -> Prehashed {+ let field_reader = self.columnfield_segment_reader.get_field_reader(doc);let hash = [field_reader.get(field1).unwrap().as_u64().unwrap(),@@ -147,9 +150,9 @@ impl TopSegmentCollector {self.num_docs_taken += 1;let simhash: Option<u64> = self- .fastfield_segment_reader+ .columnfield_segment_reader.get_field_reader(doc)- .get(fast_field::SimHash.into())+ .get(column_field::SimHash.into()).unwrap().into();@@ -157,23 +160,23 @@ impl TopSegmentCollector {hashes: Hashes {site: self.get_hash(doc,- fast_field::SiteHash1.into(),- fast_field::SiteHash2.into(),+ column_field::SiteHash1.into(),+ column_field::SiteHash2.into(),),title: self.get_hash(doc,- fast_field::TitleHash1.into(),- fast_field::TitleHash2.into(),+ column_field::TitleHash1.into(),+ column_field::TitleHash2.into(),),url: self.get_hash(doc,- fast_field::UrlHash1.into(),- fast_field::UrlHash2.into(),+ column_field::UrlHash1.into(),+ column_field::UrlHash2.into(),),url_without_tld: self.get_hash(doc,- fast_field::UrlWithoutTldHash1.into(),- fast_field::UrlWithoutTldHash2.into(),+ column_field::UrlWithoutTldHash1.into(),+ column_field::UrlWithoutTldHash2.into(),),simhash: simhash.unwrap(),},diff --git a/crates/core/src/fastfield_reader.rs b/crates/core/src/columnfield_reader.rssimilarity index 86%rename from crates/core/src/fastfield_reader.rsrename to crates/core/src/columnfield_reader.rsindex e3effc57..25a07865 100644--- a/crates/core/src/fastfield_reader.rs+++ b/crates/core/src/columnfield_reader.rs@@ -20,20 +20,20 @@ use tantivy::{columnar::ColumnValues, index::SegmentId, DocId};use crate::{enum_map::EnumMap,- schema::{fast_field::FastField, DataType, FastFieldEnum, Field},+ schema::{column_field::ColumnField, ColumnFieldEnum, DataType, Field},};#[derive(Default, Clone)]-struct InnerFastFieldReader {+struct InnerColumnFieldReader {segments: HashMap<SegmentId, Arc<SegmentReader>>,}#[derive(Default, Clone)]-pub struct FastFieldReader {- inner: Arc<InnerFastFieldReader>,+pub struct ColumnFieldReader {+ inner: Arc<InnerColumnFieldReader>,}-impl FastFieldReader {+impl ColumnFieldReader {pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {Arc::clone(self.inner.segments.get(segment).unwrap())}@@ -43,12 +43,12 @@ impl FastFieldReader {}}-impl FastFieldReader {+impl ColumnFieldReader {pub fn new(tv_searcher: &tantivy::Searcher) -> Self {let mut segments = HashMap::new();for reader in tv_searcher.segment_readers() {- let fastfield_readers = reader.fast_fields();+ let columnfield_readers = reader.column_fields();let mut u64s = EnumMap::new();let mut bytes = EnumMap::new();@@ -58,7 +58,7 @@ impl FastFieldReader {DataType::U64 => {let num_docs = reader.max_doc() as usize;let mut data = vec![0; num_docs];- if let Ok(field_reader) = fastfield_readers.u64(field.name()) {+ if let Ok(field_reader) = columnfield_readers.u64(field.name()) {for (doc, elem) in data.iter_mut().enumerate() {*elem = field_reader.values.get_val(doc as u32);}@@ -67,7 +67,8 @@ impl FastFieldReader {u64s.insert(field, data);}DataType::Bytes => {- if let Some(reader) = fastfield_readers.bytes(field.name()).ok().flatten() {+ if let Some(reader) = columnfield_readers.bytes(field.name()).ok().flatten()+ {bytes.insert(field, reader);}}@@ -83,14 +84,14 @@ impl FastFieldReader {}Self {- inner: Arc::new(InnerFastFieldReader { segments }),+ inner: Arc::new(InnerColumnFieldReader { segments }),}}}struct AllReaders {- u64s: EnumMap<FastFieldEnum, Vec<u64>>,- bytes: EnumMap<FastFieldEnum, tantivy::columnar::BytesColumn>,+ u64s: EnumMap<ColumnFieldEnum, Vec<u64>>,+ bytes: EnumMap<ColumnFieldEnum, tantivy::columnar::BytesColumn>,}pub enum Value {@@ -153,7 +154,7 @@ pub struct FieldReader<'a> {}impl<'a> FieldReader<'a> {- pub fn get(&self, field: FastFieldEnum) -> Option<Value> {+ pub fn get(&self, field: ColumnFieldEnum) -> Option<Value> {match field.data_type() {DataType::U64 => Some(self.readersdiff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex c114cb72..ed347100 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -18,7 +18,7 @@ use tantivy::merge_policy::NoMergePolicy;use tantivy::{IndexWriter, SegmentMeta};-use crate::fastfield_reader::FastFieldReader;+use crate::columnfield_reader::ColumnFieldReader;use crate::webpage::Webpage;use crate::Result;@@ -126,7 +126,7 @@ impl InvertedIndex {.expect("writer has not been prepared").commit()?;self.reader.reload()?;- self.fastfield_reader = FastFieldReader::new(&self.reader.searcher());+ self.columnfield_reader = ColumnFieldReader::new(&self.reader.searcher());Ok(())}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 77e70dc8..5b3aec9e 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -40,13 +40,13 @@ use tantivy::tokenizer::TokenizerManager;use tantivy::{IndexReader, IndexWriter, TantivyDocument};use crate::collector::{approx_count, Hashes};+use crate::columnfield_reader::ColumnFieldReader;use crate::config::SnippetConfig;-use crate::fastfield_reader::FastFieldReader;use crate::ranking::initial::Score;use crate::schema::text_field::TextField;-use crate::schema::{fast_field, text_field, FastFieldEnum, Field, TextFieldEnum};+use crate::schema::{column_field, text_field, ColumnFieldEnum, Field, TextFieldEnum};use crate::snippet::TextSnippet;use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Stemmed, TrigramTokenizer, UrlTokenizer,@@ -138,7 +138,7 @@ pub struct InvertedIndex {reader: IndexReader,schema: Arc<Schema>,snippet_config: SnippetConfig,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,}impl InvertedIndex {@@ -151,7 +151,7 @@ impl InvertedIndex {} else {let index_settings = tantivy::IndexSettings {sort_by_field: Some(tantivy::IndexSortByField {- field: Field::Fast(FastFieldEnum::from(fast_field::PreComputedScore))+ field: Field::Columnar(ColumnFieldEnum::from(column_field::PreComputedScore)).name().to_string(),order: tantivy::Order::Desc,@@ -168,7 +168,7 @@ impl InvertedIndex {let reader: IndexReader = tantivy_index.reader_builder().try_into()?;- let fastfield_reader = FastFieldReader::new(&reader.searcher());+ let columnfield_reader = ColumnFieldReader::new(&reader.searcher());Ok(InvertedIndex {writer: None,@@ -177,12 +177,12 @@ impl InvertedIndex {path: path.as_ref().to_str().unwrap().to_string(),tantivy_index,snippet_config: SnippetConfig::default(),- fastfield_reader,+ columnfield_reader,})}- pub fn fastfield_reader(&self) -> FastFieldReader {- self.fastfield_reader.clone()+ pub fn columnfield_reader(&self) -> ColumnFieldReader {+ self.columnfield_reader.clone()}pub fn set_snippet_config(&mut self, config: SnippetConfig) {@@ -277,7 +277,7 @@ impl From<TantivyDocument> for RetrievedWebpage {Some(Field::Text(TextFieldEnum::Url(_))) => {webpage.url = str_value(text_field::Url.name(), &value);}- Some(Field::Fast(FastFieldEnum::LastUpdated(_))) => {+ Some(Field::Columnar(ColumnFieldEnum::LastUpdated(_))) => {webpage.updated_time = {let timestamp = value.as_u64().unwrap() as i64;if timestamp == 0 {@@ -290,7 +290,7 @@ impl From<TantivyDocument> for RetrievedWebpage {Some(Field::Text(TextFieldEnum::AllBody(_))) => {webpage.dirty_body = str_value(text_field::AllBody.name(), &value);}- Some(Field::Fast(FastFieldEnum::Region(_))) => {+ Some(Field::Columnar(ColumnFieldEnum::Region(_))) => {webpage.region = {let id = value.as_u64().unwrap();Region::from_id(id)@@ -304,10 +304,10 @@ impl From<TantivyDocument> for RetrievedWebpage {let json = str_value(text_field::SchemaOrgJson.name(), &value);webpage.schema_org = serde_json::from_str(&json).unwrap_or_default();}- Some(Field::Fast(FastFieldEnum::LikelyHasAds(_))) => {+ Some(Field::Columnar(ColumnFieldEnum::LikelyHasAds(_))) => {webpage.likely_has_ads = value.as_u64().unwrap_or_default() != 0;}- Some(Field::Fast(FastFieldEnum::LikelyHasPaywall(_))) => {+ Some(Field::Columnar(ColumnFieldEnum::LikelyHasPaywall(_))) => {webpage.likely_has_paywall = value.as_u64().unwrap_or_default() != 0;}Some(Field::Text(TextFieldEnum::RecipeFirstIngredientTagId(_))) => {@@ -384,7 +384,7 @@ mod tests {let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);let result =@@ -416,7 +416,7 @@ mod tests {let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -465,7 +465,7 @@ mod tests {let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -512,7 +512,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -560,7 +560,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -632,7 +632,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -690,7 +690,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -737,7 +737,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -813,7 +813,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -840,7 +840,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -873,7 +873,7 @@ mod tests {let ctx = index.local_search_ctx();let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);let result =@@ -920,7 +920,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -971,7 +971,7 @@ mod tests {.expect("Failed to parse query");let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -1144,7 +1144,7 @@ mod tests {let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);@@ -1152,10 +1152,15 @@ mod tests {.search_initial(&query, &ctx, ranker.collector(ctx.clone())).unwrap();- let fastfield_reader = index.fastfield_reader();+ let columnfield_reader = index.columnfield_reader();let ranking_websites = index- .retrieve_ranking_websites(&ctx, res.top_websites, ranker.computer(), &fastfield_reader)+ .retrieve_ranking_websites(+ &ctx,+ res.top_websites,+ ranker.computer(),+ &columnfield_reader,+ ).unwrap();assert_eq!(ranking_websites.len(), 2);@@ -1210,7 +1215,7 @@ mod tests {let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),collector_config,);@@ -1261,7 +1266,7 @@ mod tests {let ranker = Ranker::new(SignalComputer::new(Some(&query)),- ctx.fastfield_reader.clone(),+ ctx.columnfield_reader.clone(),CollectorConfig::default(),);diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 5d5cc1b8..89e8cc2b 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -26,13 +26,13 @@ use url::Url;use crate::collector::approx_count::ApproxCount;use crate::collector::{approx_count, MainCollector};-use crate::fastfield_reader::FastFieldReader;+use crate::columnfield_reader::ColumnFieldReader;use crate::highlighted::HighlightedFragment;use crate::query::shortcircuit::ShortCircuitQuery;use crate::query::Query;use crate::ranking::pipeline::LocalRecallRankingWebpage;use crate::ranking::SignalComputer;-use crate::schema::{fast_field, text_field, FastFieldEnum, Field, TextFieldEnum};+use crate::schema::{column_field, text_field, ColumnFieldEnum, Field, TextFieldEnum};use crate::search_ctx::Ctx;use crate::snippet;use crate::snippet::TextSnippet;@@ -87,7 +87,7 @@ impl InvertedIndex {pub fn local_search_ctx(&self) -> Ctx {let tv_searcher = self.tv_searcher();Ctx {- fastfield_reader: self.fastfield_reader.clone(),+ columnfield_reader: self.columnfield_reader.clone(),tv_searcher,}}@@ -101,7 +101,7 @@ impl InvertedIndex {ctx: &Ctx,pointers: Vec<WebpagePointer>,mut computer: SignalComputer,- fastfield_reader: &FastFieldReader,+ columnfield_reader: &ColumnFieldReader,) -> Result<Vec<LocalRecallRankingWebpage>> {let mut top_websites = Vec::new();@@ -127,7 +127,7 @@ impl InvertedIndex {let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);if update_segment {- computer.register_segment(&ctx.tv_searcher, segment_reader, fastfield_reader)?;+ computer.register_segment(&ctx.tv_searcher, segment_reader, columnfield_reader)?;}prev_segment = Some(pointer.address.segment);@@ -136,7 +136,7 @@ impl InvertedIndex {orig_index,LocalRecallRankingWebpage::new(pointer,- fastfield_reader.borrow_segment(&segment_reader.segment_id()),+ columnfield_reader.borrow_segment(&segment_reader.segment_id()),&mut computer,),));@@ -156,7 +156,7 @@ impl InvertedIndex {let field = self.schema()- .get_field(Field::Fast(FastFieldEnum::from(fast_field::HostNodeID)).name())+ .get_field(Field::Columnar(ColumnFieldEnum::from(column_field::HostNodeID)).name()).unwrap();let id = doc.get_first(field).unwrap().as_u64().unwrap();diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 18162b30..5160a249 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -47,6 +47,7 @@ pub mod bangs;mod bincode_utils;pub mod canon_index;mod collector;+mod columnfield_reader;pub mod config;pub mod crawler;pub mod distributed;@@ -54,7 +55,6 @@ pub mod entity_index;mod enum_map;pub mod executor;mod external_sort;-mod fastfield_reader;pub mod feed;mod highlighted;pub mod hyperloglog;diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 0dfcda03..ebd592ba 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -133,7 +133,7 @@ impl Query {for optic in &optics {let mut subqueries = vec![(Occur::Must, tantivy_query.box_clone())];- subqueries.append(&mut optic.as_multiple_tantivy(&schema, &ctx.fastfield_reader));+ subqueries.append(&mut optic.as_multiple_tantivy(&schema, &ctx.columnfield_reader));tantivy_query = Box::new(BooleanQuery::new(subqueries));}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex c3f1e647..623c1216 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -22,7 +22,7 @@ use tantivy::{schema::Schema,};-use crate::{fastfield_reader::FastFieldReader, schema::text_field, webpage::schema_org};+use crate::{columnfield_reader::ColumnFieldReader, schema::text_field, webpage::schema_org};use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};@@ -30,7 +30,7 @@ pub trait AsTantivyQuery {fn as_tantivy(&self,schema: &Schema,- fastfield_reader: &FastFieldReader,+ columnfield_reader: &ColumnFieldReader,) -> Box<dyn tantivy::query::Query>;}@@ -38,7 +38,7 @@ pub trait AsMultipleTantivyQuery {fn as_multiple_tantivy(&self,schema: &Schema,- fastfield_reader: &FastFieldReader,+ columnfield_reader: &ColumnFieldReader,) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;}@@ -46,7 +46,7 @@ impl AsMultipleTantivyQuery for Optic {fn as_multiple_tantivy(&self,schema: &Schema,- fastfields: &FastFieldReader,+ columnfields: &ColumnFieldReader,) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {if self.discard_non_matching {let block = (@@ -55,7 +55,7 @@ impl AsMultipleTantivyQuery for Optic {self.rules.iter().filter(|rule| !matches!(rule.action, Action::Discard))- .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))+ .filter_map(|rule| rule.as_searchable_rule(schema, columnfields)).map(|(occur, rule)| {BooleanQuery::from(vec![(occur, rule.query)]).box_clone()})@@ -68,7 +68,7 @@ impl AsMultipleTantivyQuery for Optic {.iter().filter(|rule| matches!(rule.action, Action::Discard)).chain(iter::once(&self.host_rankings.rules()))- .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))+ .filter_map(|rule| rule.as_searchable_rule(schema, columnfields)).map(|(occur, rule)| (occur, rule.query)).chain(iter::once(block)).collect()@@ -76,7 +76,7 @@ impl AsMultipleTantivyQuery for Optic {self.rules.iter().chain(iter::once(&self.host_rankings.rules()))- .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))+ .filter_map(|rule| rule.as_searchable_rule(schema, columnfields)).map(|(occur, rule)| (occur, rule.query)).collect()}@@ -92,7 +92,7 @@ pub trait AsSearchableRule {fn as_searchable_rule(&self,schema: &Schema,- fastfield_reader: &FastFieldReader,+ columnfield_reader: &ColumnFieldReader,) -> Option<(Occur, SearchableRule)>;}@@ -100,7 +100,7 @@ impl AsSearchableRule for Rule {fn as_searchable_rule(&self,schema: &Schema,- fastfield_reader: &FastFieldReader,+ columnfield_reader: &ColumnFieldReader,) -> Option<(Occur, SearchableRule)> {let mut subqueries: Vec<_> = self.matches@@ -108,7 +108,7 @@ impl AsSearchableRule for Rule {.filter_map(|and_rule| {let mut and_queries: Vec<_> = and_rule.iter()- .map(|matching| (Occur::Must, matching.as_tantivy(schema, fastfield_reader)))+ .map(|matching| (Occur::Must, matching.as_tantivy(schema, columnfield_reader))).collect();// Empty queries never match anything. A priori these shouldn't exist, but it doesn't@@ -167,7 +167,7 @@ impl AsTantivyQuery for Matching {fn as_tantivy(&self,schema: &Schema,- fastfield_reader: &FastFieldReader,+ columnfield_reader: &ColumnFieldReader,) -> Box<dyn tantivy::query::Query> {match &self.location {MatchLocation::Site => ConstQuery::new(@@ -175,7 +175,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::UrlForSiteOperator.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),).box_clone(),1.0,@@ -186,7 +186,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::Url.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)),@@ -195,7 +195,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::Domain.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)),@@ -204,7 +204,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::Title.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)),@@ -214,7 +214,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::Description.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)) as Box<dyn tantivy::query::Query>,@@ -223,7 +223,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::DmozDescription.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)) as Box<dyn tantivy::query::Query>,@@ -234,7 +234,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::CleanBody.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)),@@ -243,7 +243,7 @@ impl AsTantivyQuery for Matching {self.pattern.clone(),text_field::MicroformatTags.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,)),@@ -263,7 +263,7 @@ impl AsTantivyQuery for Matching {pattern,text_field::FlattenedSchemaOrgJson.into(),schema,- fastfield_reader.clone(),+ columnfield_reader.clone(),)),1.0,))diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex 8b420330..22537582 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -22,7 +22,7 @@ use optics::PatternPart;use tantivy::tokenizer::Tokenizer;use crate::{- fastfield_reader::FastFieldReader,+ columnfield_reader::ColumnFieldReader,schema::{text_field::TextField, Field, TextFieldEnum},};@@ -34,7 +34,7 @@ pub struct PatternQuery {can_optimize_site_domain: bool,field: tantivy::schema::Field,raw_terms: Vec<tantivy::Term>,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,}impl std::fmt::Debug for PatternQuery {@@ -52,7 +52,7 @@ impl PatternQuery {patterns: Vec<PatternPart>,field: TextFieldEnum,schema: &tantivy::schema::Schema,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,) -> Self {let field = Field::Text(field);let tv_field = schema.get_field(field.name()).unwrap();@@ -68,7 +68,7 @@ impl PatternQuery {field: tv_field,can_optimize_site_domain: true,raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],- fastfield_reader,+ columnfield_reader,};} else {let term: String = patterns@@ -85,7 +85,7 @@ impl PatternQuery {field: tv_field,can_optimize_site_domain: true,raw_terms: vec![tantivy::Term::from_field_text(tv_field, &term)],- fastfield_reader,+ columnfield_reader,};}}@@ -120,7 +120,7 @@ impl PatternQuery {field: tv_field,raw_terms,can_optimize_site_domain: false,- fastfield_reader,+ columnfield_reader,}}}@@ -141,7 +141,7 @@ impl tantivy::query::Query for PatternQuery {raw_terms: self.raw_terms.clone(),patterns: self.patterns.clone(),field: self.field,- fastfield_reader: self.fastfield_reader.clone(),+ columnfield_reader: self.columnfield_reader.clone(),}))}diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex fbd26af2..459cb0b5 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -23,9 +23,9 @@ use tantivy::{};use crate::{- fastfield_reader::{self, FastFieldReader},+ columnfield_reader::{self, ColumnFieldReader},query::intersection::Intersection,- schema::FastFieldEnum,+ schema::ColumnFieldEnum,};use super::SmallPatternPart;@@ -126,8 +126,8 @@ impl Scorer for AllScorer {}pub struct EmptyFieldScorer {- pub segment_reader: Arc<fastfield_reader::SegmentReader>,- pub num_tokens_fastfield: FastFieldEnum,+ pub segment_reader: Arc<columnfield_reader::SegmentReader>,+ pub num_tokens_columnfield: ColumnFieldEnum,pub all_scorer: AllScorer,}@@ -136,7 +136,7 @@ impl EmptyFieldScorer {let s: Option<u64> = self.segment_reader.get_field_reader(doc)- .get(self.num_tokens_fastfield)+ .get(self.num_tokens_columnfield).and_then(|v| v.as_u64());s.unwrap_or_default()}@@ -210,8 +210,8 @@ pub struct NormalPatternScorer {left: Vec<u32>,right: Vec<u32>,phrase_count: u32,- num_tokens_field: FastFieldEnum,- segment_reader: Arc<fastfield_reader::SegmentReader>,+ num_tokens_field: ColumnFieldEnum,+ segment_reader: Arc<columnfield_reader::SegmentReader>,}impl NormalPatternScorer {@@ -219,11 +219,11 @@ impl NormalPatternScorer {term_postings_list: Vec<SegmentPostings>,pattern: Vec<SmallPatternPart>,segment: tantivy::index::SegmentId,- num_tokens_field: FastFieldEnum,- fastfield_reader: FastFieldReader,+ num_tokens_field: ColumnFieldEnum,+ columnfield_reader: ColumnFieldReader,) -> Self {let num_query_terms = term_postings_list.len();- let segment_reader = fastfield_reader.get_segment(&segment);+ let segment_reader = columnfield_reader.get_segment(&segment);let mut s = Self {pattern_all_simple: pattern.iter().all(|p| matches!(p, SmallPatternPart::Term)),diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rsindex 7b94bb7a..e8f93aed 100644--- a/crates/core/src/query/pattern_query/weight.rs+++ b/crates/core/src/query/pattern_query/weight.rs@@ -22,9 +22,9 @@ use tantivy::{};use crate::{- fastfield_reader::FastFieldReader,+ columnfield_reader::ColumnFieldReader,schema::{- fast_field,+ column_field,text_field::{self, TextField},Field, TextFieldEnum,},@@ -62,7 +62,7 @@ impl FastSiteDomainPatternWeight {let opt = match field_no_tokenizer {Field::Text(t) => t.record_option(),- Field::Fast(_) => unreachable!(),+ Field::Columnar(_) => unreachable!(),};match reader@@ -114,7 +114,7 @@ pub struct PatternWeight {pub patterns: Vec<PatternPart>,pub raw_terms: Vec<tantivy::Term>,pub field: tantivy::schema::Field,- pub fastfield_reader: FastFieldReader,+ pub columnfield_reader: ColumnFieldReader,}impl PatternWeight {@@ -126,24 +126,24 @@ impl PatternWeight {return Ok(None);}- let num_tokens_fastfield = match Field::get(self.field.field_id() as usize) {- Some(Field::Text(TextFieldEnum::Title(_))) => Ok(fast_field::NumTitleTokens.into()),+ let num_tokens_columnfield = match Field::get(self.field.field_id() as usize) {+ Some(Field::Text(TextFieldEnum::Title(_))) => Ok(column_field::NumTitleTokens.into()),Some(Field::Text(TextFieldEnum::CleanBody(_))) => {- Ok(fast_field::NumCleanBodyTokens.into())+ Ok(column_field::NumCleanBodyTokens.into())}- Some(Field::Text(TextFieldEnum::Url(_))) => Ok(fast_field::NumUrlTokens.into()),- Some(Field::Text(TextFieldEnum::Domain(_))) => Ok(fast_field::NumDomainTokens.into()),+ Some(Field::Text(TextFieldEnum::Url(_))) => Ok(column_field::NumUrlTokens.into()),+ Some(Field::Text(TextFieldEnum::Domain(_))) => Ok(column_field::NumDomainTokens.into()),Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_))) => {- Ok(fast_field::NumUrlForSiteOperatorTokens.into())+ Ok(column_field::NumUrlForSiteOperatorTokens.into())}Some(Field::Text(TextFieldEnum::Description(_))) => {- Ok(fast_field::NumDescriptionTokens.into())+ Ok(column_field::NumDescriptionTokens.into())}Some(Field::Text(TextFieldEnum::MicroformatTags(_))) => {- Ok(fast_field::NumMicroformatTagsTokens.into())+ Ok(column_field::NumMicroformatTagsTokens.into())}Some(Field::Text(TextFieldEnum::FlattenedSchemaOrgJson(_))) => {- Ok(fast_field::NumFlattenedSchemaTokens.into())+ Ok(column_field::NumFlattenedSchemaTokens.into())}Some(field) => Err(TantivyError::InvalidArgument(format!("{} is not supported in pattern query",@@ -177,8 +177,8 @@ impl PatternWeight {.all(|p| matches!(p, PatternPart::Anchor)){return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {- num_tokens_fastfield,- segment_reader: self.fastfield_reader.get_segment(&reader.segment_id()),+ num_tokens_columnfield,+ segment_reader: self.columnfield_reader.get_segment(&reader.segment_id()),all_scorer: AllScorer {doc: 0,max_doc: reader.max_doc(),@@ -212,8 +212,8 @@ impl PatternWeight {term_postings_list,small_patterns,reader.segment_id(),- num_tokens_fastfield,- self.fastfield_reader.clone(),+ num_tokens_columnfield,+ self.columnfield_reader.clone(),))))}}diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rsindex 9f45a44e..2f4264cb 100644--- a/crates/core/src/ranking/initial.rs+++ b/crates/core/src/ranking/initial.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::fastfield_reader::FastFieldReader;+use crate::columnfield_reader::ColumnFieldReader;use chrono::Utc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};@@ -24,7 +24,7 @@ use super::SignalComputer;pub struct InitialScoreTweaker {tv_searcher: tantivy::Searcher,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,computer: SignalComputer,}@@ -38,12 +38,12 @@ impl InitialScoreTweaker {pub fn new(tv_searcher: tantivy::Searcher,computer: SignalComputer,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,) -> Self {Self {tv_searcher,computer,- fastfield_reader,+ columnfield_reader,}}}@@ -58,7 +58,7 @@ impl ScoreTweaker<Score> for InitialScoreTweaker {computer.set_current_timestamp(current_timestamp);computer- .register_segment(&self.tv_searcher, segment_reader, &self.fastfield_reader)+ .register_segment(&self.tv_searcher, segment_reader, &self.columnfield_reader).unwrap();Ok(InitialSegmentScoreTweaker { computer })diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 0fc1d7d1..702f39b8 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -28,8 +28,8 @@ use initial::InitialScoreTweaker;use crate::{collector::{MainCollector, MaxDocsConsidered, TopDocs},+ columnfield_reader::ColumnFieldReader,config::CollectorConfig,- fastfield_reader::FastFieldReader,search_ctx::Ctx,searcher::NUM_RESULTS_PER_PAGE,};@@ -41,7 +41,7 @@ pub struct Ranker {max_docs: Option<MaxDocsConsidered>,offset: Option<usize>,computer: SignalComputer,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,de_rank_similar: bool,num_results: Option<usize>,collector_config: CollectorConfig,@@ -50,7 +50,7 @@ pub struct Ranker {impl Ranker {pub fn new(computer: SignalComputer,- fastfield_reader: FastFieldReader,+ columnfield_reader: ColumnFieldReader,collector_config: CollectorConfig,) -> Self {Ranker {@@ -58,7 +58,7 @@ impl Ranker {computer,max_docs: None,de_rank_similar: true,- fastfield_reader,+ columnfield_reader,num_results: None,collector_config,}@@ -94,11 +94,11 @@ impl Ranker {let computer = self.computer();let score_tweaker =- InitialScoreTweaker::new(ctx.tv_searcher, computer, self.fastfield_reader.clone());+ InitialScoreTweaker::new(ctx.tv_searcher, computer, self.columnfield_reader.clone());let mut collector = TopDocs::with_limit(self.num_results.unwrap_or(NUM_RESULTS_PER_PAGE),- self.fastfield_reader.clone(),+ self.columnfield_reader.clone(),);if self.de_rank_similar {diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex 2c068b5f..a3c87562 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -17,10 +17,9 @@use std::sync::Arc;use crate::{- collector,+ collector, columnfield_reader,config::CollectorConfig,enum_map::EnumMap,- fastfield_reader,inverted_index::WebpagePointer,models::dual_encoder::DualEncoder,ranking::{@@ -29,7 +28,7 @@ use crate::{pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},SignalComputer, SignalEnum,},- schema::fast_field,+ schema::column_field,searcher::{api, SearchQuery},webgraph,};@@ -155,21 +154,21 @@ impl LocalRecallRankingWebpage {/// the index to calculate bm25.pub fn new(pointer: WebpagePointer,- fastfield_reader: &fastfield_reader::SegmentReader,+ columnfield_reader: &columnfield_reader::SegmentReader,computer: &mut SignalComputer,) -> Self {- let fastfields = fastfield_reader.get_field_reader(pointer.address.doc_id);+ let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);- let title_embedding: Option<Vec<u8>> = fastfields- .get(fast_field::TitleEmbeddings.into())+ let title_embedding: Option<Vec<u8>> = columnfields+ .get(column_field::TitleEmbeddings.into()).and_then(|v| v.into());- let keyword_embedding: Option<Vec<u8>> = fastfields- .get(fast_field::KeywordEmbeddings.into())+ let keyword_embedding: Option<Vec<u8>> = columnfields+ .get(column_field::KeywordEmbeddings.into()).and_then(|v| v.into());- let host_id = fastfields- .get(fast_field::HostNodeID.into())+ let host_id = columnfields+ .get(column_field::HostNodeID.into()).unwrap().as_u64().unwrap()diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex ecf36c07..68ddc197 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -19,7 +19,7 @@ use crate::query::{Query, MAX_TERMS_FOR_NGRAM_LOOKUPS};use crate::ranking::bm25f::MultiBm25FWeight;use crate::schema::text_field::TextField;use crate::Result;-use crate::{enum_map::EnumMap, fastfield_reader, schema::TextFieldEnum, webpage::Webpage};+use crate::{columnfield_reader, enum_map::EnumMap, schema::TextFieldEnum, webpage::Webpage};use std::cell::RefCell;@@ -121,7 +121,7 @@ pub struct OpticBoosts {pub struct SegmentReader {text_fields: EnumMap<TextFieldEnum, TextFieldData>,optic_boosts: OpticBoosts,- fastfield_reader: Arc<fastfield_reader::SegmentReader>,+ columnfield_reader: Arc<columnfield_reader::SegmentReader>,}impl SegmentReader {@@ -129,8 +129,8 @@ impl SegmentReader {&mut self.text_fields}- pub fn fastfield_reader(&self) -> &fastfield_reader::SegmentReader {- &self.fastfield_reader+ pub fn columnfield_reader(&self) -> &columnfield_reader::SegmentReader {+ &self.columnfield_reader}}@@ -322,7 +322,7 @@ impl SignalComputer {&self,tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,- fastfield_reader: &fastfield_reader::FastFieldReader,+ columnfield_reader: &columnfield_reader::ColumnFieldReader,) -> Vec<RuleBoost> {let mut optic_rule_boosts = Vec::new();@@ -330,7 +330,9 @@ impl SignalComputer {optic_rule_boosts = query.optic_rules.iter()- .filter_map(|rule| rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader))+ .filter_map(|rule| {+ rule.as_searchable_rule(tv_searcher.schema(), columnfield_reader)+ }).map(|(_, rule)| RuleBoost {docset: rule.query@@ -353,15 +355,16 @@ impl SignalComputer {&mut self,tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,- fastfield_reader: &fastfield_reader::FastFieldReader,+ columnfield_reader: &columnfield_reader::ColumnFieldReader,) -> Result<()> {- let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());+ let columnfield_segment_reader =+ columnfield_reader.get_segment(&segment_reader.segment_id());let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;- let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);+ let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, columnfield_reader);self.segment_reader = Some(RefCell::new(SegmentReader {text_fields,- fastfield_reader: fastfield_segment_reader,+ columnfield_reader: columnfield_segment_reader,optic_boosts: OpticBoosts {rules: optic_rule_boosts,},diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex 2c9a6fb4..18228b2d 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -105,7 +105,7 @@ impl Signal for HostCentrality {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::HostCentrality.into()))+ Some(Field::Columnar(schema::column_field::HostCentrality.into()))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -114,10 +114,10 @@ impl Signal for HostCentrality {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(val as f64 / FLOAT_SCALING as f64)@@ -143,7 +143,9 @@ impl Signal for HostCentralityRank {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::HostCentralityRank.into()))+ Some(Field::Columnar(+ schema::column_field::HostCentralityRank.into(),+ ))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -152,10 +154,10 @@ impl Signal for HostCentralityRank {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_rank(val as f64))@@ -181,7 +183,7 @@ impl Signal for PageCentrality {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::PageCentrality.into()))+ Some(Field::Columnar(schema::column_field::PageCentrality.into()))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -190,10 +192,10 @@ impl Signal for PageCentrality {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(val as f64 / FLOAT_SCALING as f64)@@ -219,7 +221,9 @@ impl Signal for PageCentralityRank {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::PageCentralityRank.into()))+ Some(Field::Columnar(+ schema::column_field::PageCentralityRank.into(),+ ))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -228,10 +232,10 @@ impl Signal for PageCentralityRank {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_rank(val as f64))@@ -257,7 +261,7 @@ impl Signal for IsHomepage {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::IsHomepage.into()))+ Some(Field::Columnar(schema::column_field::IsHomepage.into()))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -266,10 +270,10 @@ impl Signal for IsHomepage {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(val as f64)@@ -295,7 +299,7 @@ impl Signal for FetchTimeMs {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::FetchTimeMs.into()))+ Some(Field::Columnar(schema::column_field::FetchTimeMs.into()))}fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {@@ -309,10 +313,10 @@ impl Signal for FetchTimeMs {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let fetch_time_ms = fastfield_reader- .get(self.as_fastfield().unwrap())+ let fetch_time_ms = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap() as usize;@@ -343,7 +347,7 @@ impl Signal for UpdateTimestamp {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::LastUpdated.into()))+ Some(Field::Columnar(schema::column_field::LastUpdated.into()))}fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {@@ -358,10 +362,10 @@ impl Signal for UpdateTimestamp {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap() as usize;@@ -388,7 +392,7 @@ impl Signal for TrackerScore {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::TrackerScore.into()))+ Some(Field::Columnar(schema::column_field::TrackerScore.into()))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -398,10 +402,10 @@ impl Signal for TrackerScore {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_trackers(val as f64))@@ -427,7 +431,7 @@ impl Signal for Region {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::Region.into()))+ Some(Field::Columnar(schema::column_field::Region.into()))}fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {@@ -438,10 +442,10 @@ impl Signal for Region {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();let region = crate::webpage::Region::from_id(val);@@ -549,8 +553,8 @@ impl Signal for UrlDigits {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(- schema::fast_field::NumPathAndQueryDigits.into(),+ Some(Field::Columnar(+ schema::column_field::NumPathAndQueryDigits.into(),))}@@ -576,10 +580,10 @@ impl Signal for UrlDigits {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_digits(val as f64))@@ -605,8 +609,8 @@ impl Signal for UrlSlashes {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(- schema::fast_field::NumPathAndQuerySlashes.into(),+ Some(Field::Columnar(+ schema::column_field::NumPathAndQuerySlashes.into(),))}@@ -623,10 +627,10 @@ impl Signal for UrlSlashes {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_slashes(val as f64))@@ -652,7 +656,7 @@ impl Signal for LinkDensity {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::LinkDensity.into()))+ Some(Field::Columnar(schema::column_field::LinkDensity.into()))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -662,10 +666,10 @@ impl Signal for LinkDensity {fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let columnfield_reader = seg_reader.columnfield_reader().get_field_reader(doc);- let val = fastfield_reader- .get(self.as_fastfield().unwrap())+ let val = columnfield_reader+ .get(self.as_columnfield().unwrap()).and_then(|v| v.as_u64()).unwrap();Some(score_link_density(val as f64 / FLOAT_SCALING as f64))diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rsindex 5c639d21..260e04e0 100644--- a/crates/core/src/ranking/signal/prelude.rs+++ b/crates/core/src/ranking/signal/prelude.rs@@ -18,7 +18,7 @@ use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};use crate::schema::Field;use crate::{- schema::{FastFieldEnum, TextFieldEnum},+ schema::{ColumnFieldEnum, TextFieldEnum},webpage::Webpage,};use enum_dispatch::enum_dispatch;@@ -46,7 +46,7 @@ pub trait Signal:self.as_field().and_then(|field| field.as_text())}- fn as_fastfield(&self) -> Option<FastFieldEnum> {+ fn as_columnfield(&self) -> Option<ColumnFieldEnum> {self.as_field().and_then(|field| field.as_fast())}}diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rsindex 5050d39a..ce1eedff 100644--- a/crates/core/src/ranking/signal/text.rs+++ b/crates/core/src/ranking/signal/text.rs@@ -806,7 +806,9 @@ impl Signal for TitleEmbeddingSimilarity {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::TitleEmbeddings.into()))+ Some(Field::Columnar(+ schema::column_field::TitleEmbeddings.into(),+ ))}fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {@@ -833,7 +835,9 @@ impl Signal for KeywordEmbeddingSimilarity {}fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::KeywordEmbeddings.into()))+ Some(Field::Columnar(+ schema::column_field::KeywordEmbeddings.into(),+ ))}fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {diff --git a/crates/core/src/schema/fast_field.rs b/crates/core/src/schema/column_field.rssimilarity index 91%rename from crates/core/src/schema/fast_field.rsrename to crates/core/src/schema/column_field.rsindex 29c0394e..a31ff09b 100644--- a/crates/core/src/schema/fast_field.rs+++ b/crates/core/src/schema/column_field.rs@@ -32,7 +32,7 @@ use crate::{use super::{IndexingOption, FLOAT_SCALING};#[enum_dispatch]-pub trait FastField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {+pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {fn name(&self) -> &str;fn add_html_tantivy(&self,@@ -66,7 +66,7 @@ pub trait FastField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hashfn indexing_option(&self) -> IndexingOption {debug_assert!(matches!(self.data_type(), DataType::U64));- let mut opt = NumericOptions::default().set_fast();+ let mut opt = NumericOptions::default().set_columnar();if self.is_stored() {opt = opt.set_stored();@@ -86,10 +86,10 @@ pub trait FastField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash}}-#[enum_dispatch(FastField)]+#[enum_dispatch(ColumnField)]#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants)]#[strum_discriminants(derive(VariantArray))]-pub enum FastFieldEnum {+pub enum ColumnFieldEnum {IsHomepage,HostCentrality,HostCentralityRank,@@ -131,7 +131,7 @@ pub enum FastFieldEnum {KeywordEmbeddings,}-enum_dispatch_from_discriminant!(FastFieldEnumDiscriminants => FastFieldEnum,+enum_dispatch_from_discriminant!(ColumnFieldEnumDiscriminants => ColumnFieldEnum,[IsHomepage,HostCentrality,@@ -174,23 +174,23 @@ enum_dispatch_from_discriminant!(FastFieldEnumDiscriminants => FastFieldEnum,KeywordEmbeddings,]);-impl FastFieldEnum {- pub fn all() -> impl Iterator<Item = FastFieldEnum> {- FastFieldEnumDiscriminants::VARIANTS+impl ColumnFieldEnum {+ pub fn all() -> impl Iterator<Item = ColumnFieldEnum> {+ ColumnFieldEnumDiscriminants::VARIANTS.iter().copied().map(|v| v.into())}- pub fn get(field_id: usize) -> Option<FastFieldEnum> {- FastFieldEnumDiscriminants::VARIANTS+ pub fn get(field_id: usize) -> Option<ColumnFieldEnum> {+ ColumnFieldEnumDiscriminants::VARIANTS.get(field_id).copied()- .map(FastFieldEnum::from)+ .map(ColumnFieldEnum::from)}pub fn num_variants() -> usize {- FastFieldEnumDiscriminants::VARIANTS.len()+ ColumnFieldEnumDiscriminants::VARIANTS.len()}}@@ -199,15 +199,15 @@ pub enum DataType {Bytes,}-impl InsertEnumMapKey for FastFieldEnum {+impl InsertEnumMapKey for ColumnFieldEnum {fn into_usize(self) -> usize {- FastFieldEnumDiscriminants::from(self) as usize+ ColumnFieldEnumDiscriminants::from(self) as usize}}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct IsHomepage;-impl FastField for IsHomepage {+impl ColumnField for IsHomepage {fn name(&self) -> &str {"is_homepage"}@@ -227,7 +227,7 @@ impl FastField for IsHomepage {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct HostCentrality;-impl FastField for HostCentrality {+impl ColumnField for HostCentrality {fn name(&self) -> &str {"host_centrality"}@@ -259,7 +259,7 @@ impl FastField for HostCentrality {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct HostCentralityRank;-impl FastField for HostCentralityRank {+impl ColumnField for HostCentralityRank {fn name(&self) -> &str {"host_centrality_rank"}@@ -288,7 +288,7 @@ impl FastField for HostCentralityRank {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct PageCentrality;-impl FastField for PageCentrality {+impl ColumnField for PageCentrality {fn name(&self) -> &str {"page_centrality"}@@ -319,7 +319,7 @@ impl FastField for PageCentrality {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct PageCentralityRank;-impl FastField for PageCentralityRank {+impl ColumnField for PageCentralityRank {fn name(&self) -> &str {"page_centrality_rank"}@@ -347,7 +347,7 @@ impl FastField for PageCentralityRank {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct FetchTimeMs;-impl FastField for FetchTimeMs {+impl ColumnField for FetchTimeMs {fn name(&self) -> &str {"fetch_time_ms"}@@ -375,7 +375,7 @@ impl FastField for FetchTimeMs {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LastUpdated;-impl FastField for LastUpdated {+impl ColumnField for LastUpdated {fn name(&self) -> &str {"last_updated"}@@ -403,7 +403,7 @@ impl FastField for LastUpdated {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TrackerScore;-impl FastField for TrackerScore {+impl ColumnField for TrackerScore {fn name(&self) -> &str {"tracker_score"}@@ -423,7 +423,7 @@ impl FastField for TrackerScore {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct Region;-impl FastField for Region {+impl ColumnField for Region {fn name(&self) -> &str {"region"}@@ -464,7 +464,7 @@ impl FastField for Region {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumUrlTokens;-impl FastField for NumUrlTokens {+impl ColumnField for NumUrlTokens {fn name(&self) -> &str {"num_url_tokens"}@@ -487,7 +487,7 @@ impl FastField for NumUrlTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumTitleTokens;-impl FastField for NumTitleTokens {+impl ColumnField for NumTitleTokens {fn name(&self) -> &str {"num_title_tokens"}@@ -514,7 +514,7 @@ impl FastField for NumTitleTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumCleanBodyTokens;-impl FastField for NumCleanBodyTokens {+impl ColumnField for NumCleanBodyTokens {fn name(&self) -> &str {"num_clean_body_tokens"}@@ -536,7 +536,7 @@ impl FastField for NumCleanBodyTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumDescriptionTokens;-impl FastField for NumDescriptionTokens {+impl ColumnField for NumDescriptionTokens {fn name(&self) -> &str {"num_description_tokens"}@@ -559,7 +559,7 @@ impl FastField for NumDescriptionTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumUrlForSiteOperatorTokens;-impl FastField for NumUrlForSiteOperatorTokens {+impl ColumnField for NumUrlForSiteOperatorTokens {fn name(&self) -> &str {"num_url_for_site_operator_tokens"}@@ -582,7 +582,7 @@ impl FastField for NumUrlForSiteOperatorTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumDomainTokens;-impl FastField for NumDomainTokens {+impl ColumnField for NumDomainTokens {fn name(&self) -> &str {"num_domain_tokens"}@@ -605,7 +605,7 @@ impl FastField for NumDomainTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumMicroformatTagsTokens;-impl FastField for NumMicroformatTagsTokens {+impl ColumnField for NumMicroformatTagsTokens {fn name(&self) -> &str {"num_microformat_tags_tokens"}@@ -628,7 +628,7 @@ impl FastField for NumMicroformatTagsTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct SiteHash1;-impl FastField for SiteHash1 {+impl ColumnField for SiteHash1 {fn name(&self) -> &str {"site_hash1"}@@ -648,7 +648,7 @@ impl FastField for SiteHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct SiteHash2;-impl FastField for SiteHash2 {+impl ColumnField for SiteHash2 {fn name(&self) -> &str {"site_hash2"}@@ -668,7 +668,7 @@ impl FastField for SiteHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutQueryHash1;-impl FastField for UrlWithoutQueryHash1 {+impl ColumnField for UrlWithoutQueryHash1 {fn name(&self) -> &str {"url_without_query_hash1"}@@ -691,7 +691,7 @@ impl FastField for UrlWithoutQueryHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutQueryHash2;-impl FastField for UrlWithoutQueryHash2 {+impl ColumnField for UrlWithoutQueryHash2 {fn name(&self) -> &str {"url_without_query_hash2"}@@ -714,7 +714,7 @@ impl FastField for UrlWithoutQueryHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TitleHash1;-impl FastField for TitleHash1 {+impl ColumnField for TitleHash1 {fn name(&self) -> &str {"title_hash1"}@@ -734,7 +734,7 @@ impl FastField for TitleHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TitleHash2;-impl FastField for TitleHash2 {+impl ColumnField for TitleHash2 {fn name(&self) -> &str {"title_hash2"}@@ -754,7 +754,7 @@ impl FastField for TitleHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlHash1;-impl FastField for UrlHash1 {+impl ColumnField for UrlHash1 {fn name(&self) -> &str {"url_hash1"}@@ -774,7 +774,7 @@ impl FastField for UrlHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlHash2;-impl FastField for UrlHash2 {+impl ColumnField for UrlHash2 {fn name(&self) -> &str {"url_hash2"}@@ -794,7 +794,7 @@ impl FastField for UrlHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct DomainHash1;-impl FastField for DomainHash1 {+impl ColumnField for DomainHash1 {fn name(&self) -> &str {"domain_hash1"}@@ -814,7 +814,7 @@ impl FastField for DomainHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct DomainHash2;-impl FastField for DomainHash2 {+impl ColumnField for DomainHash2 {fn name(&self) -> &str {"domain_hash2"}@@ -834,7 +834,7 @@ impl FastField for DomainHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutTldHash1;-impl FastField for UrlWithoutTldHash1 {+impl ColumnField for UrlWithoutTldHash1 {fn name(&self) -> &str {"url_without_tld_hash1"}@@ -854,7 +854,7 @@ impl FastField for UrlWithoutTldHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutTldHash2;-impl FastField for UrlWithoutTldHash2 {+impl ColumnField for UrlWithoutTldHash2 {fn name(&self) -> &str {"url_without_tld_hash2"}@@ -874,7 +874,7 @@ impl FastField for UrlWithoutTldHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct PreComputedScore;-impl FastField for PreComputedScore {+impl ColumnField for PreComputedScore {fn name(&self) -> &str {"pre_computed_score"}@@ -910,7 +910,7 @@ impl FastField for PreComputedScore {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct HostNodeID;-impl FastField for HostNodeID {+impl ColumnField for HostNodeID {fn name(&self) -> &str {"host_node_id"}@@ -950,7 +950,7 @@ impl FastField for HostNodeID {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct SimHash;-impl FastField for SimHash {+impl ColumnField for SimHash {fn name(&self) -> &str {"sim_hash"}@@ -981,7 +981,7 @@ impl FastField for SimHash {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumFlattenedSchemaTokens;-impl FastField for NumFlattenedSchemaTokens {+impl ColumnField for NumFlattenedSchemaTokens {fn name(&self) -> &str {"num_flattened_schema_tokens"}@@ -1004,7 +1004,7 @@ impl FastField for NumFlattenedSchemaTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumPathAndQuerySlashes;-impl FastField for NumPathAndQuerySlashes {+impl ColumnField for NumPathAndQuerySlashes {fn name(&self) -> &str {"num_path_and_query_slashes"}@@ -1034,7 +1034,7 @@ impl FastField for NumPathAndQuerySlashes {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumPathAndQueryDigits;-impl FastField for NumPathAndQueryDigits {+impl ColumnField for NumPathAndQueryDigits {fn name(&self) -> &str {"num_path_and_query_digits"}@@ -1072,7 +1072,7 @@ impl FastField for NumPathAndQueryDigits {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LikelyHasAds;-impl FastField for LikelyHasAds {+impl ColumnField for LikelyHasAds {fn name(&self) -> &str {"likely_has_ads"}@@ -1096,7 +1096,7 @@ impl FastField for LikelyHasAds {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LikelyHasPaywall;-impl FastField for LikelyHasPaywall {+impl ColumnField for LikelyHasPaywall {fn name(&self) -> &str {"likely_has_paywall"}@@ -1120,7 +1120,7 @@ impl FastField for LikelyHasPaywall {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LinkDensity;-impl FastField for LinkDensity {+impl ColumnField for LinkDensity {fn name(&self) -> &str {"link_density"}@@ -1147,7 +1147,7 @@ impl FastField for LinkDensity {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TitleEmbeddings;-impl FastField for TitleEmbeddings {+impl ColumnField for TitleEmbeddings {fn name(&self) -> &str {"title_embeddings"}@@ -1157,7 +1157,7 @@ impl FastField for TitleEmbeddings {}fn indexing_option(&self) -> IndexingOption {- IndexingOption::Bytes(BytesOptions::default().set_fast().set_stored())+ IndexingOption::Bytes(BytesOptions::default().set_columnar().set_stored())}fn add_html_tantivy(@@ -1191,7 +1191,7 @@ impl FastField for TitleEmbeddings {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct KeywordEmbeddings;-impl FastField for KeywordEmbeddings {+impl ColumnField for KeywordEmbeddings {fn name(&self) -> &str {"keyword_embeddings"}@@ -1201,7 +1201,7 @@ impl FastField for KeywordEmbeddings {}fn indexing_option(&self) -> IndexingOption {- IndexingOption::Bytes(BytesOptions::default().set_fast().set_stored())+ IndexingOption::Bytes(BytesOptions::default().set_columnar().set_stored())}fn add_html_tantivy(diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rsindex f417e1c8..ba34fa75 100644--- a/crates/core/src/schema/mod.rs+++ b/crates/core/src/schema/mod.rs@@ -14,21 +14,21 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub mod fast_field;+pub mod column_field;pub mod text_field;use tantivy::schema::{BytesOptions, DateOptions, NumericOptions, TextOptions};-pub use fast_field::{DataType, FastFieldEnum};+pub use column_field::{ColumnFieldEnum, DataType};pub use text_field::TextFieldEnum;-use self::{fast_field::FastField, text_field::TextField};+use self::{column_field::ColumnField, text_field::TextField};pub const FLOAT_SCALING: u64 = 1_000_000_000;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Field {- Fast(FastFieldEnum),+ Columnar(ColumnFieldEnum),Text(TextFieldEnum),}@@ -40,10 +40,10 @@ impl Field {}let field_id = field_id - TextFieldEnum::num_variants();- if field_id < FastFieldEnum::num_variants() {- return Some(Field::Fast(FastFieldEnum::get(field_id).unwrap()));+ if field_id < ColumnFieldEnum::num_variants() {+ return Some(Field::Columnar(ColumnFieldEnum::get(field_id).unwrap()));}- let _field_id = field_id - FastFieldEnum::num_variants();+ let _field_id = field_id - ColumnFieldEnum::num_variants();None}@@ -52,12 +52,12 @@ impl Field {pub fn all() -> impl Iterator<Item = Field> {TextFieldEnum::all().map(Field::Text)- .chain(FastFieldEnum::all().map(Field::Fast))+ .chain(ColumnFieldEnum::all().map(Field::Columnar))}pub fn has_pos(&self) -> bool {match self {- Field::Fast(_) => false,+ Field::Columnar(_) => false,Field::Text(text) => text.has_pos(),}}@@ -65,34 +65,34 @@ impl Field {pub fn indexing_option(&self) -> IndexingOption {match self {Field::Text(f) => f.indexing_option(),- Field::Fast(f) => f.indexing_option(),+ Field::Columnar(f) => f.indexing_option(),}}pub fn name(&self) -> &str {match self {Field::Text(f) => f.name(),- Field::Fast(f) => f.name(),+ Field::Columnar(f) => f.name(),}}pub fn is_searchable(&self) -> bool {match self {Field::Text(f) => f.is_searchable(),- Field::Fast(_) => false,+ Field::Columnar(_) => false,}}pub fn as_text(&self) -> Option<TextFieldEnum> {match self {- Field::Fast(_) => None,+ Field::Columnar(_) => None,Field::Text(field) => Some(*field),}}- pub fn as_fast(&self) -> Option<FastFieldEnum> {+ pub fn as_fast(&self) -> Option<ColumnFieldEnum> {match self {- Field::Fast(field) => Some(*field),+ Field::Columnar(field) => Some(*field),Field::Text(_) => None,}}diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rsindex 2bb3af57..d0f768ae 100644--- a/crates/core/src/search_ctx.rs+++ b/crates/core/src/search_ctx.rs@@ -14,10 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::fastfield_reader::FastFieldReader;+use crate::columnfield_reader::ColumnFieldReader;#[derive(Clone)]pub struct Ctx {pub tv_searcher: tantivy::Searcher,- pub fastfield_reader: FastFieldReader,+ pub columnfield_reader: ColumnFieldReader,}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex f1b25414..433e927c 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -176,7 +176,7 @@ where) -> Result<Ranker> {let mut ranker = Ranker::new(computer,- guard.inverted_index().fastfield_reader(),+ guard.inverted_index().columnfield_reader(),self.collector_config.clone(),);@@ -232,13 +232,13 @@ whereranker.collector(ctx.clone()),)?;- let fastfield_reader = guard.inverted_index().fastfield_reader();+ let columnfield_reader = guard.inverted_index().columnfield_reader();let ranking_websites = guard.inverted_index().retrieve_ranking_websites(ctx,res.top_websites,ranker.computer(),- &fastfield_reader,+ &columnfield_reader,)?;let pipe_top_n = pipeline.top_n;diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex b26bdf7d..bfaa0235 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -19,7 +19,7 @@ use crate::{prehashed::hash,rake::RakeModel,schema::{- fast_field::FastField,+ column_field::ColumnField,text_field::{self, TextField},TextFieldEnum,},@@ -201,7 +201,7 @@ impl Html {{match field {Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,- Field::Fast(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,+ Field::Columnar(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,}}diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex a84f2d56..3f88b2ab 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- schema::{fast_field::FastField, text_field::TextField, Field},+ schema::{column_field::ColumnField, text_field::TextField, Field},webgraph::NodeID,Result,};@@ -134,7 +134,7 @@ impl Webpage {for field in Field::all() {match field {- Field::Fast(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,+ Field::Columnar(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,}}diff --git a/crates/tantivy/doc/src/index_sorting.md b/crates/tantivy/doc/src/index_sorting.mdindex c2c430f8..2a6c4a79 100644--- a/crates/tantivy/doc/src/index_sorting.md+++ b/crates/tantivy/doc/src/index_sorting.md@@ -4,8 +4,9 @@- [Compression](#compression)- [Top-N Optimization](#top-n-optimization)- [Pruning](#pruning)- - [Other](#other)+ - [Other?](#other)- [Usage](#usage)+ - [Implementation details](#implementation-details)# Index Sorting@@ -19,7 +20,7 @@ Presorting an index has several advantages:When data is sorted it is easier to compress the data. E.g. the numbers sequence [5, 2, 3, 1, 4] would be sorted to [1, 2, 3, 4, 5].If we apply delta encoding this list would be unsorted [5, -3, 1, -2, 3] vs. [1, 1, 1, 1, 1].-Compression ratio is mainly affected on the fast field of the sorted property, every thing else is likely unaffected.+Compression ratio is mainly affected on the columnar field of the sorted property, every thing else is likely unaffected.### Top-N Optimization@@ -30,7 +31,7 @@ Note: Tantivy 0.16 does not do this optimization yet.### Pruning-Let's say we want all documents and want to apply the filter `>= 2010-08-11`. When the data is sorted, we could make a lookup in the fast field to find the docid range and use this as the filter.+Let's say we want all documents and want to apply the filter `>= 2010-08-11`. When the data is sorted, we could make a lookup in the columnar field to find the docid range and use this as the filter.Note: Tantivy 0.16 does not do this optimization yet.@@ -40,7 +41,7 @@ In principle there are many algorithms possible that exploit the monotonically i## Usage-The index sorting can be configured setting [`sort_by_field`](https://github.com/quickwit-oss/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/core/index_meta.rs#L238) on `IndexSettings` and passing it to a `IndexBuilder`. As of Tantivy 0.16 only fast fields are allowed to be used.+The index sorting can be configured setting [`sort_by_field`](https://github.com/quickwit-oss/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/core/index_meta.rs#L238) on `IndexSettings` and passing it to a `IndexBuilder`. As of Tantivy 0.16 only columnar fields are allowed to be used.```rustlet settings = IndexSettings {@@ -59,4 +60,4 @@ let index = index_builder.create_in_ram().unwrap();Sorting an index is applied in the serialization step. In general there are two serialization steps: [Finishing a single segment](https://github.com/quickwit-oss/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/indexer/segment_writer.rs#L338) and [merging multiple segments](https://github.com/quickwit-oss/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/indexer/merger.rs#L1073).-In both cases we generate a docid mapping reflecting the sort. This mapping is used when serializing the different components (doc store, fastfields, posting list, normfield, facets).+In both cases we generate a docid mapping reflecting the sort. This mapping is used when serializing the different components (doc store, columnfields, posting list, normfield, facets).diff --git a/crates/tantivy/src/collector/filter_collector_wrapper.rs b/crates/tantivy/src/collector/filter_collector_wrapper.rsindex e68ffe37..c7b78e4a 100644--- a/crates/tantivy/src/collector/filter_collector_wrapper.rs+++ b/crates/tantivy/src/collector/filter_collector_wrapper.rs@@ -2,7 +2,7 @@//// This example shows how you can implement your own// collector. As an example, we will compute a collector-// that computes the standard deviation of a given fast field.+// that computes the standard deviation of a given columnar field.//// Of course, you can have a look at the tantivy's built-in collectors// such as the `CountCollector` for more examples.@@ -14,7 +14,7 @@ use crate::columnar::{BytesColumn, Column, DynamicColumn, HasAssociatedColumnTypuse crate::collector::{Collector, SegmentCollector};use crate::{DocId, Score, SegmentReader};-/// The `FilterCollector` filters docs using a fast field value and a predicate.+/// The `FilterCollector` filters docs using a columnar field value and a predicate.////// Only the documents containing at least one value for which the predicate returns `true`/// will be passed on to the next collector.@@ -23,9 +23,9 @@ use crate::{DocId, Score, SegmentReader};/// - documents with no values are filtered out./// - documents with several values are accepted if at least one value matches the predicate.///-/// Note that this is limited to fast fields which implement the-/// [`FastValue`][crate::fastfield::FastValue] trait, e.g. `u64` but not `&[u8]`.-/// To filter based on a bytes fast field, use a [`BytesFilterCollector`] instead.+/// Note that this is limited to columnar fields which implement the+/// [`FastValue`][crate::columnfield::FastValue] trait, e.g. `u64` but not `&[u8]`.+/// To filter based on a bytes columnar field, use a [`BytesFilterCollector`] instead.pub struct FilterCollector<TCollector, TPredicate, TPredicateValue>whereTPredicate: 'static + Clone,@@ -70,7 +70,7 @@ wheresegment_local_id: u32,segment_reader: &SegmentReader,) -> crate::Result<Self::Child> {- let column_opt = segment_reader.fast_fields().column_opt(&self.field)?;+ let column_opt = segment_reader.column_fields().column_opt(&self.field)?;let segment_collector = self.collector@@ -142,9 +142,9 @@ where}}-/// A variant of the [`FilterCollector`] specialized for bytes fast fields, i.e.+/// A variant of the [`FilterCollector`] specialized for bytes columnar fields, i.e./// it transparently wraps an inner [`Collector`] but filters documents-/// based on the result of applying the predicate to the bytes fast field.+/// based on the result of applying the predicate to the bytes columnar field.////// A document is accepted if and only if the predicate returns `true` for at least one value.///@@ -189,7 +189,7 @@ wheresegment_local_id: u32,segment_reader: &SegmentReader,) -> crate::Result<Self::Child> {- let column_opt = segment_reader.fast_fields().bytes(&self.field)?;+ let column_opt = segment_reader.column_fields().bytes(&self.field)?;let segment_collector = self.collectordiff --git a/crates/tantivy/src/collector/mod.rs b/crates/tantivy/src/collector/mod.rsindex bd458555..6ab60e14 100644--- a/crates/tantivy/src/collector/mod.rs+++ b/crates/tantivy/src/collector/mod.rs@@ -5,7 +5,7 @@//!//! Your fruit could for instance be ://! - [the count of matching documents](crate::collector::Count)-//! - [the top 10 documents, by relevancy or by a fast field](crate::collector::TopDocs)+//! - [the top 10 documents, by relevancy or by a columnar field](crate::collector::TopDocs)//! - [facet counts](FacetCollector)//!//! At some point in your code, you will trigger the actual search operation by calling@@ -93,7 +93,7 @@ impl<T> Fruit for T where T: Send + downcast_rs::Downcast {}/// For instance,////// - keeping track of the top 10 best documents-/// - computing a breakdown over a fast field+/// - computing a breakdown over a columnar field/// - computing the number of documents matching the query////// Our search index is in fact a collection of segments, sodiff --git a/crates/tantivy/src/collector/tests.rs b/crates/tantivy/src/collector/tests.rsindex ee3aa8bd..a0d4f361 100644--- a/crates/tantivy/src/collector/tests.rs+++ b/crates/tantivy/src/collector/tests.rs@@ -2,7 +2,7 @@ use crate::columnar::{BytesColumn, Column};use super::*;use crate::query::{AllQuery, QueryParser};-use crate::schema::{Schema, FAST, TEXT};+use crate::schema::{Schema, COLUMN, TEXT};use crate::time::format_description::well_known::Rfc3339;use crate::time::OffsetDateTime;use crate::{DateTime, DocAddress, Index, Searcher, TantivyDocument};@@ -19,8 +19,8 @@ pub const TEST_COLLECTOR_WITHOUT_SCORE: TestCollector = TestCollector {pub fn test_filter_collector() -> crate::Result<()> {let mut schema_builder = Schema::builder();let title = schema_builder.add_text_field("title", TEXT);- let price = schema_builder.add_u64_field("price", FAST);- let date = schema_builder.add_date_field("date", FAST);+ let price = schema_builder.add_u64_field("price", COLUMN);+ let date = schema_builder.add_date_field("date", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);@@ -149,41 +149,41 @@ impl SegmentCollector for TestSegmentCollector {}}-/// Collects in order all of the fast fields for all of the+/// Collects in order all of the columnar fields for all of the/// doc in the `DocSet`////// This collector is mainly useful for tests.-pub struct FastFieldTestCollector {+pub struct ColumnFieldTestCollector {field: String,}-pub struct FastFieldSegmentCollector {+pub struct ColumnFieldSegmentCollector {vals: Vec<u64>,reader: Column,}-impl FastFieldTestCollector {- pub fn for_field(field: impl ToString) -> FastFieldTestCollector {- FastFieldTestCollector {+impl ColumnFieldTestCollector {+ pub fn for_field(field: impl ToString) -> ColumnFieldTestCollector {+ ColumnFieldTestCollector {field: field.to_string(),}}}-impl Collector for FastFieldTestCollector {+impl Collector for ColumnFieldTestCollector {type Fruit = Vec<u64>;- type Child = FastFieldSegmentCollector;+ type Child = ColumnFieldSegmentCollector;fn for_segment(&self,_: SegmentOrdinal,segment_reader: &SegmentReader,- ) -> crate::Result<FastFieldSegmentCollector> {+ ) -> crate::Result<ColumnFieldSegmentCollector> {let reader = segment_reader- .fast_fields()+ .column_fields().u64(&self.field)- .expect("Requested field is not a fast field.");- Ok(FastFieldSegmentCollector {+ .expect("Requested field is not a columnar field.");+ Ok(ColumnFieldSegmentCollector {vals: Vec::new(),reader,})@@ -198,7 +198,7 @@ impl Collector for FastFieldTestCollector {}}-impl SegmentCollector for FastFieldSegmentCollector {+impl SegmentCollector for ColumnFieldSegmentCollector {type Fruit = Vec<u64>;fn collect(&mut self, doc: DocId, _score: Score) {@@ -210,40 +210,40 @@ impl SegmentCollector for FastFieldSegmentCollector {}}-/// Collects in order all of the fast field bytes for all of the+/// Collects in order all of the columnar field bytes for all of the/// docs in the `DocSet`////// This collector is mainly useful for tests./// It is very slow.-pub struct BytesFastFieldTestCollector {+pub struct BytesColumnFieldTestCollector {field: String,}-pub struct BytesFastFieldSegmentCollector {+pub struct BytesColumnFieldSegmentCollector {vals: Vec<u8>,column_opt: Option<BytesColumn>,buffer: Vec<u8>,}-impl BytesFastFieldTestCollector {- pub fn for_field(field: impl ToString) -> BytesFastFieldTestCollector {- BytesFastFieldTestCollector {+impl BytesColumnFieldTestCollector {+ pub fn for_field(field: impl ToString) -> BytesColumnFieldTestCollector {+ BytesColumnFieldTestCollector {field: field.to_string(),}}}-impl Collector for BytesFastFieldTestCollector {+impl Collector for BytesColumnFieldTestCollector {type Fruit = Vec<u8>;- type Child = BytesFastFieldSegmentCollector;+ type Child = BytesColumnFieldSegmentCollector;fn for_segment(&self,_segment_local_id: u32,segment_reader: &SegmentReader,- ) -> crate::Result<BytesFastFieldSegmentCollector> {- let column_opt = segment_reader.fast_fields().bytes(&self.field)?;- Ok(BytesFastFieldSegmentCollector {+ ) -> crate::Result<BytesColumnFieldSegmentCollector> {+ let column_opt = segment_reader.column_fields().bytes(&self.field)?;+ Ok(BytesColumnFieldSegmentCollector {vals: Vec::new(),column_opt,buffer: Vec::new(),@@ -259,7 +259,7 @@ impl Collector for BytesFastFieldTestCollector {}}-impl SegmentCollector for BytesFastFieldSegmentCollector {+impl SegmentCollector for BytesColumnFieldSegmentCollector {type Fruit = Vec<u8>;fn collect(&mut self, doc: DocId, _score: Score) {diff --git a/crates/tantivy/src/collector/top_score_collector.rs b/crates/tantivy/src/collector/top_score_collector.rsindex c8254d79..ac064fab 100644--- a/crates/tantivy/src/collector/top_score_collector.rs+++ b/crates/tantivy/src/collector/top_score_collector.rs@@ -13,26 +13,27 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;use crate::collector::{CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,};-use crate::fastfield::{FastFieldNotAvailableError, FastValue};+use crate::columnfield::{ColumnFieldNotAvailableError, ColumnarValue};use crate::query::Weight;use crate::{DocAddress, DocId, Order, Score, SegmentOrdinal, SegmentReader, TantivyError};-struct FastFieldConvertCollector<+struct ColumnFieldConvertCollector<TCollector: Collector<Fruit = Vec<(u64, DocAddress)>>,- TFastValue: FastValue,+ TColumnarValue: ColumnarValue,> {pub collector: TCollector,pub field: String,- pub fast_value: std::marker::PhantomData<TFastValue>,+ pub columnar_value: std::marker::PhantomData<TColumnarValue>,order: Order,}-impl<TCollector, TFastValue> Collector for FastFieldConvertCollector<TCollector, TFastValue>+impl<TCollector, TColumnarValue> Collector+ for ColumnFieldConvertCollector<TCollector, TColumnarValue>whereTCollector: Collector<Fruit = Vec<(u64, DocAddress)>>,- TFastValue: FastValue,+ TColumnarValue: ColumnarValue,{- type Fruit = Vec<(TFastValue, DocAddress)>;+ type Fruit = Vec<(TColumnarValue, DocAddress)>;type Child = TCollector::Child;@@ -44,13 +45,13 @@ wherelet schema = segment.schema();let field = schema.get_field(&self.field)?;let field_entry = schema.get_field_entry(field);- if !field_entry.is_fast() {+ if !field_entry.is_columnar() {return Err(TantivyError::SchemaError(format!(- "Field {:?} is not a fast field.",+ "Field {:?} is not a columnar field.",field_entry.name())));}- let schema_type = TFastValue::to_type();+ let schema_type = TColumnarValue::to_type();let requested_type = field_entry.field_type().value_type();if schema_type != requested_type {return Err(TantivyError::SchemaError(format!(@@ -74,9 +75,9 @@ where.into_iter().map(|(score, doc_address)| {if self.order.is_desc() {- (TFastValue::from_u64(score), doc_address)+ (TColumnarValue::from_u64(score), doc_address)} else {- (TFastValue::from_u64(u64::MAX - score), doc_address)+ (TColumnarValue::from_u64(u64::MAX - score), doc_address)}}).collect::<Vec<_>>();@@ -108,12 +109,12 @@ impl fmt::Debug for TopDocs {}}-struct ScorerByFastFieldReader {+struct ScorerByColumnFieldReader {sort_column: Arc<dyn ColumnValues<u64>>,order: Order,}-impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {+impl CustomSegmentScorer<u64> for ScorerByColumnFieldReader {fn score(&mut self, doc: DocId) -> u64 {let value = self.sort_column.get_val(doc);if self.order.is_desc() {@@ -130,20 +131,20 @@ struct ScorerByField {}impl CustomScorer<u64> for ScorerByField {- type Child = ScorerByFastFieldReader;+ type Child = ScorerByColumnFieldReader;fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {// We interpret this field as u64, regardless of its type, that way,- // we avoid needless conversion. Regardless of the fast field type, the+ // we avoid needless conversion. Regardless of the columnar field type, the// mapping is monotonic, so it is sufficient to compute our top-K docs.//// The conversion will then happen only on the top-K docs.- let sort_column_opt = segment_reader.fast_fields().u64_lenient(&self.field)?;+ let sort_column_opt = segment_reader.column_fields().u64_lenient(&self.field)?;let (sort_column, _sort_column_type) =- sort_column_opt.ok_or_else(|| FastFieldNotAvailableError {+ sort_column_opt.ok_or_else(|| ColumnFieldNotAvailableError {field_name: self.field.clone(),})?;- Ok(ScorerByFastFieldReader {+ Ok(ScorerByColumnFieldReader {sort_column: sort_column.values,order: self.order.clone(),})@@ -170,19 +171,19 @@ impl TopDocs {TopDocs(self.0.and_offset(offset))}- /// Set top-K to rank documents by a given fast field.+ /// Set top-K to rank documents by a given columnar field.///- /// If the field is not a fast or does not exist, this method returns successfully (it is not+ /// If the field is not a columnar or does not exist, this method returns successfully (it is not/// aware of any schema). An error will be returned at the moment of search.///- /// If the field is a FAST field but not a u64 field, search will return successfully but it+ /// If the field is a COLUMN field but not a u64 field, search will return successfully but it/// will return returns a monotonic u64-representation (ie. the order is still correct) of/// the requested field type.////// # See also////// To comfortably work with `u64`s, `i64`s, `f64`s, or `date`s, please refer to- /// the [.order_by_fast_field(...)](TopDocs::order_by_fast_field) method.+ /// the [.order_by_column_field(...)](TopDocs::order_by_column_field) method.pub fn order_by_u64_field(self,field: impl ToString,@@ -197,22 +198,22 @@ impl TopDocs {)}- /// Set top-K to rank documents by a given fast field.+ /// Set top-K to rank documents by a given columnar field.///- /// If the field is not a fast field, or its field type does not match the generic type, this+ /// If the field is not a columnar field, or its field type does not match the generic type, this/// method does not panic, but an explicit error will be returned at the moment of/// collection.///- /// Note that this method is a generic. The requested fast field type will be often+ /// Note that this method is a generic. The requested columnar field type will be often/// inferred in your code by the rust compiler.////// Implementation-wise, for performance reason, tantivy will manipulate the u64 representation- /// of your fast field until the last moment.+ /// of your columnar field until the last moment.////// # Example////// ```rust- /// # use tantivy::schema::{Schema, FAST, TEXT};+ /// # use tantivy::schema::{Schema, COLUMN, TEXT};/// # use tantivy::{doc, Index, DocAddress,Order};/// # use tantivy::query::{Query, AllQuery};/// use tantivy::Searcher;@@ -221,7 +222,7 @@ impl TopDocs {/// # fn main() -> tantivy::Result<()> {/// # let mut schema_builder = Schema::builder();/// # let title = schema_builder.add_text_field("company", TEXT);- /// # let revenue = schema_builder.add_i64_field("revenue", FAST);+ /// # let revenue = schema_builder.add_i64_field("revenue", COLUMN);/// # let schema = schema_builder.build();/// #/// # let index = Index::create_in_ram(schema);@@ -248,13 +249,13 @@ impl TopDocs {/// // This is where we build our topdocs collector/// ///// // Note the generics parameter that needs to match the- /// // type `sort_by_field`. revenue_field here is a FAST i64 field.+ /// // type `sort_by_field`. revenue_field here is a COLUMN i64 field./// let top_company_by_revenue = TopDocs/// ::with_limit(2)- /// .order_by_fast_field("revenue", Order::Desc);+ /// .order_by_column_field("revenue", Order::Desc);////// // ... and here are our documents. Note this is a simple vec.- /// // The `i64` in the pair is the value of our fast field for+ /// // The `i64` in the pair is the value of our columnar field for/// // each documents./// ///// // The vec is sorted decreasingly by `sort_by_field`, and has a@@ -266,19 +267,19 @@ impl TopDocs {/// Ok(resulting_docs)/// }/// ```- pub fn order_by_fast_field<TFastValue>(+ pub fn order_by_column_field<TColumnarValue>(self,- fast_field: impl ToString,+ column_field: impl ToString,order: Order,- ) -> impl Collector<Fruit = Vec<(TFastValue, DocAddress)>>+ ) -> impl Collector<Fruit = Vec<(TColumnarValue, DocAddress)>>where- TFastValue: FastValue,+ TColumnarValue: ColumnarValue,{- let u64_collector = self.order_by_u64_field(fast_field.to_string(), order.clone());- FastFieldConvertCollector {+ let u64_collector = self.order_by_u64_field(column_field.to_string(), order.clone());+ ColumnFieldConvertCollector {collector: u64_collector,- field: fast_field.to_string(),- fast_value: PhantomData,+ field: column_field.to_string(),+ columnar_value: PhantomData,order,}}@@ -294,11 +295,11 @@ impl TopDocs {////// # Example///- /// Typically, you will want to rely on one or more fast fields,+ /// Typically, you will want to rely on one or more columnar fields,/// to alter the original relevance `Score`.////// For instance, in the following, we assume that we are implementing- /// an e-commerce website that has a fast field called `popularity`+ /// an e-commerce website that has a columnar field called `popularity`/// that rates whether a product is typically often bought by users.////// In the following example will will tweak our ranking a bit by@@ -333,7 +334,7 @@ impl TopDocs {/// # Limitation////// This method only makes it possible to compute the score from a given- /// `DocId`, fastfield values for the doc and any information you could+ /// `DocId`, columnfield values for the doc and any information you could/// have precomputed beforehand. It does not make it possible for instance/// to compute something like TfIdf as it does not have access to the list of query/// terms present in the document, nor the term frequencies for the different terms.@@ -578,7 +579,7 @@ mod tests {use crate::collector::top_collector::ComparableDoc;use crate::collector::Collector;use crate::query::{AllQuery, Query, QueryParser};- use crate::schema::{Field, Schema, FAST, STORED, TEXT};+ use crate::schema::{Field, Schema, COLUMN, STORED, TEXT};use crate::time::format_description::well_known::Rfc3339;use crate::time::OffsetDateTime;use crate::{@@ -790,7 +791,7 @@ mod tests {fn test_top_field_collector_not_at_capacity() -> crate::Result<()> {let mut schema_builder = Schema::builder();let title = schema_builder.add_text_field(TITLE, TEXT);- let size = schema_builder.add_u64_field(SIZE, FAST);+ let size = schema_builder.add_u64_field(SIZE, COLUMN);let schema = schema_builder.build();let (index, query) = index("beer", title, schema, |index_writer| {index_writer@@ -831,7 +832,7 @@ mod tests {fn test_top_field_collector_datetime() -> crate::Result<()> {let mut schema_builder = Schema::builder();let name = schema_builder.add_text_field("name", TEXT);- let birthday = schema_builder.add_date_field("birthday", FAST);+ let birthday = schema_builder.add_date_field("birthday", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer = index.writer_for_tests()?;@@ -853,7 +854,7 @@ mod tests {))?;index_writer.commit()?;let searcher = index.reader()?.searcher();- let top_collector = TopDocs::with_limit(3).order_by_fast_field("birthday", Order::Desc);+ let top_collector = TopDocs::with_limit(3).order_by_column_field("birthday", Order::Desc);let top_docs: Vec<(DateTime, DocAddress)> = searcher.search(&AllQuery, &top_collector)?;assert_eq!(&top_docs[..],@@ -869,7 +870,7 @@ mod tests {fn test_top_field_collector_i64() -> crate::Result<()> {let mut schema_builder = Schema::builder();let city = schema_builder.add_text_field("city", TEXT);- let altitude = schema_builder.add_i64_field("altitude", FAST);+ let altitude = schema_builder.add_i64_field("altitude", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer = index.writer_for_tests()?;@@ -883,7 +884,7 @@ mod tests {))?;index_writer.commit()?;let searcher = index.reader()?.searcher();- let top_collector = TopDocs::with_limit(3).order_by_fast_field("altitude", Order::Desc);+ let top_collector = TopDocs::with_limit(3).order_by_column_field("altitude", Order::Desc);let top_docs: Vec<(i64, DocAddress)> = searcher.search(&AllQuery, &top_collector)?;assert_eq!(&top_docs[..],@@ -899,7 +900,7 @@ mod tests {fn test_top_field_collector_f64() -> crate::Result<()> {let mut schema_builder = Schema::builder();let city = schema_builder.add_text_field("city", TEXT);- let altitude = schema_builder.add_f64_field("altitude", FAST);+ let altitude = schema_builder.add_f64_field("altitude", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer = index.writer_for_tests()?;@@ -913,7 +914,7 @@ mod tests {))?;index_writer.commit()?;let searcher = index.reader()?.searcher();- let top_collector = TopDocs::with_limit(3).order_by_fast_field("altitude", Order::Desc);+ let top_collector = TopDocs::with_limit(3).order_by_column_field("altitude", Order::Desc);let top_docs: Vec<(f64, DocAddress)> = searcher.search(&AllQuery, &top_collector)?;assert_eq!(&top_docs[..],@@ -930,7 +931,7 @@ mod tests {fn test_field_does_not_exist() {let mut schema_builder = Schema::builder();let title = schema_builder.add_text_field(TITLE, TEXT);- let size = schema_builder.add_u64_field(SIZE, FAST);+ let size = schema_builder.add_u64_field(SIZE, COLUMN);let schema = schema_builder.build();let (index, _) = index("beer", title, schema, |index_writer| {index_writer@@ -949,7 +950,7 @@ mod tests {}#[test]- fn test_field_not_fast_field() -> crate::Result<()> {+ fn test_field_not_column_field() -> crate::Result<()> {let mut schema_builder = Schema::builder();let size = schema_builder.add_u64_field(SIZE, STORED);let schema = schema_builder.build();@@ -976,10 +977,10 @@ mod tests {index_writer.commit()?;let searcher = index.reader()?.searcher();let segment = searcher.segment_reader(0);- let top_collector = TopDocs::with_limit(4).order_by_fast_field::<i64>(SIZE, Order::Desc);+ let top_collector = TopDocs::with_limit(4).order_by_column_field::<i64>(SIZE, Order::Desc);let err = top_collector.for_segment(0, segment).err().unwrap();assert!(- matches!(err, crate::TantivyError::SchemaError(msg) if msg == "Field \"size\" is not a fast field.")+ matches!(err, crate::TantivyError::SchemaError(msg) if msg == "Field \"size\" is not a columnar field."));Ok(())}@@ -1039,10 +1040,10 @@ mod tests {(index, query)}#[test]- fn test_fast_field_ascending_order() -> crate::Result<()> {+ fn test_column_field_ascending_order() -> crate::Result<()> {let mut schema_builder = Schema::builder();let title = schema_builder.add_text_field(TITLE, TEXT);- let size = schema_builder.add_u64_field(SIZE, FAST);+ let size = schema_builder.add_u64_field(SIZE, COLUMN);let schema = schema_builder.build();let (index, query) = index("beer", title, schema, |index_writer| {index_writer@@ -1072,7 +1073,7 @@ mod tests {});let searcher = index.reader()?.searcher();- let top_collector = TopDocs::with_limit(4).order_by_fast_field(SIZE, Order::Asc);+ let top_collector = TopDocs::with_limit(4).order_by_column_field(SIZE, Order::Asc);let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector)?;assert_eq!(&top_docs[..],diff --git a/crates/tantivy/src/columnar/TODO.md b/crates/tantivy/src/columnar/TODO.mdindex ed92cd40..3b1c6f47 100644--- a/crates/tantivy/src/columnar/TODO.md+++ b/crates/tantivy/src/columnar/TODO.md@@ -35,9 +35,9 @@ use the rank & select naming in unit tests branch.multi-linear -> blockwiselinear codec -> simply a multiplication for the index columnrename columnar to something more explicit, like column_dictionary or columnar_table-rename fastfield -> column+rename columnfield -> columndocument changes-rationalization FastFieldValue, HasColumnType+rationalization ColumnFieldValue, HasColumnTypeisolate u128_based and uniform naming# Otherdiff --git a/crates/tantivy/src/columnar/column_values/mod.rs b/crates/tantivy/src/columnar/column_values/mod.rsindex 39261688..42353dc3 100644--- a/crates/tantivy/src/columnar/column_values/mod.rs+++ b/crates/tantivy/src/columnar/column_values/mod.rs@@ -1,6 +1,6 @@#![warn(missing_docs)]-//! # `fastfield_codecs`+//! # `columnfield_codecs`//!//! - Columnar storage of data for tantivy [`super::Column`].//! - Encode data in different codecs.@@ -104,7 +104,7 @@ pub trait ColumnValues<T: PartialOrd = u64>: Send + Sync + DowncastSync {}}- /// Fills an output buffer with the fast field values+ /// Fills an output buffer with the columnar field values/// associated with the `DocId` going from/// `start` to `start + output.len()`.///@@ -121,7 +121,7 @@ pub trait ColumnValues<T: PartialOrd = u64>: Send + Sync + DowncastSync {/// Get the row ids of values which are in the provided value range.///- /// Note that position == docid for single value fast fields+ /// Note that position == docid for single value columnar fieldsfn get_row_ids_for_value_range(&self,value_range: RangeInclusive<T>,diff --git a/crates/tantivy/src/columnar/column_values/monotonic_mapping.rs b/crates/tantivy/src/columnar/column_values/monotonic_mapping.rsindex 25b16ee1..3ce0de42 100644--- a/crates/tantivy/src/columnar/column_values/monotonic_mapping.rs+++ b/crates/tantivy/src/columnar/column_values/monotonic_mapping.rs@@ -11,18 +11,18 @@ use super::RowId;pub trait MonotonicallyMappableToU64: 'static + PartialOrd + Debug + Copy + Send + Sync {/// Converts a value to u64.///- /// Internally all fast field values are encoded as u64.+ /// Internally all columnar field values are encoded as u64.fn to_u64(self) -> u64;/// Converts a value from u64///- /// Internally all fast field values are encoded as u64.+ /// Internally all columnar field values are encoded as u64./// **Note: To be used for converting encoded Term, Posting values.**fn from_u64(val: u64) -> Self;}/// Values need to be strictly monotonic mapped to a `Internal` value (u64 or u128) that can be-/// used in fast field codecs.+/// used in columnar field codecs.////// The monotonic mapping is required so that `PartialOrd` can be used on `Internal` without/// converting to `External`.diff --git a/crates/tantivy/src/columnar/column_values/monotonic_mapping_u128.rs b/crates/tantivy/src/columnar/column_values/monotonic_mapping_u128.rsindex 4d59c675..7c5ecf16 100644--- a/crates/tantivy/src/columnar/column_values/monotonic_mapping_u128.rs+++ b/crates/tantivy/src/columnar/column_values/monotonic_mapping_u128.rs@@ -6,12 +6,12 @@ use std::net::Ipv6Addr;pub trait MonotonicallyMappableToU128: 'static + PartialOrd + Copy + Debug + Send + Sync {/// Converts a value to u128.///- /// Internally all fast field values are encoded as u64.+ /// Internally all columnar field values are encoded as u64.fn to_u128(self) -> u128;/// Converts a value from u128///- /// Internally all fast field values are encoded as u64.+ /// Internally all columnar field values are encoded as u64./// **Note: To be used for converting encoded Term, Posting values.**fn from_u128(val: u128) -> Self;}diff --git a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rsindex bd070533..e7d269a3 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs@@ -10,7 +10,7 @@ use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimatouse crate::columnar::{ColumnValues, RowId};/// Depending on the field type, a different-/// fast field is required.+/// columnar field is required.#[derive(Clone)]pub struct BitpackedReader {data: OwnedBytes,@@ -138,7 +138,7 @@ impl ColumnCodec for BitpackedCodec {type ColumnValues = BitpackedReader;type Estimator = BitpackedCodecEstimator;- /// Opens a fast field given a file.+ /// Opens a columnar field given a file.fn load(mut data: OwnedBytes) -> io::Result<Self::ColumnValues> {let stats = ColumnStats::deserialize(&mut data)?;let num_bits = num_bits(&stats);@@ -177,7 +177,7 @@ mod tests {}#[test]- fn bitpacked_fast_field_rand() {+ fn bitpacked_column_field_rand() {for _ in 0..500 {let mut data = (0..1 + rand::random::<u8>() as usize).map(|_| rand::random::<i64>() as u64 / 2)diff --git a/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs b/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rsindex 691523bd..7111c688 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs@@ -271,7 +271,7 @@ mod tests {}#[test]- fn test_blockwise_linear_fast_field_rand() {+ fn test_blockwise_linear_column_field_rand() {for _ in 0..500 {let mut data = (0..1 + rand::random::<u8>() as usize).map(|_| rand::random::<i64>() as u64 / 2)diff --git a/crates/tantivy/src/columnar/column_values/u64_based/linear.rs b/crates/tantivy/src/columnar/column_values/u64_based/linear.rsindex bfc90dba..038254e7 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/linear.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/linear.rs@@ -13,7 +13,7 @@ const HALF_SPACE: u64 = u64::MAX / 2;const LINE_ESTIMATION_BLOCK_LEN: usize = 512;/// Depending on the field type, a different-/// fast field is required.+/// columnar field is required.#[derive(Clone)]pub struct LinearReader {data: OwnedBytes,@@ -238,7 +238,7 @@ mod tests {}}#[test]- fn linear_interpol_fast_field_test_large_amplitude() {+ fn linear_interpol_column_field_test_large_amplitude() {let data = vec![i64::MAX as u64 / 2,i64::MAX as u64 / 3,@@ -264,13 +264,13 @@ mod tests {create_and_validate::<LinearCodec>(&data, "convex data");}#[test]- fn linear_interpol_fast_field_test_simple() {+ fn linear_interpol_column_field_test_simple() {let data = (10..=20_u64).collect::<Vec<_>>();create_and_validate::<LinearCodec>(&data, "simple monotonically");}#[test]- fn linear_interpol_fast_field_rand() {+ fn linear_interpol_column_field_rand() {let mut rng = rand::thread_rng();for _ in 0..50 {let mut data = (0..10_000).map(|_| rng.next_u64()).collect::<Vec<_>>();diff --git a/crates/tantivy/src/columnar/column_values/u64_based/tests.rs b/crates/tantivy/src/columnar/column_values/u64_based/tests.rsindex 0fac3980..a739fb26 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/tests.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/tests.rs@@ -307,7 +307,7 @@ fn estimation_test_bad_interpolation_case_monotonically_increasing() {}#[test]-fn test_fast_field_codec_type_to_code() {+fn test_column_field_codec_type_to_code() {let mut count_codec = 0;for code in 0..=255 {if let Some(codec_type) = CodecType::try_from_code(code) {@@ -318,7 +318,7 @@ fn test_fast_field_codec_type_to_code() {assert_eq!(count_codec, 3);}-fn test_fastfield_gcd_i64_with_codec(codec_type: CodecType, num_vals: usize) -> io::Result<()> {+fn test_columnfield_gcd_i64_with_codec(codec_type: CodecType, num_vals: usize) -> io::Result<()> {let mut vals: Vec<i64> = (-4..=(num_vals as i64) - 5).map(|val| val * 1000).collect();let mut buffer: Vec<u8> = Vec::new();crate::columnar::column_values::serialize_u64_based_column_values(@@ -351,18 +351,18 @@ fn test_fastfield_gcd_i64_with_codec(codec_type: CodecType, num_vals: usize) ->}#[test]-fn test_fastfield_gcd_i64() -> io::Result<()> {+fn test_columnfield_gcd_i64() -> io::Result<()> {for &codec_type in &[CodecType::Bitpacked,CodecType::BlockwiseLinear,CodecType::Linear,] {- test_fastfield_gcd_i64_with_codec(codec_type, 5500)?;+ test_columnfield_gcd_i64_with_codec(codec_type, 5500)?;}Ok(())}-fn test_fastfield_gcd_u64_with_codec(codec_type: CodecType, num_vals: usize) -> io::Result<()> {+fn test_columnfield_gcd_u64_with_codec(codec_type: CodecType, num_vals: usize) -> io::Result<()> {let mut vals: Vec<u64> = (1..=num_vals).map(|i| i as u64 * 1000u64).collect();let mut buffer: Vec<u8> = Vec::new();crate::columnar::column_values::serialize_u64_based_column_values(@@ -394,23 +394,25 @@ fn test_fastfield_gcd_u64_with_codec(codec_type: CodecType, num_vals: usize) ->}#[test]-fn test_fastfield_gcd_u64() -> io::Result<()> {+fn test_columnfield_gcd_u64() -> io::Result<()> {for &codec_type in &[CodecType::Bitpacked,CodecType::BlockwiseLinear,CodecType::Linear,] {- test_fastfield_gcd_u64_with_codec(codec_type, 5500)?;+ test_columnfield_gcd_u64_with_codec(codec_type, 5500)?;}Ok(())}#[test]-pub fn test_fastfield2() {- let test_fastfield = crate::columnar::column_values::serialize_and_load_u64_based_column_values::<- u64,- >(&&[100u64, 200u64, 300u64][..], &ALL_U64_CODEC_TYPES);- assert_eq!(test_fastfield.get_val(0), 100);- assert_eq!(test_fastfield.get_val(1), 200);- assert_eq!(test_fastfield.get_val(2), 300);+pub fn test_columnfield2() {+ let test_columnfield =+ crate::columnar::column_values::serialize_and_load_u64_based_column_values::<u64>(+ &&[100u64, 200u64, 300u64][..],+ &ALL_U64_CODEC_TYPES,+ );+ assert_eq!(test_columnfield.get_val(0), 100);+ assert_eq!(test_columnfield.get_val(1), 200);+ assert_eq!(test_columnfield.get_val(2), 300);}diff --git a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs b/crates/tantivy/src/columnar/columnar/writer/column_operation.rsindex 3b4da37d..4afb7b03 100644--- a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs+++ b/crates/tantivy/src/columnar/columnar/writer/column_operation.rs@@ -118,7 +118,7 @@ impl<T> From<T> for ColumnOperation<T> {}// Serialization trait very local to the writer.-// As we write fast fields, we accumulate them in "in memory".+// As we write columnar fields, we accumulate them in "in memory".// In order to limit memory usage, and in order// to benefit from the stacker, we do this by serialization our data// as "Symbols".diff --git a/crates/tantivy/src/columnar/columnar/writer/mod.rs b/crates/tantivy/src/columnar/columnar/writer/mod.rsindex 65da1134..0887afe4 100644--- a/crates/tantivy/src/columnar/columnar/writer/mod.rs+++ b/crates/tantivy/src/columnar/columnar/writer/mod.rs@@ -23,7 +23,7 @@ use crate::columnar::value::{Coerce, NumericalType, NumericalValue};use crate::columnar::{Cardinality, RowId};/// This is a set of buffers that are used to temporarily write the values into before passing them-/// to the fast field codecs.+/// to the columnar field codecs.#[derive(Default)]struct SpareBuffers {value_index_builders: PreallocatedIndexBuilders,diff --git a/crates/tantivy/src/columnar/dynamic_column.rs b/crates/tantivy/src/columnar/dynamic_column.rsindex f0674634..797d4067 100644--- a/crates/tantivy/src/columnar/dynamic_column.rs+++ b/crates/tantivy/src/columnar/dynamic_column.rs@@ -229,15 +229,15 @@ impl DynamicColumnHandle {&self.file_slice}- /// Returns the `u64` fast field reader reader associated with `fields` of types+ /// Returns the `u64` columnar field reader reader associated with `fields` of types/// Str, u64, i64, f64, bool, ip, or datetime.///- /// Notice that for IpAddr, the fastfield reader will return the u64 representation of the+ /// Notice that for IpAddr, the columnfield reader will return the u64 representation of the/// IpAddr./// In order to convert to u128 back cast to `CompactSpaceU64Accessor` and call/// `compact_to_u128`.///- /// If not, the fastfield reader will returns the u64-value associated with the original+ /// If not, the columnfield reader will returns the u64-value associated with the original/// FastValue.pub fn open_u64_lenient(&self) -> io::Result<Option<Column<u64>>> {let column_bytes = self.file_slice.read_bytes()?;diff --git a/crates/tantivy/src/columnfield/error.rs b/crates/tantivy/src/columnfield/error.rsnew file mode 100644index 00000000..a9195112--- /dev/null+++ b/crates/tantivy/src/columnfield/error.rs@@ -0,0 +1,26 @@+use std::result;++use crate::schema::FieldEntry;++/// `ColumnFieldNotAvailableError` is returned when the+/// user requested for a columnar field reader, and the field was not+/// defined in the schema as a columnar field.+#[derive(Debug, Error)]+#[error("Fast field not available: '{field_name:?}'")]+pub struct ColumnFieldNotAvailableError {+ pub(crate) field_name: String,+}++impl ColumnFieldNotAvailableError {+ /// Creates a `ColumnFieldNotAvailable` error.+ /// `field_entry` is the configuration of the field+ /// for which columnar fields are not available.+ pub fn new(field_entry: &FieldEntry) -> ColumnFieldNotAvailableError {+ ColumnFieldNotAvailableError {+ field_name: field_entry.name().to_string(),+ }+ }+}++/// Result when trying to access a columnar field reader.+pub type Result<R> = result::Result<R, ColumnFieldNotAvailableError>;diff --git a/crates/tantivy/src/fastfield/mod.rs b/crates/tantivy/src/columnfield/mod.rssimilarity index 74%rename from crates/tantivy/src/fastfield/mod.rsrename to crates/tantivy/src/columnfield/mod.rsindex 8654b635..794a3a0a 100644--- a/crates/tantivy/src/fastfield/mod.rs+++ b/crates/tantivy/src/columnfield/mod.rs@@ -2,19 +2,19 @@//!//! It is the equivalent of `Lucene`'s `DocValues`.//!-//! A fast field is a column-oriented fashion storage for `tantivy`.+//! A columnar field is a column-oriented fashion storage for `tantivy`.//!//! It is designed for the fast random access of some document//! fields given a document id.//!-//! Fast fields are useful when a field is required for all or most of+//! Columnar fields are useful when a field is required for all or most of//! the `DocSet`: for instance for scoring, grouping, aggregation or filtering//!//!-//! Fields have to be declared as `FAST` in the schema.+//! Fields have to be declared as `COLUMN` in the schema.//! Currently supported fields are: u64, i64, f64, bytes, ip and text.//!-//! Fast fields are stored in with [different codecs](columnar). The best codec is detected+//! Columnar fields are stored in with [different codecs](columnar). The best codec is detected//! automatically, when serializing.//!//! Read access performance is comparable to that of an array lookup.@@ -22,9 +22,9 @@pub use crate::columnar::Column;use crate::columnar::MonotonicallyMappableToU64;-pub use self::error::{FastFieldNotAvailableError, Result};-pub use self::readers::FastFieldReaders;-pub use self::writer::FastFieldsWriter;+pub use self::error::{ColumnFieldNotAvailableError, Result};+pub use self::readers::ColumnFieldReaders;+pub use self::writer::ColumnFieldsWriter;use crate::schema::Type;use crate::DateTime;@@ -32,37 +32,37 @@ mod error;mod readers;mod writer;-/// Trait for types that are allowed for fast fields:+/// Trait for types that are allowed for columnar fields:/// (u64, i64 and f64, bool, DateTime).-pub trait FastValue: MonotonicallyMappableToU64 {- /// Returns the `schema::Type` for this FastValue.+pub trait ColumnarValue: MonotonicallyMappableToU64 {+ /// Returns the `schema::Type` for this ColumnarValue.fn to_type() -> Type;}-impl FastValue for u64 {+impl ColumnarValue for u64 {fn to_type() -> Type {Type::U64}}-impl FastValue for i64 {+impl ColumnarValue for i64 {fn to_type() -> Type {Type::I64}}-impl FastValue for f64 {+impl ColumnarValue for f64 {fn to_type() -> Type {Type::F64}}-impl FastValue for bool {+impl ColumnarValue for bool {fn to_type() -> Type {Type::Bool}}-impl FastValue for DateTime {+impl ColumnarValue for DateTime {fn to_type() -> Type {Type::Date}@@ -85,7 +85,7 @@ mod tests {use crate::index::SegmentId;use crate::merge_policy::NoMergePolicy;use crate::schema::{- DateOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument, FAST,+ DateOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument, COLUMN,INDEXED,};use crate::time::OffsetDateTime;@@ -93,7 +93,7 @@ mod tests {pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {let mut schema_builder = Schema::builder();- schema_builder.add_u64_field("field", FAST);+ schema_builder.add_u64_field("field", COLUMN);schema_builder.build()});pub static FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("field").unwrap());@@ -105,29 +105,29 @@ mod tests {}#[test]- fn test_intfastfield_small() -> crate::Result<()> {+ fn test_intcolumnfield_small() -> crate::Result<()> {let path = Path::new("test");let directory: RamDirectory = RamDirectory::create();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA).unwrap();- fast_field_writers+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&SCHEMA).unwrap();+ column_field_writers.add_document(&doc!(*FIELD=>13u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>14u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>2u64)).unwrap();- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 80);- let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let column = fast_field_readers.u64("field").unwrap().values;+ let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();+ let column = column_field_readers.u64("field").unwrap().values;assert_eq!(column.get_val(0), 13u64);assert_eq!(column.get_val(1), 14u64);assert_eq!(column.get_val(2), 2u64);@@ -135,46 +135,46 @@ mod tests {}#[test]- fn test_intfastfield_large() {+ fn test_intcolumnfield_large() {let path = Path::new("test");let directory: RamDirectory = RamDirectory::create();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA).unwrap();- fast_field_writers+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&SCHEMA).unwrap();+ column_field_writers.add_document(&doc!(*FIELD=>4u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>14_082_001u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>3_052u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>9_002u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>15_001u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>777u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>1_002u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>1_501u64)).unwrap();- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>215u64)).unwrap();- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 108);- let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let col = fast_field_readers.u64("field").unwrap().values;+ let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();+ let col = column_field_readers.u64("field").unwrap().values;assert_eq!(col.get_val(0), 4u64);assert_eq!(col.get_val(1), 14_082_001u64);assert_eq!(col.get_val(2), 3_052u64);@@ -187,54 +187,54 @@ mod tests {}#[test]- fn test_intfastfield_null_amplitude() {+ fn test_intcolumnfield_null_amplitude() {let path = Path::new("test");let directory: RamDirectory = RamDirectory::create();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA).unwrap();+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&SCHEMA).unwrap();for _ in 0..10_000 {- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>100_000u64)).unwrap();}- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 81);- let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let fast_field_reader = fast_field_readers.u64("field").unwrap().values;+ let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();+ let column_field_reader = column_field_readers.u64("field").unwrap().values;for doc in 0..10_000 {- assert_eq!(fast_field_reader.get_val(doc), 100_000u64);+ assert_eq!(column_field_reader.get_val(doc), 100_000u64);}}#[test]- fn test_intfastfield_large_numbers() {+ fn test_intcolumnfield_large_numbers() {let path = Path::new("test");let directory: RamDirectory = RamDirectory::create();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA).unwrap();+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&SCHEMA).unwrap();// forcing the amplitude to be high- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>0u64)).unwrap();for doc_id in 1u64..10_000u64 {- fast_field_writers+ column_field_writers.add_document(&doc!(*FIELD=>5_000_000_000_000_000_000u64 + doc_id)).unwrap();}- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 4476);{- let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let col = fast_field_readers.u64("field").unwrap().values;+ let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();+ let col = column_field_readers.u64("field").unwrap().values;for doc in 1..10_000 {assert_eq!(col.get_val(doc), 5_000_000_000_000_000_000u64 + doc as u64);}@@ -242,30 +242,30 @@ mod tests {}#[test]- fn test_signed_intfastfield_normal() -> crate::Result<()> {+ fn test_signed_intcolumnfield_normal() -> crate::Result<()> {let path = Path::new("test");let directory: RamDirectory = RamDirectory::create();let mut schema_builder = Schema::builder();- let i64_field = schema_builder.add_i64_field("field", FAST);+ let i64_field = schema_builder.add_i64_field("field", COLUMN);let schema = schema_builder.build();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&schema).unwrap();for i in -100i64..10_000i64 {let mut doc = TantivyDocument::default();doc.add_i64(i64_field, i);- fast_field_writers.add_document(&doc).unwrap();+ column_field_writers.add_document(&doc).unwrap();}- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 252);{- let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();- let col = fast_field_readers.i64("field").unwrap().values;+ let column_field_readers = ColumnFieldReaders::open(file, schema).unwrap();+ let col = column_field_readers.i64("field").unwrap().values;assert_eq!(col.min_value(), -100i64);assert_eq!(col.max_value(), 9_999i64);for (doc, i) in (-100i64..10_000i64).enumerate() {@@ -294,43 +294,43 @@ mod tests {permutation}- fn test_intfastfield_permutation_with_data(permutation: Vec<u64>) {+ fn test_intcolumnfield_permutation_with_data(permutation: Vec<u64>) {let path = Path::new("test");let n = permutation.len();let directory = RamDirectory::create();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA).unwrap();+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&SCHEMA).unwrap();for &x in &permutation {- fast_field_writers.add_document(&doc!(*FIELD=>x)).unwrap();+ column_field_writers.add_document(&doc!(*FIELD=>x)).unwrap();}- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();- let col = fast_field_readers.u64("field").unwrap().values;+ let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();+ let col = column_field_readers.u64("field").unwrap().values;for a in 0..n {assert_eq!(col.get_val(a as u32), permutation[a]);}}#[test]- fn test_intfastfield_permutation_gcd() {+ fn test_intcolumnfield_permutation_gcd() {let permutation = generate_permutation_gcd();- test_intfastfield_permutation_with_data(permutation);+ test_intcolumnfield_permutation_with_data(permutation);}#[test]- fn test_intfastfield_permutation() {+ fn test_intcolumnfield_permutation() {let permutation = generate_permutation();- test_intfastfield_permutation_with_data(permutation);+ test_intcolumnfield_permutation_with_data(permutation);}#[test]- fn test_merge_missing_date_fast_field() {+ fn test_merge_missing_date_column_field() {let mut schema_builder = Schema::builder();- let date_field = schema_builder.add_date_field("date", FAST);+ let date_field = schema_builder.add_date_field("date", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();@@ -355,17 +355,17 @@ mod tests {}#[test]- fn test_datefastfield() -> crate::Result<()> {+ fn test_datecolumnfield() -> crate::Result<()> {let mut schema_builder = Schema::builder();let date_field = schema_builder.add_date_field("date",- DateOptions::from(FAST).set_precision(DateTimePrecision::Nanoseconds),+ DateOptions::from(COLUMN).set_precision(DateTimePrecision::Nanoseconds),);let multi_date_field = schema_builder.add_date_field("multi_date",DateOptions::default().set_precision(DateTimePrecision::Nanoseconds)- .set_fast(),+ .set_columnar(),);let schema = schema_builder.build();let index = Index::create_in_ram(schema);@@ -389,47 +389,51 @@ mod tests {let searcher = reader.searcher();assert_eq!(searcher.segment_readers().len(), 1);let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();- let date_fast_field = fast_fields+ let column_fields = segment_reader.column_fields();+ let date_column_field = column_fields.column_opt::<DateTime>("date").unwrap().unwrap().values;- assert_eq!(date_fast_field.get_val(0).into_timestamp_nanos(), 1i64);- assert_eq!(date_fast_field.get_val(1).into_timestamp_nanos(), 4i64);- assert_eq!(date_fast_field.get_val(2).into_timestamp_nanos(), 0i64);+ assert_eq!(date_column_field.get_val(0).into_timestamp_nanos(), 1i64);+ assert_eq!(date_column_field.get_val(1).into_timestamp_nanos(), 4i64);+ assert_eq!(date_column_field.get_val(2).into_timestamp_nanos(), 0i64);Ok(())}#[test]- pub fn test_fastfield_bool_small() {+ pub fn test_columnfield_bool_small() {let path = Path::new("test_bool");let directory: RamDirectory = RamDirectory::create();let mut schema_builder = Schema::builder();- schema_builder.add_bool_field("field_bool", FAST);+ schema_builder.add_bool_field("field_bool", COLUMN);let schema = schema_builder.build();let field = schema.get_field("field_bool").unwrap();{let mut write: WritePtr = directory.open_write(path).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();- fast_field_writers.add_document(&doc!(field=>true)).unwrap();- fast_field_writers+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&schema).unwrap();+ column_field_writers+ .add_document(&doc!(field=>true))+ .unwrap();+ column_field_writers.add_document(&doc!(field=>false)).unwrap();- fast_field_writers.add_document(&doc!(field=>true)).unwrap();- fast_field_writers+ column_field_writers+ .add_document(&doc!(field=>true))+ .unwrap();+ column_field_writers.add_document(&doc!(field=>false)).unwrap();- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 84);- let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();- let bool_col = fast_field_readers.bool("field_bool").unwrap();+ let column_field_readers = ColumnFieldReaders::open(file, schema).unwrap();+ let bool_col = column_field_readers.bool("field_bool").unwrap();assert_eq!(bool_col.first(0), Some(true));assert_eq!(bool_col.first(1), Some(false));assert_eq!(bool_col.first(2), Some(true));@@ -437,30 +441,32 @@ mod tests {}#[test]- pub fn test_fastfield_bool_large() {+ pub fn test_columnfield_bool_large() {let path = Path::new("test_bool");let directory: RamDirectory = RamDirectory::create();let mut schema_builder = Schema::builder();- schema_builder.add_bool_field("field_bool", FAST);+ schema_builder.add_bool_field("field_bool", COLUMN);let schema = schema_builder.build();let field = schema.get_field("field_bool").unwrap();{let mut write: WritePtr = directory.open_write(path).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();+ let mut column_field_writers = ColumnFieldsWriter::from_schema(&schema).unwrap();for _ in 0..50 {- fast_field_writers.add_document(&doc!(field=>true)).unwrap();- fast_field_writers+ column_field_writers+ .add_document(&doc!(field=>true))+ .unwrap();+ column_field_writers.add_document(&doc!(field=>false)).unwrap();}- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}let file = directory.open_read(path).unwrap();assert_eq!(file.len(), 96);- let readers = FastFieldReaders::open(file, schema).unwrap();+ let readers = ColumnFieldReaders::open(file, schema).unwrap();let bool_col = readers.bool("field_bool").unwrap();for i in 0..25 {assert_eq!(bool_col.first(i * 2), Some(true));@@ -472,11 +478,11 @@ mod tests {let directory: RamDirectory = RamDirectory::create();{let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();- let mut fast_field_writers = FastFieldsWriter::from_schema(schema).unwrap();+ let mut column_field_writers = ColumnFieldsWriter::from_schema(schema).unwrap();for doc in docs {- fast_field_writers.add_document(doc).unwrap();+ column_field_writers.add_document(doc).unwrap();}- fast_field_writers.serialize(&mut write, None).unwrap();+ column_field_writers.serialize(&mut write, None).unwrap();write.terminate().unwrap();}Ok(directory)@@ -505,7 +511,9 @@ mod tests {}).take(1_000).collect();- let date_options = DateOptions::default().set_fast().set_precision(precision);+ let date_options = DateOptions::default()+ .set_columnar()+ .set_precision(precision);let mut schema_builder = SchemaBuilder::default();let field = schema_builder.add_date_field("field", date_options);let schema = schema_builder.build();@@ -515,7 +523,7 @@ mod tests {let directory = get_index(&docs[..], &schema).unwrap();let path = Path::new("test");let file = directory.open_read(path).unwrap();- let readers = FastFieldReaders::open(file, schema).unwrap();+ let readers = ColumnFieldReaders::open(file, schema).unwrap();let col = readers.date("field").unwrap();for (i, time) in times.iter().enumerate() {@@ -528,7 +536,7 @@ mod tests {#[test]fn test_gcd_bug_regression_1757() {let mut schema_builder = Schema::builder();- let num_field = schema_builder.add_u64_field("url_norm_hash", FAST | INDEXED);+ let num_field = schema_builder.add_u64_field("url_norm_hash", COLUMN | INDEXED);let schema = schema_builder.build();let index = Index::create_in_ram(schema);{@@ -555,7 +563,7 @@ mod tests {let reader = index.reader().unwrap();let searcher = reader.searcher();let segment = &searcher.segment_readers()[0];- let field = segment.fast_fields().u64("url_norm_hash").unwrap().values;+ let field = segment.column_fields().u64("url_norm_hash").unwrap().values;let numbers = [100, 200, 300];let test_range = |range: RangeInclusive<u64>| {@@ -577,7 +585,7 @@ mod tests {#[test]fn test_mapping_bug_docids_for_value_range() {let mut schema_builder = Schema::builder();- let num_field = schema_builder.add_u64_field("url_norm_hash", FAST | INDEXED);+ let num_field = schema_builder.add_u64_field("url_norm_hash", COLUMN | INDEXED);let schema = schema_builder.build();let index = Index::create_in_ram(schema);{@@ -604,7 +612,7 @@ mod tests {let reader = index.reader().unwrap();let searcher = reader.searcher();let segment = &searcher.segment_readers()[0];- let field = segment.fast_fields().u64("url_norm_hash").unwrap().values;+ let field = segment.column_fields().u64("url_norm_hash").unwrap().values;let numbers = [1000, 1001, 1003];let test_range = |range: RangeInclusive<u64>| {@@ -639,9 +647,9 @@ mod tests {}#[test]- fn test_fast_field_in_json_field_expand_dots_disabled() {+ fn test_column_field_in_json_field_expand_dots_disabled() {let mut schema_builder = Schema::builder();- let json_option = JsonObjectOptions::default().set_fast(None);+ let json_option = JsonObjectOptions::default().set_columnar(None);let json = schema_builder.add_json_field("json", json_option);let schema = schema_builder.build();let index = Index::create_in_ram(schema);@@ -651,12 +659,12 @@ mod tests {.unwrap();index_writer.commit().unwrap();let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();- assert!(fast_field_reader+ let column_field_reader = searcher.segment_reader(0u32).column_fields();+ assert!(column_field_reader.column_opt::<i64>("json.attr.age").unwrap().is_none());- let column = fast_field_reader+ let column = column_field_reader.column_opt::<i64>(r"json.attr\.age").unwrap().unwrap();@@ -664,10 +672,10 @@ mod tests {}#[test]- fn test_fast_field_in_json_field_expand_dots_enabled() {+ fn test_column_field_in_json_field_expand_dots_enabled() {let mut schema_builder = Schema::builder();let json_option = JsonObjectOptions::default()- .set_fast(None)+ .set_columnar(None).set_expand_dots_enabled();let json = schema_builder.add_json_field("json", json_option);let schema = schema_builder.build();@@ -678,9 +686,9 @@ mod tests {.unwrap();index_writer.commit().unwrap();let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();+ let column_field_reader = searcher.segment_reader(0u32).column_fields();for test_column_name in &["json.attr.age", "json.attr\\.age"] {- let column = fast_field_reader+ let column = column_field_reader.column_opt::<i64>(test_column_name).unwrap().unwrap();@@ -689,9 +697,9 @@ mod tests {}#[test]- fn test_fast_field_dot_in_schema_field_name() {+ fn test_column_field_dot_in_schema_field_name() {let mut schema_builder = Schema::builder();- let field_with_dot = schema_builder.add_i64_field("field.with.dot", FAST);+ let field_with_dot = schema_builder.add_i64_field("field.with.dot", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();@@ -700,8 +708,8 @@ mod tests {.unwrap();index_writer.commit().unwrap();let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();- let column = fast_field_reader+ let column_field_reader = searcher.segment_reader(0u32).column_fields();+ let column = column_field_reader.column_opt::<i64>("field.with.dot").unwrap().unwrap();@@ -709,10 +717,10 @@ mod tests {}#[test]- fn test_shadowing_fast_field() {+ fn test_shadowing_column_field() {let mut schema_builder = Schema::builder();- let json_field = schema_builder.add_json_field("jsonfield", FAST);- let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", FAST);+ let json_field = schema_builder.add_json_field("jsonfield", COLUMN);+ let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();@@ -721,8 +729,8 @@ mod tests {.unwrap();index_writer.commit().unwrap();let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();- let column = fast_field_reader+ let column_field_reader = searcher.segment_reader(0u32).column_fields();+ let column = column_field_reader.column_opt::<i64>("jsonfield.attr.age").unwrap().unwrap();@@ -730,10 +738,10 @@ mod tests {}#[test]- fn test_shadowing_fast_field_with_expand_dots() {+ fn test_shadowing_column_field_with_expand_dots() {let mut schema_builder = Schema::builder();let json_option = JsonObjectOptions::default()- .set_fast(None)+ .set_columnar(None).set_expand_dots_enabled();let json_field = schema_builder.add_json_field("jsonfield", json_option.clone());let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", json_option);@@ -745,14 +753,14 @@ mod tests {.unwrap();index_writer.commit().unwrap();let searcher = index.reader().unwrap().searcher();- let fast_field_reader = searcher.segment_reader(0u32).fast_fields();+ let column_field_reader = searcher.segment_reader(0u32).column_fields();// Supported for now, maybe dropped in the future.- let column = fast_field_reader+ let column = column_field_reader.column_opt::<i64>("jsonfield.attr.age").unwrap().unwrap();assert_eq!(column.first(0u32), Some(33));- let column = fast_field_reader+ let column = column_field_reader.column_opt::<i64>("jsonfield\\.attr.age").unwrap().unwrap();diff --git a/crates/tantivy/src/fastfield/readers.rs b/crates/tantivy/src/columnfield/readers.rssimilarity index 83%rename from crates/tantivy/src/fastfield/readers.rsrename to crates/tantivy/src/columnfield/readers.rsindex 735f9835..119f6b6c 100644--- a/crates/tantivy/src/fastfield/readers.rs+++ b/crates/tantivy/src/columnfield/readers.rs@@ -13,20 +13,23 @@ use crate::schema::{Field, FieldEntry, FieldType, Schema};use crate::space_usage::{FieldUsage, PerFieldSpaceUsage};use crate::TantivyError;-/// Provides access to all of the BitpackedFastFieldReader.+/// Provides access to all of the BitpackedColumnFieldReader.///-/// Internally, `FastFieldReaders` have preloaded fast field readers,+/// Internally, `ColumnFieldReaders` have preloaded columnar field readers,/// and just wraps several `HashMap`.#[derive(Clone)]-pub struct FastFieldReaders {+pub struct ColumnFieldReaders {columnar: Arc<ColumnarReader>,schema: Schema,}-impl FastFieldReaders {- pub(crate) fn open(fast_field_file: FileSlice, schema: Schema) -> io::Result<FastFieldReaders> {- let columnar = Arc::new(ColumnarReader::open(fast_field_file)?);- Ok(FastFieldReaders { columnar, schema })+impl ColumnFieldReaders {+ pub(crate) fn open(+ column_field_file: FileSlice,+ schema: Schema,+ ) -> io::Result<ColumnFieldReaders> {+ let columnar = Arc::new(ColumnarReader::open(column_field_file)?);+ Ok(ColumnFieldReaders { columnar, schema })}fn resolve_field(&self, column_name: &str) -> crate::Result<Option<String>> {@@ -53,19 +56,19 @@ impl FastFieldReaders {self.columnar.as_ref()}- /// Transforms a user-supplied fast field name into a column name.+ /// Transforms a user-supplied columnar field name into a column name.///- /// A user-supplied fast field name is not necessarily a schema field name- /// because we handle fast fields.+ /// A user-supplied columnar field name is not necessarily a schema field name+ /// because we handle columnar fields.////// For instance, if the documents look like `{.., "attributes": {"color": "red"}}` and- /// `attributes` is a json fast field, a user could want to run a term aggregation over+ /// `attributes` is a json columnar field, a user could want to run a term aggregation over/// colors, by referring to the field as `attributes.color`.////// This function transforms `attributes.color` into a column key to be used in the `columnar`.////// The logic works as follows, first we identify which field is targetted by calling- /// `schema.find_field(..)`. This method will attempt to split the user splied fast field+ /// `schema.find_field(..)`. This method will attempt to split the user splied columnar field/// name by non-escaped dots, and find the longest matching schema field name./// In our case, it would return the (attribute_field, "color").///@@ -86,9 +89,9 @@ impl FastFieldReaders {return Ok(None);};let field_entry: &FieldEntry = self.schema.get_field_entry(field);- if !field_entry.is_fast() {+ if !field_entry.is_columnar() {return Err(TantivyError::InvalidArgument(format!(- "Field {field_name:?} is not configured as fast field"+ "Field {field_name:?} is not configured as columnar field")));}Ok(match (field_entry.field_type(), path) {@@ -149,21 +152,21 @@ impl FastFieldReaders {let col_opt: Option<Column<T>> = self.column_opt(field)?;col_opt.ok_or_else(|| {crate::TantivyError::SchemaError(format!(- "Field `{field}` is missing or is not configured as a fast field."+ "Field `{field}` is missing or is not configured as a columnar field."))})}- /// Returns the `u64` fast field reader reader associated with `field`.+ /// Returns the `u64` columnar field reader reader associated with `field`.///- /// If `field` is not a u64 fast field, this method returns an Error.+ /// If `field` is not a u64 columnar field, this method returns an Error.pub fn u64(&self, field: &str) -> crate::Result<Column<u64>> {self.column(field)}- /// Returns the `date` fast field reader reader associated with `field`.+ /// Returns the `date` columnar field reader reader associated with `field`.///- /// If `field` is not a date fast field, this method returns an Error.+ /// If `field` is not a date columnar field, this method returns an Error.pub fn date(&self, field: &str) -> crate::Result<Column<crate::common::DateTime>> {self.column(field)}@@ -292,23 +295,23 @@ impl FastFieldReaders {self.u64_lenient_for_type(None, field_name)}- /// Returns the `i64` fast field reader reader associated with `field`.+ /// Returns the `i64` columnar field reader reader associated with `field`.///- /// If `field` is not a i64 fast field, this method returns an Error.+ /// If `field` is not a i64 columnar field, this method returns an Error.pub fn i64(&self, field_name: &str) -> crate::Result<Column<i64>> {self.column(field_name)}- /// Returns the `f64` fast field reader reader associated with `field`.+ /// Returns the `f64` columnar field reader reader associated with `field`.///- /// If `field` is not a f64 fast field, this method returns an Error.+ /// If `field` is not a f64 columnar field, this method returns an Error.pub fn f64(&self, field_name: &str) -> crate::Result<Column<f64>> {self.column(field_name)}- /// Returns the `bool` fast field reader reader associated with `field`.+ /// Returns the `bool` columnar field reader reader associated with `field`.///- /// If `field` is not a bool fast field, this method returns an Error.+ /// If `field` is not a bool columnar field, this method returns an Error.pub fn bool(&self, field_name: &str) -> crate::Result<Column<bool>> {self.column(field_name)}@@ -318,21 +321,21 @@ impl FastFieldReaders {mod tests {use crate::columnar::ColumnType;- use crate::schema::{JsonObjectOptions, Schema, FAST};+ use crate::schema::{JsonObjectOptions, Schema, COLUMN};use crate::{Index, IndexWriter, TantivyDocument};#[test]- fn test_fast_field_reader_resolve_with_dynamic_internal() {+ fn test_column_field_reader_resolve_with_dynamic_internal() {let mut schema_builder = Schema::builder();- schema_builder.add_i64_field("age", FAST);- schema_builder.add_json_field("json_expand_dots_disabled", FAST);+ schema_builder.add_i64_field("age", COLUMN);+ schema_builder.add_json_field("json_expand_dots_disabled", COLUMN);schema_builder.add_json_field("json_expand_dots_enabled",JsonObjectOptions::default()- .set_fast(None)+ .set_columnar(None).set_expand_dots_enabled(),);- let dynamic_field = schema_builder.add_json_field("_dyna", FAST);+ let dynamic_field = schema_builder.add_json_field("_dyna", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();@@ -343,21 +346,21 @@ mod tests {let reader = index.reader().unwrap();let searcher = reader.searcher();let reader = searcher.segment_reader(0u32);- let fast_field_readers = reader.fast_fields();+ let column_field_readers = reader.column_fields();assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("age", None).unwrap(),Some("age".to_string()));assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("age", Some(dynamic_field)).unwrap(),Some("age".to_string()));assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("json_expand_dots_disabled.attr.color",None@@ -366,7 +369,7 @@ mod tests {Some("json_expand_dots_disabled\u{1}attr\u{1}color".to_string()));assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("json_expand_dots_disabled.attr\\.color",Some(dynamic_field)@@ -375,7 +378,7 @@ mod tests {Some("json_expand_dots_disabled\u{1}attr.color".to_string()));assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("json_expand_dots_enabled.attr\\.color",Some(dynamic_field)@@ -384,13 +387,13 @@ mod tests {Some("json_expand_dots_enabled\u{1}attr\u{1}color".to_string()));assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("notinschema.attr.color", None).unwrap(),None);assert_eq!(- fast_field_readers+ column_field_readers.resolve_column_name_given_default_field("notinschema.attr.color",Some(dynamic_field)@@ -401,10 +404,10 @@ mod tests {}#[test]- fn test_fast_field_reader_dynamic_column_handles() {+ fn test_column_field_reader_dynamic_column_handles() {let mut schema_builder = Schema::builder();- let id = schema_builder.add_u64_field("id", FAST);- let json = schema_builder.add_json_field("json", FAST);+ let id = schema_builder.add_u64_field("id", COLUMN);+ let json = schema_builder.add_json_field("json", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();@@ -421,12 +424,12 @@ mod tests {let reader = index.reader().unwrap();let searcher = reader.searcher();let reader = searcher.segment_reader(0u32);- let fast_fields = reader.fast_fields();- let id_columns = fast_fields.dynamic_column_handles("id").unwrap();+ let column_fields = reader.column_fields();+ let id_columns = column_fields.dynamic_column_handles("id").unwrap();assert_eq!(id_columns.len(), 1);assert_eq!(id_columns.first().unwrap().column_type(), ColumnType::U64);- let foo_columns = fast_fields.dynamic_column_handles("json.foo").unwrap();+ let foo_columns = column_fields.dynamic_column_handles("json.foo").unwrap();assert_eq!(foo_columns.len(), 2);assert!(foo_columns.iter()@@ -435,6 +438,6 @@ mod tests {.iter().any(|column| column.column_type() == ColumnType::Bool));- println!("*** {:?}", fast_fields.columnar().list_columns());+ println!("*** {:?}", column_fields.columnar().list_columns());}}diff --git a/crates/tantivy/src/fastfield/writer.rs b/crates/tantivy/src/columnfield/writer.rssimilarity index 89%rename from crates/tantivy/src/fastfield/writer.rsrename to crates/tantivy/src/columnfield/writer.rsindex 9eefa602..c452b388 100644--- a/crates/tantivy/src/fastfield/writer.rs+++ b/crates/tantivy/src/columnfield/writer.rs@@ -13,10 +13,10 @@ use crate::{DocId, TantivyError};/// This is mostly to guard us from a stack overflow triggered by malicious input.const JSON_DEPTH_LIMIT: usize = 20;-/// The `FastFieldsWriter` groups all of the fast field writers.-pub struct FastFieldsWriter {+/// The `ColumnFieldsWriter` groups all of the columnar field writers.+pub struct ColumnFieldsWriter {columnar_writer: ColumnarWriter,- fast_field_names: Vec<Option<String>>, //< TODO see if we can hash the field name hash too.+ column_field_names: Vec<Option<String>>, //< TODO see if we can hash the field name hash too.per_field_tokenizer: Vec<Option<TextAnalyzer>>,date_precisions: Vec<DateTimePrecision>,expand_dots: Vec<bool>,@@ -25,21 +25,21 @@ pub struct FastFieldsWriter {json_path_buffer: JsonPathWriter,}-impl FastFieldsWriter {- /// Create all `FastFieldWriter` required by the schema.+impl ColumnFieldsWriter {+ /// Create all `ColumnFieldWriter` required by the schema.#[cfg(test)]- pub fn from_schema(schema: &Schema) -> crate::Result<FastFieldsWriter> {- FastFieldsWriter::from_schema_and_tokenizer_manager(schema, TokenizerManager::new())+ pub fn from_schema(schema: &Schema) -> crate::Result<ColumnFieldsWriter> {+ ColumnFieldsWriter::from_schema_and_tokenizer_manager(schema, TokenizerManager::new())}- /// Create all `FastFieldWriter` required by the schema.+ /// Create all `ColumnFieldWriter` required by the schema.pub fn from_schema_and_tokenizer_manager(schema: &Schema,tokenizer_manager: TokenizerManager,- ) -> crate::Result<FastFieldsWriter> {+ ) -> crate::Result<ColumnFieldsWriter> {let mut columnar_writer = ColumnarWriter::default();- let mut fast_field_names: Vec<Option<String>> = vec![None; schema.num_fields()];+ let mut column_field_names: Vec<Option<String>> = vec![None; schema.num_fields()];let mut date_precisions: Vec<DateTimePrecision> =std::iter::repeat_with(DateTimePrecision::default).take(schema.num_fields())@@ -48,16 +48,17 @@ impl FastFieldsWriter {let mut per_field_tokenizer: Vec<Option<TextAnalyzer>> = vec![None; schema.num_fields()];// TODO see other typesfor (field_id, field_entry) in schema.fields() {- if !field_entry.field_type().is_fast() {+ if !field_entry.field_type().is_columnar() {continue;}- fast_field_names[field_id.field_id() as usize] = Some(field_entry.name().to_string());+ column_field_names[field_id.field_id() as usize] = Some(field_entry.name().to_string());let value_type = field_entry.field_type().value_type();if let FieldType::Date(date_options) = field_entry.field_type() {date_precisions[field_id.field_id() as usize] = date_options.get_precision();}if let FieldType::JsonObject(json_object_options) = field_entry.field_type() {- if let Some(tokenizer_name) = json_object_options.get_fast_field_tokenizer_name() {+ if let Some(tokenizer_name) = json_object_options.get_column_field_tokenizer_name()+ {let text_analyzer = tokenizer_manager.get(tokenizer_name).ok_or_else(|| {TantivyError::InvalidArgument(format!("Tokenizer {tokenizer_name:?} not found"@@ -70,7 +71,7 @@ impl FastFieldsWriter {json_object_options.is_expand_dots_enabled();}if let FieldType::Str(text_options) = field_entry.field_type() {- if let Some(tokenizer_name) = text_options.get_fast_field_tokenizer_name() {+ if let Some(tokenizer_name) = text_options.get_column_field_tokenizer_name() {let text_analyzer = tokenizer_manager.get(tokenizer_name).ok_or_else(|| {TantivyError::InvalidArgument(format!("Tokenizer {tokenizer_name:?} not found"@@ -89,9 +90,9 @@ impl FastFieldsWriter {);}}- Ok(FastFieldsWriter {+ Ok(ColumnFieldsWriter {columnar_writer,- fast_field_names,+ column_field_names,per_field_tokenizer,num_docs: 0u32,date_precisions,@@ -115,7 +116,7 @@ impl FastFieldsWriter {.sort_order(sort_field, num_docs, reversed)}- /// Indexes all of the fastfields of a new document.+ /// Indexes all of the columnfields of a new document.pub fn add_document<D: Document>(&mut self, doc: &D) -> crate::Result<()> {let doc_id = self.num_docs;for (field, value) in doc.iter_fields_and_values() {@@ -133,7 +134,7 @@ impl FastFieldsWriter {field: Field,value: V,) -> crate::Result<()> {- let field_name = match &self.fast_field_names[field.field_id() as usize] {+ let field_name = match &self.column_field_names[field.field_id() as usize] {None => return Ok(()),Some(name) => name,};@@ -208,8 +209,8 @@ impl FastFieldsWriter {Ok(())}- /// Serializes all of the `FastFieldWriter`s by pushing them in- /// order to the fast field serializer.+ /// Serializes all of the `ColumnFieldWriter`s by pushing them in+ /// order to the columnar field serializer.pub fn serialize(mut self,wrt: &mut dyn io::Write,diff --git a/crates/tantivy/src/common/datetime.rs b/crates/tantivy/src/common/datetime.rsindex 7a1c3228..2808d6cc 100644--- a/crates/tantivy/src/common/datetime.rs+++ b/crates/tantivy/src/common/datetime.rs@@ -7,8 +7,8 @@ use time::{OffsetDateTime, PrimitiveDateTime, UtcOffset};use super::BinarySerializable;-/// Precision with which datetimes are truncated when stored in fast fields. This setting is only-/// relevant for fast fields. In the docstore, datetimes are always saved with nanosecond precision.+/// Precision with which datetimes are truncated when stored in columnar fields. This setting is only+/// relevant for columnar fields. In the docstore, datetimes are always saved with nanosecond precision.#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default,)]diff --git a/crates/tantivy/src/core/json_utils.rs b/crates/tantivy/src/core/json_utils.rsindex 72d8c9e8..95523042 100644--- a/crates/tantivy/src/core/json_utils.rs+++ b/crates/tantivy/src/core/json_utils.rs@@ -147,9 +147,9 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(.get_or_allocate_unordered_id(json_path_writer.as_str()),);if let Ok(i64_val) = val.try_into() {- term_buffer.append_type_and_fast_value::<i64>(i64_val);+ term_buffer.append_type_and_columnar_value::<i64>(i64_val);} else {- term_buffer.append_type_and_fast_value(val);+ term_buffer.append_type_and_columnar_value(val);}postings_writer.subscribe(doc, 0u32, term_buffer, ctx);}@@ -159,7 +159,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(ctx.path_to_unordered_id.get_or_allocate_unordered_id(json_path_writer.as_str()),);- term_buffer.append_type_and_fast_value(val);+ term_buffer.append_type_and_columnar_value(val);postings_writer.subscribe(doc, 0u32, term_buffer, ctx);}ReferenceValueLeaf::F64(val) => {@@ -168,7 +168,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(ctx.path_to_unordered_id.get_or_allocate_unordered_id(json_path_writer.as_str()),);- term_buffer.append_type_and_fast_value(val);+ term_buffer.append_type_and_columnar_value(val);postings_writer.subscribe(doc, 0u32, term_buffer, ctx);}ReferenceValueLeaf::Bool(val) => {@@ -177,7 +177,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(ctx.path_to_unordered_id.get_or_allocate_unordered_id(json_path_writer.as_str()),);- term_buffer.append_type_and_fast_value(val);+ term_buffer.append_type_and_columnar_value(val);postings_writer.subscribe(doc, 0u32, term_buffer, ctx);}ReferenceValueLeaf::Date(val) => {@@ -186,7 +186,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(ctx.path_to_unordered_id.get_or_allocate_unordered_id(json_path_writer.as_str()),);- term_buffer.append_type_and_fast_value(val);+ term_buffer.append_type_and_columnar_value(val);postings_writer.subscribe(doc, 0u32, term_buffer, ctx);}ReferenceValueLeaf::PreTokStr(_) => {@@ -233,7 +233,10 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(/// Tries to infer a JSON type from a string and append it to the term.////// The term must be json + JSON path.-pub fn convert_to_fast_value_and_append_to_json_term(mut term: Term, phrase: &str) -> Option<Term> {+pub fn convert_to_columnar_value_and_append_to_json_term(+ mut term: Term,+ phrase: &str,+) -> Option<Term> {assert_eq!(term.value().as_json_value_bytes()@@ -245,23 +248,23 @@ pub fn convert_to_fast_value_and_append_to_json_term(mut term: Term, phrase: &st);if let Ok(dt) = OffsetDateTime::parse(phrase, &Rfc3339) {let dt_utc = dt.to_offset(UtcOffset::UTC);- term.append_type_and_fast_value(DateTime::from_utc(dt_utc));+ term.append_type_and_columnar_value(DateTime::from_utc(dt_utc));return Some(term);}if let Ok(i64_val) = str::parse::<i64>(phrase) {- term.append_type_and_fast_value(i64_val);+ term.append_type_and_columnar_value(i64_val);return Some(term);}if let Ok(u64_val) = str::parse::<u64>(phrase) {- term.append_type_and_fast_value(u64_val);+ term.append_type_and_columnar_value(u64_val);return Some(term);}if let Ok(f64_val) = str::parse::<f64>(phrase) {- term.append_type_and_fast_value(f64_val);+ term.append_type_and_columnar_value(f64_val);return Some(term);}if let Ok(bool_val) = str::parse::<bool>(phrase) {- term.append_type_and_fast_value(bool_val);+ term.append_type_and_columnar_value(bool_val);return Some(term);}None@@ -340,7 +343,7 @@ mod tests {);let mut term = Term::from_field_json_path(field, "attributes.dimensions.width", false);- term.append_type_and_fast_value(400i64);+ term.append_type_and_columnar_value(400i64);assert_eq!(format!("{term:?}"),"Term(field=1, type=Json, path=attributes.dimensions.width, type=I64, 400)"@@ -360,7 +363,7 @@ mod tests {fn test_i64_term() {let field = Field::from_field_id(1);let mut term = Term::from_field_json_path(field, "color", false);- term.append_type_and_fast_value(-4i64);+ term.append_type_and_columnar_value(-4i64);assert_eq!(term.serialized_term(),@@ -372,7 +375,7 @@ mod tests {fn test_u64_term() {let field = Field::from_field_id(1);let mut term = Term::from_field_json_path(field, "color", false);- term.append_type_and_fast_value(4u64);+ term.append_type_and_columnar_value(4u64);assert_eq!(term.serialized_term(),@@ -384,7 +387,7 @@ mod tests {fn test_f64_term() {let field = Field::from_field_id(1);let mut term = Term::from_field_json_path(field, "color", false);- term.append_type_and_fast_value(4.0f64);+ term.append_type_and_columnar_value(4.0f64);assert_eq!(term.serialized_term(),b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00"@@ -395,7 +398,7 @@ mod tests {fn test_bool_term() {let field = Field::from_field_id(1);let mut term = Term::from_field_json_path(field, "color", false);- term.append_type_and_fast_value(true);+ term.append_type_and_columnar_value(true);assert_eq!(term.serialized_term(),b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01"diff --git a/crates/tantivy/src/core/tests.rs b/crates/tantivy/src/core/tests.rsindex 7a17aaf6..1411aa8e 100644--- a/crates/tantivy/src/core/tests.rs+++ b/crates/tantivy/src/core/tests.rs@@ -411,7 +411,7 @@ fn test_non_text_json_term_freq() {let inv_idx = segment_reader.inverted_index(field).unwrap();let mut term = Term::from_field_json_path(field, "tenant_id", false);- term.append_type_and_fast_value(75i64);+ term.append_type_and_columnar_value(75i64);let postings = inv_idx.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)@@ -445,7 +445,7 @@ fn test_non_text_json_term_freq_bitpacked() {let inv_idx = segment_reader.inverted_index(field).unwrap();let mut term = Term::from_field_json_path(field, "tenant_id", false);- term.append_type_and_fast_value(75i64);+ term.append_type_and_columnar_value(75i64);let mut postings = inv_idx.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)diff --git a/crates/tantivy/src/error.rs b/crates/tantivy/src/error.rsindex 6eeca6ec..d714d907 100644--- a/crates/tantivy/src/error.rs+++ b/crates/tantivy/src/error.rs@@ -6,10 +6,10 @@ use std::{fmt, io};use thiserror::Error;+use crate::columnfield::ColumnFieldNotAvailableError;use crate::directory::error::{Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,};-use crate::fastfield::FastFieldNotAvailableError;use crate::schema;use crate::schema::document::DeserializeError;@@ -121,9 +121,9 @@ impl From<DataCorruption> for TantivyError {TantivyError::DataCorruption(data_corruption)}}-impl From<FastFieldNotAvailableError> for TantivyError {- fn from(fastfield_error: FastFieldNotAvailableError) -> TantivyError {- TantivyError::SchemaError(format!("{fastfield_error}"))+impl From<ColumnFieldNotAvailableError> for TantivyError {+ fn from(columnfield_error: ColumnFieldNotAvailableError) -> TantivyError {+ TantivyError::SchemaError(format!("{columnfield_error}"))}}impl From<LockError> for TantivyError {diff --git a/crates/tantivy/src/fastfield/error.rs b/crates/tantivy/src/fastfield/error.rsdeleted file mode 100644index b682429b..00000000--- a/crates/tantivy/src/fastfield/error.rs+++ /dev/null@@ -1,26 +0,0 @@-use std::result;--use crate::schema::FieldEntry;--/// `FastFieldNotAvailableError` is returned when the-/// user requested for a fast field reader, and the field was not-/// defined in the schema as a fast field.-#[derive(Debug, Error)]-#[error("Fast field not available: '{field_name:?}'")]-pub struct FastFieldNotAvailableError {- pub(crate) field_name: String,-}--impl FastFieldNotAvailableError {- /// Creates a `FastFieldNotAvailable` error.- /// `field_entry` is the configuration of the field- /// for which fast fields are not available.- pub fn new(field_entry: &FieldEntry) -> FastFieldNotAvailableError {- FastFieldNotAvailableError {- field_name: field_entry.name().to_string(),- }- }-}--/// Result when trying to access a fast field reader.-pub type Result<R> = result::Result<R, FastFieldNotAvailableError>;diff --git a/crates/tantivy/src/functional_test.rs b/crates/tantivy/src/functional_test.rsindex e864a50c..b58ea118 100644--- a/crates/tantivy/src/functional_test.rs+++ b/crates/tantivy/src/functional_test.rs@@ -63,7 +63,7 @@ fn get_num_iterations() -> usize {fn test_functional_indexing_sorted() -> crate::Result<()> {let mut schema_builder = Schema::builder();- let id_field = schema_builder.add_u64_field("id", INDEXED | FAST);+ let id_field = schema_builder.add_u64_field("id", INDEXED | COLUMN);let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);let text_field_options = TextOptions::default().set_indexing_options(diff --git a/crates/tantivy/src/index/index.rs b/crates/tantivy/src/index/index.rsindex af2593e2..5d589878 100644--- a/crates/tantivy/src/index/index.rs+++ b/crates/tantivy/src/index/index.rs@@ -91,7 +91,7 @@ fn save_new_metas(/// let body_field = schema_builder.add_text_field("body", TEXT);/// let number_field = schema_builder.add_u64_field(/// "number",-/// NumericOptions::default().set_fast(),+/// NumericOptions::default().set_columnar(),/// );////// let schema = schema_builder.build();@@ -105,7 +105,7 @@ pub struct IndexBuilder {schema: Option<Schema>,index_settings: IndexSettings,tokenizer_manager: TokenizerManager,- fast_field_tokenizer_manager: TokenizerManager,+ column_field_tokenizer_manager: TokenizerManager,}impl Default for IndexBuilder {fn default() -> Self {@@ -119,7 +119,7 @@ impl IndexBuilder {schema: None,index_settings: IndexSettings::default(),tokenizer_manager: TokenizerManager::default(),- fast_field_tokenizer_manager: TokenizerManager::default(),+ column_field_tokenizer_manager: TokenizerManager::default(),}}@@ -143,9 +143,9 @@ impl IndexBuilder {self}- /// Set the fast field tokenizers.- pub fn fast_field_tokenizers(mut self, tokenizers: TokenizerManager) -> Self {- self.fast_field_tokenizer_manager = tokenizers;+ /// Set the columnar field tokenizers.+ pub fn column_field_tokenizers(mut self, tokenizers: TokenizerManager) -> Self {+ self.column_field_tokenizer_manager = tokenizers;self}@@ -241,9 +241,9 @@ impl IndexBuilder {))})?;let entry = schema.get_field_entry(schema_field);- if !entry.is_fast() {+ if !entry.is_columnar() {return Err(TantivyError::InvalidArgument(format!(- "Field {} is no fast field. Field needs to be a single value fast field \+ "Field {} is no columnar field. Field needs to be a single value columnar field \to be used to sort an index",sort_by_field.field)));@@ -281,7 +281,7 @@ impl IndexBuilder {metas.index_settings = self.index_settings;let mut index = Index::open_from_metas(directory, &metas, SegmentMetaInventory::default());index.set_tokenizers(self.tokenizer_manager);- index.set_fast_field_tokenizers(self.fast_field_tokenizer_manager);+ index.set_column_field_tokenizers(self.column_field_tokenizer_manager);Ok(index)}}@@ -294,7 +294,7 @@ pub struct Index {settings: IndexSettings,executor: Executor,tokenizers: TokenizerManager,- fast_field_tokenizers: TokenizerManager,+ column_field_tokenizers: TokenizerManager,inventory: SegmentMetaInventory,}@@ -412,7 +412,7 @@ impl Index {directory,schema,tokenizers: TokenizerManager::default(),- fast_field_tokenizers: TokenizerManager::default(),+ column_field_tokenizers: TokenizerManager::default(),executor: Executor::single_thread(),inventory,}@@ -428,14 +428,14 @@ impl Index {&self.tokenizers}- /// Setter for the fast field tokenizer manager.- pub fn set_fast_field_tokenizers(&mut self, tokenizers: TokenizerManager) {- self.fast_field_tokenizers = tokenizers;+ /// Setter for the columnar field tokenizer manager.+ pub fn set_column_field_tokenizers(&mut self, tokenizers: TokenizerManager) {+ self.column_field_tokenizers = tokenizers;}- /// Accessor for the fast field tokenizer manager.- pub fn fast_field_tokenizer(&self) -> &TokenizerManager {- &self.fast_field_tokenizers+ /// Accessor for the columnar field tokenizer manager.+ pub fn column_field_tokenizer(&self) -> &TokenizerManager {+ &self.column_field_tokenizers}/// Get the tokenizer associated with a specific field.@@ -510,7 +510,7 @@ impl Index {/// browsing through the inverted index term dictionary and the columnar field dictionary.////// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json- /// field that is not indexed nor a fast field but is stored, it is possible for the field+ /// field that is not indexed nor a columnar field but is stored, it is possible for the field/// to not be listed.pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {let segments = self.searchable_segments()?;diff --git a/crates/tantivy/src/index/index_meta.rs b/crates/tantivy/src/index/index_meta.rsindex 03e9bbd3..73822d44 100644--- a/crates/tantivy/src/index/index_meta.rs+++ b/crates/tantivy/src/index/index_meta.rs@@ -139,7 +139,7 @@ impl SegmentMeta {SegmentComponent::Terms => ".term".to_string(),SegmentComponent::Store => ".store".to_string(),SegmentComponent::TempStore => ".store.temp".to_string(),- SegmentComponent::FastFields => ".fast".to_string(),+ SegmentComponent::ColumnFields => ".fast".to_string(),SegmentComponent::FieldNorms => ".fieldnorm".to_string(),});PathBuf::from(path)@@ -435,7 +435,7 @@ mod tests {let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");assert_eq!(json,- r#"{"index_settings":{"sort_by_field":{"field":"text","order":"Asc"},"docstore_compression":"lz4","docstore_blocksize":16384},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false,"fast":false}}],"opstamp":0}"#+ r#"{"index_settings":{"sort_by_field":{"field":"text","order":"Asc"},"docstore_compression":"lz4","docstore_blocksize":16384},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false,"columnar":false}}],"opstamp":0}"#);let deser_meta: UntrackedIndexMeta = serde_json::from_str(&json).unwrap();diff --git a/crates/tantivy/src/index/segment_component.rs b/crates/tantivy/src/index/segment_component.rsindex 4ccbf06e..75714069 100644--- a/crates/tantivy/src/index/segment_component.rs+++ b/crates/tantivy/src/index/segment_component.rs@@ -11,9 +11,9 @@ pub enum SegmentComponent {/// Positions of terms in each document.Positions,/// Column-oriented random-access storage of fields.- FastFields,+ ColumnFields,/// Stores the sum of the length (in terms) of each field for each document.- /// Field norms are stored as a special u64 fast field.+ /// Field norms are stored as a special u64 columnar field.FieldNorms,/// Dictionary associating `Term`s to `TermInfo`s which is/// simply an address into the `postings` file and the `positions` file.@@ -32,7 +32,7 @@ impl SegmentComponent {static SEGMENT_COMPONENTS: [SegmentComponent; 7] = [SegmentComponent::Postings,SegmentComponent::Positions,- SegmentComponent::FastFields,+ SegmentComponent::ColumnFields,SegmentComponent::FieldNorms,SegmentComponent::Terms,SegmentComponent::Store,diff --git a/crates/tantivy/src/index/segment_reader.rs b/crates/tantivy/src/index/segment_reader.rsindex ce4dbd10..f41ddb3c 100644--- a/crates/tantivy/src/index/segment_reader.rs+++ b/crates/tantivy/src/index/segment_reader.rs@@ -6,9 +6,9 @@ use std::{fmt, io};use fnv::FnvHashMap;use itertools::Itertools;+use crate::columnfield::ColumnFieldReaders;use crate::directory::{CompositeFile, FileSlice};use crate::error::DataCorruption;-use crate::fastfield::FastFieldReaders;use crate::fieldnorm::{FieldNormReader, FieldNormReaders};use crate::index::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};use crate::json_utils::json_path_sep_to_dot;@@ -23,7 +23,7 @@ use crate::DocId;/// - term dictionary/// - postings/// - store-/// - fast field readers+/// - columnar field readers/// - field norm reader////// The segment reader has a very low memory footprint,@@ -39,7 +39,7 @@ pub struct SegmentReader {termdict_composite: CompositeFile,postings_composite: CompositeFile,positions_composite: CompositeFile,- fast_fields_readers: FastFieldReaders,+ column_fields_readers: ColumnFieldReaders,fieldnorm_readers: FieldNormReaders,store_file: FileSlice,@@ -63,18 +63,18 @@ impl SegmentReader {&self.schema}- /// Accessor to a segment's fast field reader given a field.+ /// Accessor to a segment's columnar field reader given a field.///- /// Returns the u64 fast value reader if the field- /// is a u64 field indexed as "fast".+ /// Returns the u64 columnar value reader if the field+ /// is a u64 field indexed as "columnar".///- /// Return a FastFieldNotAvailableError if the field is not- /// declared as a fast field in the schema.+ /// Return a ColumnFieldNotAvailableError if the field is not+ /// declared as a columnar field in the schema.////// # Panics/// May panic if the index is corrupted.- pub fn fast_fields(&self) -> &FastFieldReaders {- &self.fast_fields_readers+ pub fn column_fields(&self) -> &ColumnFieldReaders {+ &self.column_fields_readers}/// Accessor to the segment's `Field norms`'s reader.@@ -82,7 +82,7 @@ impl SegmentReader {/// Field norms are the length (in tokens) of the fields./// It is used in the computation of the [TfIdf](https://fulmicoton.gitbooks.io/tantivy-doc/content/tfidf.html).///- /// They are simply stored as a fast field, serialized in+ /// They are simply stored as a columnar field, serialized in/// the `.fieldnorm` file of the segment.pub fn get_fieldnorms_reader(&self, field: Field) -> crate::Result<FieldNormReader> {self.fieldnorm_readers.get_field(field)?.ok_or_else(|| {@@ -130,8 +130,8 @@ impl SegmentReader {let schema = segment.schema();- let fast_fields_data = segment.open_read(SegmentComponent::FastFields)?;- let fast_fields_readers = FastFieldReaders::open(fast_fields_data, schema.clone())?;+ let column_fields_data = segment.open_read(SegmentComponent::ColumnFields)?;+ let column_fields_readers = ColumnFieldReaders::open(column_fields_data, schema.clone())?;let fieldnorm_data = segment.open_read(SegmentComponent::FieldNorms)?;let fieldnorm_readers = FieldNormReaders::open(fieldnorm_data)?;@@ -142,7 +142,7 @@ impl SegmentReader {max_doc,termdict_composite,postings_composite,- fast_fields_readers,+ column_fields_readers,fieldnorm_readers,segment_id: segment.id(),store_file,@@ -240,7 +240,7 @@ impl SegmentReader {/// browsing through the inverted index term dictionary and the columnar field dictionary.////// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json- /// field that is not indexed nor a fast field but is stored, it is possible for the field+ /// field that is not indexed nor a columnar field but is stored, it is possible for the field/// to not be listed.pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {let mut indexed_fields: Vec<FieldMetadata> = Vec::new();@@ -255,7 +255,7 @@ impl SegmentReader {let inv_index = self.inverted_index(field)?;let encoded_fields_in_index = inv_index.list_encoded_fields()?;let mut build_path = |field_name: &str, mut json_path: String| {- // In this case we need to map the potential fast field to the field name+ // In this case we need to map the potential columnar field to the field name// accepted by the query parser.let create_canonical =!field_entry.is_expand_dots_enabled() && json_path.contains('.');@@ -280,7 +280,7 @@ impl SegmentReader {indexed: true,stored: false,field_name,- fast: false,+ columnar: false,typ,}),);@@ -289,14 +289,14 @@ impl SegmentReader {indexed: true,stored: false,field_name: field_name.to_string(),- fast: false,+ columnar: false,typ: field_entry.field_type().value_type(),});}}}- let mut fast_fields: Vec<FieldMetadata> = self- .fast_fields()+ let mut column_fields: Vec<FieldMetadata> = self+ .column_fields().columnar().iter_columns()?.map(|(mut field_name, handle)| {@@ -311,17 +311,17 @@ impl SegmentReader {indexed: false,stored: false,field_name,- fast: true,+ columnar: true,typ: Type::from(handle.column_type()),}}).collect();- // Since the type is encoded differently in the fast field and in the inverted index,+ // Since the type is encoded differently in the columnar field and in the inverted index,// the order of the fields is not guaranteed to be the same. Therefore, we sort the fields.// If we are sure that the order is the same, we can remove this sort.indexed_fields.sort_unstable();- fast_fields.sort_unstable();- let merged = merge_field_meta_data(vec![indexed_fields, fast_fields], &self.schema);+ column_fields.sort_unstable();+ let merged = merge_field_meta_data(vec![indexed_fields, column_fields], &self.schema);Ok(merged)}@@ -343,7 +343,7 @@ impl SegmentReader {self.termdict_composite.space_usage(),self.postings_composite.space_usage(),self.positions_composite.space_usage(),- self.fast_fields_readers.space_usage(self.schema())?,+ self.column_fields_readers.space_usage(self.schema())?,self.fieldnorm_readers.space_usage(),self.get_store_reader(0)?.space_usage(),))@@ -366,7 +366,7 @@ pub struct FieldMetadata {/// Is the field stored in the doc storepub stored: bool,/// Is the field stored in the columnar storage- pub fast: bool,+ pub columnar: bool,}impl BitOrAssign for FieldMetadata {fn bitor_assign(&mut self, rhs: Self) {@@ -374,7 +374,7 @@ impl BitOrAssign for FieldMetadata {assert!(self.typ == rhs.typ);self.indexed |= rhs.indexed;self.stored |= rhs.stored;- self.fast |= rhs.fast;+ self.columnar |= rhs.columnar;}}@@ -430,14 +430,14 @@ mod test {typ: crate::schema::Type::Str,indexed: true,stored: false,- fast: true,+ columnar: true,};let field_metadata2 = FieldMetadata {field_name: "a".to_string(),typ: crate::schema::Type::Str,indexed: true,stored: false,- fast: true,+ columnar: true,};let res = merge_field_meta_data(vec![vec![field_metadata1.clone()], vec![field_metadata2]],@@ -453,21 +453,21 @@ mod test {typ: crate::schema::Type::Str,indexed: false,stored: false,- fast: true,+ columnar: true,};let field_metadata2 = FieldMetadata {field_name: "b".to_string(),typ: crate::schema::Type::Str,indexed: false,stored: false,- fast: true,+ columnar: true,};let field_metadata3 = FieldMetadata {field_name: "a".to_string(),typ: crate::schema::Type::Str,indexed: true,stored: false,- fast: false,+ columnar: false,};let res = merge_field_meta_data(vec![@@ -481,7 +481,7 @@ mod test {typ: crate::schema::Type::Str,indexed: true,stored: false,- fast: true,+ columnar: true,};assert_eq!(res, vec![field_metadata_expected1, field_metadata2.clone()]);}@@ -493,7 +493,7 @@ mod test {typ,indexed: false,stored: false,- fast: true,+ columnar: true,};let schema = SchemaBuilder::new().build();let mut metas = vec![get_meta_data("d", Type::Str), get_meta_data("e", Type::U64)];@@ -515,21 +515,21 @@ mod test {typ: crate::schema::Type::Str,indexed: false,stored: false,- fast: true,+ columnar: true,};let field_metadata2 = FieldMetadata {field_name: "a".to_string(),typ: crate::schema::Type::Str,indexed: true,stored: false,- fast: false,+ columnar: false,};let field_metadata_expected = FieldMetadata {field_name: "a".to_string(),typ: crate::schema::Type::Str,indexed: true,stored: false,- fast: true,+ columnar: true,};let mut res1 = field_metadata1.clone();res1 |= field_metadata2.clone();diff --git a/crates/tantivy/src/indexer/doc_id_mapping.rs b/crates/tantivy/src/indexer/doc_id_mapping.rsindex db204248..c21fd0a8 100644--- a/crates/tantivy/src/indexer/doc_id_mapping.rs+++ b/crates/tantivy/src/indexer/doc_id_mapping.rs@@ -135,12 +135,12 @@ pub(crate) fn get_doc_id_mapping_from_field() -> crate::Result<DocIdMapping> {let schema = segment_writer.segment_serializer.segment().schema();expect_field_id_for_sort_field(&schema, &sort_by_field)?; // for now expect- let new_doc_id_to_old = segment_writer.fast_field_writers.sort_order(+ let new_doc_id_to_old = segment_writer.column_field_writers.sort_order(sort_by_field.field.as_str(),segment_writer.max_doc(),sort_by_field.order.is_desc(),);- // create new doc_id to old doc_id index (used in fast_field_writers)+ // create new doc_id to old doc_id index (used in column_field_writers)Ok(DocIdMapping::from_new_id_to_old_id(new_doc_id_to_old))}@@ -164,10 +164,10 @@ mod tests_indexsorting {let my_text_field = schema_builder.add_text_field("text_field", text_field_options);let my_string_field = schema_builder.add_text_field("string_field", STRING | STORED);let my_number =- schema_builder.add_u64_field("my_number", NumericOptions::default().set_fast());+ schema_builder.add_u64_field("my_number", NumericOptions::default().set_columnar());let multi_numbers =- schema_builder.add_u64_field("multi_numbers", NumericOptions::default().set_fast());+ schema_builder.add_u64_field("multi_numbers", NumericOptions::default().set_columnar());let schema = schema_builder.build();let mut index_builder = Index::builder().schema(schema);@@ -440,7 +440,7 @@ mod tests_indexsorting {}#[test]- fn test_sort_index_fast_field() -> crate::Result<()> {+ fn test_sort_index_column_field() -> crate::Result<()> {let index = create_test_index(Some(IndexSettings {sort_by_field: Some(IndexSortByField {@@ -459,12 +459,12 @@ mod tests_indexsorting {let searcher = index.reader()?.searcher();assert_eq!(searcher.segment_readers().len(), 1);let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();+ let column_fields = segment_reader.column_fields();- let fast_field = fast_fields.u64("my_number").unwrap().values;- assert_eq!(fast_field.get_val(0), 10u64);- assert_eq!(fast_field.get_val(1), 20u64);- assert_eq!(fast_field.get_val(2), 30u64);+ let column_field = column_fields.u64("my_number").unwrap().values;+ assert_eq!(column_field.get_val(0), 10u64);+ assert_eq!(column_field.get_val(1), 20u64);+ assert_eq!(column_field.get_val(2), 30u64);Ok(())}@@ -472,7 +472,7 @@ mod tests_indexsorting {#[test]fn test_with_sort_by_date_field() -> crate::Result<()> {let mut schema_builder = Schema::builder();- let date_field = schema_builder.add_date_field("date", INDEXED | STORED | FAST);+ let date_field = schema_builder.add_date_field("date", INDEXED | STORED | COLUMN);let schema = schema_builder.build();let settings = IndexSettings {@@ -504,12 +504,12 @@ mod tests_indexsorting {let searcher = index.reader()?.searcher();assert_eq!(searcher.segment_readers().len(), 1);let segment_reader = searcher.segment_reader(0);- let fast_fields = segment_reader.fast_fields();+ let column_fields = segment_reader.column_fields();- let fast_field = fast_fields.date("date").unwrap().values;- assert_eq!(fast_field.get_val(0), DateTime::from_timestamp_secs(1001));- assert_eq!(fast_field.get_val(1), DateTime::from_timestamp_secs(1000));- assert_eq!(fast_field.get_val(2), DateTime::from_timestamp_secs(999));+ let column_field = column_fields.date("date").unwrap().values;+ assert_eq!(column_field.get_val(0), DateTime::from_timestamp_secs(1001));+ assert_eq!(column_field.get_val(1), DateTime::from_timestamp_secs(1000));+ assert_eq!(column_field.get_val(2), DateTime::from_timestamp_secs(999));Ok(())}@@ -539,7 +539,7 @@ mod tests_indexsorting {#[test]fn test_text_sort() -> crate::Result<()> {let mut schema_builder = SchemaBuilder::new();- schema_builder.add_text_field("id", STRING | FAST | STORED);+ schema_builder.add_text_field("id", STRING | COLUMN | STORED);schema_builder.add_text_field("name", TEXT | STORED);let resp = IndexBuilder::new()diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rsindex 1c4119a5..08796af9 100644--- a/crates/tantivy/src/indexer/index_writer.rs+++ b/crates/tantivy/src/indexer/index_writer.rs@@ -589,7 +589,7 @@ mod tests {use crate::indexer::NoMergePolicy;use crate::query::{QueryParser, TermQuery};use crate::schema::{- self, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Value, FAST, INDEXED,+ self, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Value, COLUMN, INDEXED,STORED, STRING, TEXT,};use crate::store::DOCSTORE_CACHE_CAPACITY;@@ -1138,13 +1138,13 @@ mod tests {force_end_merge: bool,) -> crate::Result<Index> {let mut schema_builder = schema::Schema::builder();- let json_field = schema_builder.add_json_field("json", FAST | TEXT | STORED);+ let json_field = schema_builder.add_json_field("json", COLUMN | TEXT | STORED);let i64_field = schema_builder.add_i64_field("i64", INDEXED);- let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);+ let id_field = schema_builder.add_u64_field("id", COLUMN | INDEXED | STORED);let f64_field = schema_builder.add_f64_field("f64", INDEXED);let date_field = schema_builder.add_date_field("date", INDEXED);- let bytes_field = schema_builder.add_bytes_field("bytes", FAST | INDEXED | STORED);- let bool_field = schema_builder.add_bool_field("bool", FAST | INDEXED | STORED);+ let bytes_field = schema_builder.add_bytes_field("bytes", COLUMN | INDEXED | STORED);+ let bool_field = schema_builder.add_bool_field("bool", COLUMN | INDEXED | STORED);let text_field = schema_builder.add_text_field("text_field",TextOptions::default()@@ -1262,7 +1262,7 @@ mod tests {.segment_readers().iter().flat_map(|segment_reader| {- let ff_reader = segment_reader.fast_fields().u64("id").unwrap();+ let ff_reader = segment_reader.column_fields().u64("id").unwrap();segment_reader.doc_ids().flat_map(move |doc| ff_reader.first(doc).into_iter())@@ -1273,7 +1273,7 @@ mod tests {.segment_readers().iter().flat_map(|segment_reader| {- let ff_reader = segment_reader.fast_fields().u64("id").unwrap();+ let ff_reader = segment_reader.column_fields().u64("id").unwrap();segment_reader.doc_ids().flat_map(move |doc| ff_reader.first(doc).into_iter())@@ -1305,13 +1305,13 @@ mod tests {);if force_end_merge && num_segments_before_merge > 1 && num_segments_after_merge == 1 {- // Test fastfield num_docs+ // Test columnfield num_docslet num_docs: usize = searcher.segment_readers().iter().map(|segment_reader| {let ff_reader = segment_reader- .fast_fields()+ .column_fields().column_opt::<i64>("i64").unwrap().unwrap();@@ -1409,7 +1409,7 @@ mod tests {// load all id in each segment and check they are in orderfor reader in searcher.segment_readers() {- let (ff_reader, _) = reader.fast_fields().u64_lenient("id").unwrap().unwrap();+ let (ff_reader, _) = reader.column_fields().u64_lenient("id").unwrap().unwrap();let mut ids_in_segment: Vec<u64> = Vec::new();for doc in 0..reader.num_docs() {@@ -1430,7 +1430,7 @@ mod tests {}#[test]- fn test_fast_field_range() {+ fn test_column_field_range() {let ops: Vec<_> = (0..1000).map(IndexingOp::add).collect();assert!(test_operation_strategy(&ops, false, true).is_ok());}@@ -1487,8 +1487,8 @@ mod tests {#[test]fn test_minimal_sort() {let mut schema_builder = Schema::builder();- let val = schema_builder.add_u64_field("val", FAST);- let id = schema_builder.add_u64_field("id", FAST);+ let val = schema_builder.add_u64_field("val", COLUMN);+ let id = schema_builder.add_u64_field("id", COLUMN);let schema = schema_builder.build();let settings = IndexSettings {sort_by_field: Some(IndexSortByField {@@ -1517,7 +1517,7 @@ mod tests {let searcher = reader.searcher();let segment_reader = searcher.segment_reader(0);let id_col: Column = segment_reader- .fast_fields()+ .column_fields().column_opt("id").unwrap().unwrap();@@ -1594,7 +1594,7 @@ mod tests {#[test]fn test_delete_with_sort_by_field_last_opstamp_is_not_max() -> crate::Result<()> {let mut schema_builder = schema::Schema::builder();- let sort_by_field = schema_builder.add_u64_field("sort_by", FAST);+ let sort_by_field = schema_builder.add_u64_field("sort_by", COLUMN);let id_field = schema_builder.add_u64_field("id", INDEXED);let schema = schema_builder.build();diff --git a/crates/tantivy/src/indexer/merger.rs b/crates/tantivy/src/indexer/merger.rsindex a48a4251..77944cb7 100644--- a/crates/tantivy/src/indexer/merger.rs+++ b/crates/tantivy/src/indexer/merger.rs@@ -7,10 +7,10 @@ use crate::columnar::{use itertools::Itertools;use measure_time::debug_time;+use crate::columnfield::ColumnFieldNotAvailableError;use crate::directory::WritePtr;use crate::docset::{DocSet, TERMINATED};use crate::error::DataCorruption;-use crate::fastfield::FastFieldNotAvailableError;use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};use crate::index::{Segment, SegmentComponent, SegmentReader};use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};@@ -98,11 +98,11 @@ fn convert_to_merge_order(}}-fn extract_fast_field_required_columns(schema: &Schema) -> Vec<(String, ColumnType)> {+fn extract_column_field_required_columns(schema: &Schema) -> Vec<(String, ColumnType)> {schema.fields().map(|(_, field_entry)| field_entry)- .filter(|field_entry| field_entry.is_fast())+ .filter(|field_entry| field_entry.is_columnar()).filter_map(|field_entry| {let column_name = field_entry.name().to_string();let column_type = value_type_to_column_type(field_entry.field_type().value_type())?;@@ -194,24 +194,24 @@ impl IndexMerger {Ok(())}- fn write_fast_fields(+ fn write_column_fields(&self,- fast_field_wrt: &mut WritePtr,+ column_field_wrt: &mut WritePtr,doc_id_mapping: SegmentDocIdMapping,) -> crate::Result<()> {- debug_time!("write-fast-fields");- let required_columns = extract_fast_field_required_columns(&self.schema);+ debug_time!("write-columnar-fields");+ let required_columns = extract_column_field_required_columns(&self.schema);let columnars: Vec<&ColumnarReader> = self.readers.iter()- .map(|reader| reader.fast_fields().columnar())+ .map(|reader| reader.column_fields().columnar()).collect();let merge_row_order = convert_to_merge_order(&columnars[..], doc_id_mapping);crate::columnar::merge_columnar(&columnars[..],&required_columns,merge_row_order,- fast_field_wrt,+ column_field_wrt,)?;Ok(())}@@ -245,9 +245,9 @@ impl IndexMerger {) -> crate::Result<Arc<dyn ColumnValues>> {reader.schema().get_field(&sort_by_field.field)?;let (value_accessor, _column_type) = reader- .fast_fields()+ .column_fields().u64_lenient(&sort_by_field.field)?- .ok_or_else(|| FastFieldNotAvailableError {+ .ok_or_else(|| ColumnFieldNotAvailableError {field_name: sort_by_field.field.to_string(),})?;Ok(value_accessor.values)@@ -656,8 +656,8 @@ impl IndexMerger {debug!("write-storagefields");self.write_storable_fields(serializer.get_store_writer(), &doc_id_mapping)?;- debug!("write-fastfields");- self.write_fast_fields(serializer.get_fast_field_write(), doc_id_mapping)?;+ debug!("write-columnfields");+ self.write_column_fields(serializer.get_column_field_write(), doc_id_mapping)?;debug!("close-serializer");serializer.close()?;@@ -668,10 +668,10 @@ impl IndexMerger {#[cfg(test)]mod tests {- use schema::FAST;+ use schema::COLUMN;use crate::collector::tests::{- BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,+ BytesColumnFieldTestCollector, ColumnFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,};use crate::index::{Index, SegmentId};use crate::query::{BooleanQuery, EnableScoring, Scorer, TermQuery};@@ -691,9 +691,9 @@ mod tests {.set_stored();let text_field = schema_builder.add_text_field("text", text_fieldtype);let date_field = schema_builder.add_date_field("date", INDEXED);- let score_fieldtype = schema::NumericOptions::default().set_fast();+ let score_fieldtype = schema::NumericOptions::default().set_columnar();let score_field = schema_builder.add_u64_field("score", score_fieldtype);- let bytes_score_field = schema_builder.add_bytes_field("score_bytes", FAST);+ let bytes_score_field = schema_builder.add_bytes_field("score_bytes", COLUMN);let index = Index::create_in_ram(schema_builder.build());let reader = index.reader()?;let curr_time = OffsetDateTime::now_utc();@@ -814,23 +814,23 @@ mod tests {}{- let get_fast_vals = |terms: Vec<Term>| {+ let get_columnar_vals = |terms: Vec<Term>| {let query = BooleanQuery::new_multiterms_query(terms);- searcher.search(&query, &FastFieldTestCollector::for_field("score"))+ searcher.search(&query, &ColumnFieldTestCollector::for_field("score"))};- let get_fast_vals_bytes = |terms: Vec<Term>| {+ let get_columnar_vals_bytes = |terms: Vec<Term>| {let query = BooleanQuery::new_multiterms_query(terms);searcher.search(&query,- &BytesFastFieldTestCollector::for_field("score_bytes"),+ &BytesColumnFieldTestCollector::for_field("score_bytes"),)};assert_eq!(- get_fast_vals(vec![Term::from_field_text(text_field, "a")])?,+ get_columnar_vals(vec![Term::from_field_text(text_field, "a")])?,vec![5, 7, 13]);assert_eq!(- get_fast_vals_bytes(vec![Term::from_field_text(text_field, "a")])?,+ get_columnar_vals_bytes(vec![Term::from_field_text(text_field, "a")])?,vec![0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0, 13]);}@@ -849,13 +849,13 @@ mod tests {// let text_field = schema_builder.add_text_field("text", text_fieldtype);// let score_fieldtype = schema::NumericOptions::default().set_fast();// let score_field = schema_builder.add_u64_field("score", score_fieldtype);- // let bytes_score_field = schema_builder.add_bytes_field("score_bytes", FAST);+ // let bytes_score_field = schema_builder.add_bytes_field("score_bytes", COLUMN);// let index = Index::create_in_ram(schema_builder.build());// let mut index_writer = index.writer_for_tests()?;// let reader = index.reader().unwrap();// let search_term = |searcher: &Searcher, term: Term| {- // let collector = FastFieldTestCollector::for_field("score");- // // let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);+ // let collector = ColumnFieldTestCollector::for_field("score");+ // // let bytes_collector = BytesColumnFieldTestCollector::for_field(bytes_score_field);// let term_query = TermQuery::new(term, IndexRecordOption::Basic);// // searcher// // .search(&term_query, &(collector, bytes_collector))@@ -976,7 +976,7 @@ mod tests {// let score_field_reader = searcher// .segment_reader(0)- // .fast_fields()+ // .column_fields()// .u64("score")// .unwrap();// assert_eq!(score_field_reader.min_value(), 4000);@@ -984,7 +984,7 @@ mod tests {// let score_field_reader = searcher// .segment_reader(1)- // .fast_fields()+ // .column_fields()// .u64("score")// .unwrap();// assert_eq!(score_field_reader.min_value(), 1);@@ -1030,7 +1030,7 @@ mod tests {// );// let score_field_reader = searcher// .segment_reader(0)- // .fast_fields()+ // .column_fields()// .u64("score")// .unwrap();// assert_eq!(score_field_reader.min_value(), 3);@@ -1077,7 +1077,7 @@ mod tests {// );// let score_field_reader = searcher// .segment_reader(0)- // .fast_fields()+ // .column_fields()// .u64("score")// .unwrap();// assert_eq!(score_field_reader.min_value(), 3);@@ -1124,7 +1124,7 @@ mod tests {// );// let score_field_reader = searcher// .segment_reader(0)- // .fast_fields()+ // .column_fields()// .u64("score")// .unwrap();// assert_eq!(score_field_reader.min_value(), 6000);@@ -1179,10 +1179,10 @@ mod tests {// }#[test]- fn merges_f64_fast_fields_correctly() -> crate::Result<()> {+ fn merges_f64_column_fields_correctly() -> crate::Result<()> {let mut builder = schema::SchemaBuilder::new();- let field = builder.add_f64_field("f64", schema::FAST);+ let field = builder.add_f64_field("f64", schema::COLUMN);let index = Index::create_in_ram(builder.build());diff --git a/crates/tantivy/src/indexer/merger_sorted_index_test.rs b/crates/tantivy/src/indexer/merger_sorted_index_test.rsindex 1432eb2d..dfe74722 100644--- a/crates/tantivy/src/indexer/merger_sorted_index_test.rs+++ b/crates/tantivy/src/indexer/merger_sorted_index_test.rs@@ -14,7 +14,7 @@ mod tests {fn create_test_index_posting_list_issue(index_settings: Option<IndexSettings>) -> Index {let mut schema_builder = schema::Schema::builder();- let int_options = NumericOptions::default().set_fast().set_indexed();+ let int_options = NumericOptions::default().set_columnar().set_indexed();let int_field = schema_builder.add_u64_field("intval", int_options);let schema = schema_builder.build();@@ -54,7 +54,7 @@ mod tests {) -> crate::Result<Index> {let mut schema_builder = schema::Schema::builder();let int_options = NumericOptions::default()- .set_fast()+ .set_columnar().set_stored().set_indexed();let int_field = schema_builder.add_u64_field("intval", int_options);@@ -165,21 +165,21 @@ mod tests {assert_eq!(searcher.segment_readers().len(), 1);let segment_reader = searcher.segment_readers().last().unwrap();- let fast_fields = segment_reader.fast_fields();- let fast_field = fast_fields.u64("intval").unwrap();- assert_eq!(fast_field.first(7), Some(1u64));+ let column_fields = segment_reader.column_fields();+ let column_field = column_fields.u64("intval").unwrap();+ assert_eq!(column_field.first(7), Some(1u64));if force_disjunct_segment_sort_values {- assert_eq!(fast_field.first(6), Some(2u64));- assert_eq!(fast_field.first(5), Some(3u64));- assert_eq!(fast_field.first(4), Some(10u64));- assert_eq!(fast_field.first(3), Some(20u64));+ assert_eq!(column_field.first(6), Some(2u64));+ assert_eq!(column_field.first(5), Some(3u64));+ assert_eq!(column_field.first(4), Some(10u64));+ assert_eq!(column_field.first(3), Some(20u64));} else {- assert_eq!(fast_field.first(6), Some(1u64));- assert_eq!(fast_field.first(5), Some(2u64));- assert_eq!(fast_field.first(4), Some(3u64));- assert_eq!(fast_field.first(3), Some(5u64));+ assert_eq!(column_field.first(6), Some(1u64));+ assert_eq!(column_field.first(5), Some(2u64));+ assert_eq!(column_field.first(4), Some(3u64));+ assert_eq!(column_field.first(3), Some(5u64));}- assert_eq!(fast_field.first(0), Some(1_000u64));+ assert_eq!(column_field.first(0), Some(1_000u64));// test new field norm mapping{@@ -356,33 +356,33 @@ mod tests {// assert_eq!(searcher.segment_readers().len(), 1);// let segment_reader = searcher.segment_readers().last().unwrap();- // let fast_fields = segment_reader.fast_fields();- // let fast_field = fast_fields.u64(int_field).unwrap();- // assert_eq!(fast_field.get_val(0), 1u64);- // assert_eq!(fast_field.get_val(1), 2u64);- // assert_eq!(fast_field.get_val(2), 3u64);- // assert_eq!(fast_field.get_val(3), 10u64);- // assert_eq!(fast_field.get_val(4), 20u64);- // assert_eq!(fast_field.get_val(5), 1_000u64);+ // let column_fields = segment_reader.column_fields();+ // let column_field = column_fields.u64(int_field).unwrap();+ // assert_eq!(column_field.get_val(0), 1u64);+ // assert_eq!(column_field.get_val(1), 2u64);+ // assert_eq!(column_field.get_val(2), 3u64);+ // assert_eq!(column_field.get_val(3), 10u64);+ // assert_eq!(column_field.get_val(4), 20u64);+ // assert_eq!(column_field.get_val(5), 1_000u64);- // let get_vals = |fast_field: &MultiValuedFastFieldReader<u64>, doc_id: u32| -> Vec<u64> {+ // let get_vals = |column_field: &MultiValuedColumnFieldReader<u64>, doc_id: u32| -> Vec<u64> {// let mut vals = vec![];- // fast_field.get_vals(doc_id, &mut vals);+ // column_field.get_vals(doc_id, &mut vals);// vals// };- // let fast_fields = segment_reader.fast_fields();- // let fast_field = fast_fields.u64s(multi_numbers).unwrap();- // assert_eq!(&get_vals(&fast_field, 0), &[] as &[u64]);- // assert_eq!(&get_vals(&fast_field, 1), &[2, 3]);- // assert_eq!(&get_vals(&fast_field, 2), &[3, 4]);- // assert_eq!(&get_vals(&fast_field, 3), &[10, 11]);- // assert_eq!(&get_vals(&fast_field, 4), &[20]);- // assert_eq!(&get_vals(&fast_field, 5), &[1001, 1002]);-- // let fast_field = fast_fields.bytes(bytes_field).unwrap();- // assert_eq!(fast_field.get_bytes(0), &[] as &[u8]);- // assert_eq!(fast_field.get_bytes(2), &[1, 2, 3]);- // assert_eq!(fast_field.get_bytes(5), &[5, 5]);+ // let column_fields = segment_reader.column_fields();+ // let column_field = column_fields.u64s(multi_numbers).unwrap();+ // assert_eq!(&get_vals(&column_field, 0), &[] as &[u64]);+ // assert_eq!(&get_vals(&column_field, 1), &[2, 3]);+ // assert_eq!(&get_vals(&column_field, 2), &[3, 4]);+ // assert_eq!(&get_vals(&column_field, 3), &[10, 11]);+ // assert_eq!(&get_vals(&column_field, 4), &[20]);+ // assert_eq!(&get_vals(&column_field, 5), &[1001, 1002]);++ // let column_field = column_fields.bytes(bytes_field).unwrap();+ // assert_eq!(column_field.get_bytes(0), &[] as &[u8]);+ // assert_eq!(column_field.get_bytes(2), &[1, 2, 3]);+ // assert_eq!(column_field.get_bytes(5), &[5, 5]);// // test new field norm mapping// {diff --git a/crates/tantivy/src/indexer/segment_serializer.rs b/crates/tantivy/src/indexer/segment_serializer.rsindex 70747359..2edf1d3f 100644--- a/crates/tantivy/src/indexer/segment_serializer.rs+++ b/crates/tantivy/src/indexer/segment_serializer.rs@@ -11,7 +11,7 @@ use crate::store::StoreWriter;pub struct SegmentSerializer {segment: Segment,pub(crate) store_writer: StoreWriter,- fast_field_write: WritePtr,+ column_field_write: WritePtr,fieldnorms_serializer: Option<FieldNormsSerializer>,postings_serializer: InvertedIndexSerializer,}@@ -48,7 +48,7 @@ impl SegmentSerializer {)?};- let fast_field_write = segment.open_write(SegmentComponent::FastFields)?;+ let column_field_write = segment.open_write(SegmentComponent::ColumnFields)?;let fieldnorms_write = segment.open_write(SegmentComponent::FieldNorms)?;let fieldnorms_serializer = FieldNormsSerializer::from_write(fieldnorms_write)?;@@ -57,7 +57,7 @@ impl SegmentSerializer {Ok(SegmentSerializer {segment,store_writer,- fast_field_write,+ column_field_write,fieldnorms_serializer: Some(fieldnorms_serializer),postings_serializer,})@@ -81,9 +81,9 @@ impl SegmentSerializer {&mut self.postings_serializer}- /// Accessor to the `FastFieldSerializer`.- pub fn get_fast_field_write(&mut self) -> &mut WritePtr {- &mut self.fast_field_write+ /// Accessor to the `ColumnFieldSerializer`.+ pub fn get_column_field_write(&mut self) -> &mut WritePtr {+ &mut self.column_field_write}/// Extract the field norm serializer.@@ -103,7 +103,7 @@ impl SegmentSerializer {if let Some(fieldnorms_serializer) = self.extract_fieldnorms_serializer() {fieldnorms_serializer.close()?;}- self.fast_field_write.terminate()?;+ self.column_field_write.terminate()?;self.postings_serializer.close()?;self.store_writer.close()?;Ok(())diff --git a/crates/tantivy/src/indexer/segment_writer.rs b/crates/tantivy/src/indexer/segment_writer.rsindex 4d716391..ae004771 100644--- a/crates/tantivy/src/indexer/segment_writer.rs+++ b/crates/tantivy/src/indexer/segment_writer.rs@@ -5,7 +5,7 @@ use itertools::Itertools;use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};use super::operation::AddOperation;-use crate::fastfield::FastFieldsWriter;+use crate::columnfield::ColumnFieldsWriter;use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};use crate::index::{Segment, SegmentComponent};use crate::indexer::segment_serializer::SegmentSerializer;@@ -65,7 +65,7 @@ pub struct SegmentWriter {pub(crate) ctx: IndexingContext,pub(crate) per_field_postings_writers: PerFieldPostingsWriter,pub(crate) segment_serializer: SegmentSerializer,- pub(crate) fast_field_writers: FastFieldsWriter,+ pub(crate) column_field_writers: ColumnFieldsWriter,pub(crate) fieldnorms_writer: FieldNormsWriter,pub(crate) json_path_writer: JsonPathWriter,pub(crate) json_positions_per_path: IndexingPositionsPerPath,@@ -88,7 +88,7 @@ impl SegmentWriter {pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result<Self> {let schema = segment.schema();let tokenizer_manager = segment.index().tokenizers().clone();- let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();+ let tokenizer_manager_column_field = segment.index().column_field_tokenizer().clone();let table_size = compute_initial_table_size(memory_budget_in_bytes)?;let segment_serializer = SegmentSerializer::for_segment(segment, false)?;let per_field_postings_writers = PerFieldPostingsWriter::for_schema(&schema);@@ -122,9 +122,9 @@ impl SegmentWriter {json_path_writer: JsonPathWriter::default(),json_positions_per_path: IndexingPositionsPerPath::default(),segment_serializer,- fast_field_writers: FastFieldsWriter::from_schema_and_tokenizer_manager(+ column_field_writers: ColumnFieldsWriter::from_schema_and_tokenizer_manager(&schema,- tokenizer_manager_fast_field,+ tokenizer_manager_column_field,)?,doc_opstamps: Vec::with_capacity(1_000),per_field_text_analyzers,@@ -152,7 +152,7 @@ impl SegmentWriter {self.schema,&self.per_field_postings_writers,self.ctx,- self.fast_field_writers,+ self.column_field_writers,&self.fieldnorms_writer,self.segment_serializer,mapping.as_ref(),@@ -166,7 +166,7 @@ impl SegmentWriter {pub fn mem_usage(&self) -> usize {self.ctx.mem_usage()+ self.fieldnorms_writer.mem_usage()- + self.fast_field_writers.mem_usage()+ + self.column_field_writers.mem_usage()+ self.segment_serializer.mem_usage()}@@ -361,7 +361,7 @@ impl SegmentWriter {) -> crate::Result<()> {let AddOperation { document, opstamp } = add_operation;self.doc_opstamps.push(opstamp);- self.fast_field_writers.add_document(&document)?;+ self.column_field_writers.add_document(&document)?;self.index_document(&document)?;let doc_writer = self.segment_serializer.get_store_writer();doc_writer.store(&document, &self.schema)?;@@ -399,7 +399,7 @@ fn remap_and_write(schema: Schema,per_field_postings_writers: &PerFieldPostingsWriter,ctx: IndexingContext,- fast_field_writers: FastFieldsWriter,+ column_field_writers: ColumnFieldsWriter,fieldnorms_writer: &FieldNormsWriter,mut serializer: SegmentSerializer,doc_id_map: Option<&DocIdMapping>,@@ -420,8 +420,8 @@ fn remap_and_write(doc_id_map,serializer.get_postings_serializer(),)?;- debug!("fastfield-serialize");- fast_field_writers.serialize(serializer.get_fast_field_write(), doc_id_map)?;+ debug!("columnfield-serialize");+ column_field_writers.serialize(serializer.get_column_field_write(), doc_id_map)?;// finalize temp docstore and create version, which reflects the doc_id_mapif let Some(doc_id_map) = doc_id_map {@@ -465,8 +465,8 @@ mod tests {use tempfile::TempDir;use crate::collector::{Count, TopDocs};+ use crate::columnfield::ColumnarValue;use crate::directory::RamDirectory;- use crate::fastfield::FastValue;use crate::postings::{Postings, TermInfo};use crate::query::{PhraseQuery, QueryParser};use crate::schema::{@@ -658,8 +658,8 @@ mod tests {let term_from_path =|path: &str| -> Term { Term::from_field_json_path(json_field, path, false) };- fn set_fast_val<T: FastValue>(val: T, mut term: Term) -> Term {- term.append_type_and_fast_value(val);+ fn set_columnar_val<T: ColumnarValue>(val: T, mut term: Term) -> Term {+ term.append_type_and_columnar_value(val);term}fn set_str(val: &str, mut term: Term) -> Term {@@ -671,14 +671,14 @@ mod tests {assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(true, term).serialized_value_bytes()+ set_columnar_val(true, term).serialized_value_bytes());let term = term_from_path("complexobject.field\\.with\\.dot");assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(1i64, term).serialized_value_bytes()+ set_columnar_val(1i64, term).serialized_value_bytes());// Date@@ -687,7 +687,7 @@ mod tests {assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(+ set_columnar_val(DateTime::from_utc(OffsetDateTime::parse("1985-04-12T23:20:50.52Z", &Rfc3339).unwrap(),),@@ -701,7 +701,7 @@ mod tests {assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(-0.2f64, term).serialized_value_bytes()+ set_columnar_val(-0.2f64, term).serialized_value_bytes());// Number In Array@@ -709,21 +709,21 @@ mod tests {assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(2i64, term).serialized_value_bytes()+ set_columnar_val(2i64, term).serialized_value_bytes());let term = term_from_path("my_arr");assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(3i64, term).serialized_value_bytes()+ set_columnar_val(3i64, term).serialized_value_bytes());let term = term_from_path("my_arr");assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(4i64, term).serialized_value_bytes()+ set_columnar_val(4i64, term).serialized_value_bytes());// El in Array@@ -745,7 +745,7 @@ mod tests {assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(-2i64, term).serialized_value_bytes()+ set_columnar_val(-2i64, term).serialized_value_bytes());let term = term_from_path("toto");@@ -759,7 +759,7 @@ mod tests {assert!(term_stream.advance());assert_eq!(term_stream.key(),- set_fast_val(1i64, term).serialized_value_bytes()+ set_columnar_val(1i64, term).serialized_value_bytes());assert!(!term_stream.advance());diff --git a/crates/tantivy/src/lib.rs b/crates/tantivy/src/lib.rsindex 34672277..b4e6883e 100644--- a/crates/tantivy/src/lib.rs+++ b/crates/tantivy/src/lib.rs@@ -108,8 +108,8 @@ pub mod error;pub mod tokenizer;pub mod collector;+pub mod columnfield;pub mod directory;-pub mod fastfield;pub mod fieldnorm;pub mod index;pub mod positions;@@ -777,11 +777,11 @@ pub mod tests {}#[test]- fn test_wrong_fast_field_type() -> crate::Result<()> {+ fn test_wrong_column_field_type() -> crate::Result<()> {let mut schema_builder = Schema::builder();- let fast_field_unsigned = schema_builder.add_u64_field("unsigned", FAST);- let fast_field_signed = schema_builder.add_i64_field("signed", FAST);- let fast_field_float = schema_builder.add_f64_field("float", FAST);+ let column_field_unsigned = schema_builder.add_u64_field("unsigned", COLUMN);+ let column_field_signed = schema_builder.add_i64_field("signed", COLUMN);+ let column_field_float = schema_builder.add_f64_field("float", COLUMN);schema_builder.add_text_field("text", TEXT);schema_builder.add_u64_field("stored_int", STORED);let schema = schema_builder.build();@@ -789,8 +789,7 @@ pub mod tests {let index = Index::create_in_ram(schema);let mut index_writer: IndexWriter = index.writer_for_tests()?;{- let document =- doc!(fast_field_unsigned => 4u64, fast_field_signed=>4i64, fast_field_float=>4f64);+ let document = doc!(column_field_unsigned => 4u64, column_field_signed=>4i64, column_field_float=>4f64);index_writer.add_document(document)?;index_writer.commit()?;}@@ -798,40 +797,40 @@ pub mod tests {let searcher = reader.searcher();let segment_reader: &SegmentReader = searcher.segment_reader(0);{- let fast_field_reader_res = segment_reader.fast_fields().u64("text");- assert!(fast_field_reader_res.is_err());+ let column_field_reader_res = segment_reader.column_fields().u64("text");+ assert!(column_field_reader_res.is_err());}{- let fast_field_reader_opt = segment_reader.fast_fields().u64("stored_int");- assert!(fast_field_reader_opt.is_err());+ let column_field_reader_opt = segment_reader.column_fields().u64("stored_int");+ assert!(column_field_reader_opt.is_err());}{- let fast_field_reader_opt = segment_reader.fast_fields().u64("signed");- assert!(fast_field_reader_opt.is_err());+ let column_field_reader_opt = segment_reader.column_fields().u64("signed");+ assert!(column_field_reader_opt.is_err());}{- let fast_field_reader_opt = segment_reader.fast_fields().u64("float");- assert!(fast_field_reader_opt.is_err());+ let column_field_reader_opt = segment_reader.column_fields().u64("float");+ assert!(column_field_reader_opt.is_err());}{- let fast_field_reader_opt = segment_reader.fast_fields().u64("unsigned");- assert!(fast_field_reader_opt.is_ok());- let fast_field_reader = fast_field_reader_opt.unwrap();- assert_eq!(fast_field_reader.first(0), Some(4u64))+ let column_field_reader_opt = segment_reader.column_fields().u64("unsigned");+ assert!(column_field_reader_opt.is_ok());+ let column_field_reader = column_field_reader_opt.unwrap();+ assert_eq!(column_field_reader.first(0), Some(4u64))}{- let fast_field_reader_res = segment_reader.fast_fields().i64("signed");- assert!(fast_field_reader_res.is_ok());- let fast_field_reader = fast_field_reader_res.unwrap();- assert_eq!(fast_field_reader.first(0), Some(4i64))+ let column_field_reader_res = segment_reader.column_fields().i64("signed");+ assert!(column_field_reader_res.is_ok());+ let column_field_reader = column_field_reader_res.unwrap();+ assert_eq!(column_field_reader.first(0), Some(4i64))}{- let fast_field_reader_res = segment_reader.fast_fields().f64("float");- assert!(fast_field_reader_res.is_ok());- let fast_field_reader = fast_field_reader_res.unwrap();- assert_eq!(fast_field_reader.first(0), Some(4f64))+ let column_field_reader_res = segment_reader.column_fields().f64("float");+ assert!(column_field_reader_res.is_ok());+ let column_field_reader = column_field_reader_res.unwrap();+ assert_eq!(column_field_reader.first(0), Some(4f64))}Ok(())}diff --git a/crates/tantivy/src/macros.rs b/crates/tantivy/src/macros.rsindex ad32154d..119fdb11 100644--- a/crates/tantivy/src/macros.rs+++ b/crates/tantivy/src/macros.rs@@ -23,7 +23,7 @@/// # Example////// ```rust-/// use tantivy::schema::{Schema, TEXT, FAST};+/// use tantivy::schema::{Schema, TEXT, COLUMN};/// use tantivy::doc;////// //...@@ -32,7 +32,7 @@/// let mut schema_builder = Schema::builder();/// let title = schema_builder.add_text_field("title", TEXT);/// let author = schema_builder.add_text_field("text", TEXT);-/// let likes = schema_builder.add_u64_field("num_u64", FAST);+/// let likes = schema_builder.add_u64_field("num_u64", COLUMN);/// let schema = schema_builder.build();/// let doc = doc!(/// title => "Life Aquatic",@@ -66,14 +66,14 @@ macro_rules! doc(#[cfg(test)]mod test {- use crate::schema::{Schema, FAST, TEXT};+ use crate::schema::{Schema, COLUMN, TEXT};#[test]fn test_doc_basic() {let mut schema_builder = Schema::builder();let title = schema_builder.add_text_field("title", TEXT);let author = schema_builder.add_text_field("text", TEXT);- let likes = schema_builder.add_u64_field("num_u64", FAST);+ let likes = schema_builder.add_u64_field("num_u64", COLUMN);let _schema = schema_builder.build();let _doc = doc!(title => "Life Aquatic",@@ -87,7 +87,7 @@ mod test {let mut schema_builder = Schema::builder();let title = schema_builder.add_text_field("title", TEXT);let author = schema_builder.add_text_field("text", TEXT);- let likes = schema_builder.add_u64_field("num_u64", FAST);+ let likes = schema_builder.add_u64_field("num_u64", COLUMN);let _schema = schema_builder.build();let _doc = doc!(title => "Life Aquatic",diff --git a/crates/tantivy/src/query/mod.rs b/crates/tantivy/src/query/mod.rsindex 5ab71842..4d6957a3 100644--- a/crates/tantivy/src/query/mod.rs+++ b/crates/tantivy/src/query/mod.rs@@ -54,7 +54,7 @@ pub use self::phrase_query::PhraseQuery;pub use self::query::{EnableScoring, Query, QueryClone};#[cfg(test)]pub use self::query_parser::{QueryParser, QueryParserError};-pub use self::range_query::{FastFieldRangeWeight, RangeQuery};+pub use self::range_query::{ColumnFieldRangeWeight, RangeQuery};pub use self::reqopt_scorer::RequiredOptionalScorer;pub use self::score_combiner::{DisjunctionMaxCombiner, ScoreCombiner, SumCombiner, SumWithCoordsCombiner,diff --git a/crates/tantivy/src/query/query_parser/query_parser.rs b/crates/tantivy/src/query/query_parser/query_parser.rsindex f37ab2d8..ba86641d 100644--- a/crates/tantivy/src/query/query_parser/query_parser.rs+++ b/crates/tantivy/src/query/query_parser/query_parser.rs@@ -11,8 +11,8 @@ use rustc_hash::FxHashMap;use super::logical_ast::*;use crate::index::Index;-use crate::json_utils::convert_to_fast_value_and_append_to_json_term;-use crate::query::range_query::{is_type_valid_for_fastfield_range_query, RangeQuery};+use crate::json_utils::convert_to_columnar_value_and_append_to_json_term;+use crate::query::range_query::{is_type_valid_for_columnfield_range_query, RangeQuery};use crate::query::{AllQuery, BooleanQuery, BoostQuery, EmptyQuery, FuzzyTermQuery, Occur, PhrasePrefixQuery,PhraseQuery, Query, TermQuery, TermSetQuery,@@ -424,8 +424,8 @@ impl QueryParser {) -> Result<Term, QueryParserError> {let field_entry = self.schema.get_field_entry(field);let field_type = field_entry.field_type();- let field_supports_ff_range_queries = field_type.is_fast()- && is_type_valid_for_fastfield_range_query(field_type.value_type());+ let field_supports_ff_range_queries = field_type.is_columnar()+ && is_type_valid_for_columnfield_range_query(field_type.value_type());if !field_type.is_indexed() && !field_supports_ff_range_queries {return Err(QueryParserError::FieldNotIndexed(@@ -947,8 +947,9 @@ fn generate_literals_for_json_object(let get_term_with_path =|| Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());- // Try to convert the phrase to a fast value- if let Some(term) = convert_to_fast_value_and_append_to_json_term(get_term_with_path(), phrase)+ // Try to convert the phrase to a columnar value+ if let Some(term) =+ convert_to_columnar_value_and_append_to_json_term(get_term_with_path(), phrase){logical_literals.push(LogicalLiteral::Term(term));}diff --git a/crates/tantivy/src/query/range_query/fast_field_range_query.rs b/crates/tantivy/src/query/range_query/column_field_range_query.rssimilarity index 98%rename from crates/tantivy/src/query/range_query/fast_field_range_query.rsrename to crates/tantivy/src/query/range_query/column_field_range_query.rsindex f49cdbc4..94aa660c 100644--- a/crates/tantivy/src/query/range_query/fast_field_range_query.rs+++ b/crates/tantivy/src/query/range_query/column_field_range_query.rs@@ -186,10 +186,10 @@ mod tests {use crate::{schema, IndexBuilder, TantivyDocument};#[test]- fn range_query_fast_optional_field_minimum() {+ fn range_query_columnar_optional_field_minimum() {let mut schema_builder = schema::SchemaBuilder::new();let id_field = schema_builder.add_text_field("id", schema::STRING);- let score_field = schema_builder.add_u64_field("score", schema::FAST | schema::INDEXED);+ let score_field = schema_builder.add_u64_field("score", schema::COLUMN | schema::INDEXED);let dir = RamDirectory::default();let index = IndexBuilder::new()diff --git a/crates/tantivy/src/query/range_query/mod.rs b/crates/tantivy/src/query/range_query/mod.rsindex 6214c1e5..663fe8b3 100644--- a/crates/tantivy/src/query/range_query/mod.rs+++ b/crates/tantivy/src/query/range_query/mod.rs@@ -2,15 +2,15 @@ use std::ops::Bound;use crate::schema::Type;-mod fast_field_range_query;+mod column_field_range_query;mod range_query;-mod range_query_u64_fastfield;+mod range_query_u64_columnfield;pub use self::range_query::RangeQuery;-pub use self::range_query_u64_fastfield::FastFieldRangeWeight;+pub use self::range_query_u64_columnfield::ColumnFieldRangeWeight;// TODO is this correct?-pub(crate) fn is_type_valid_for_fastfield_range_query(typ: Type) -> bool {+pub(crate) fn is_type_valid_for_columnfield_range_query(typ: Type) -> bool {match typ {Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,Type::IpAddr => true,diff --git a/crates/tantivy/src/query/range_query/range_query.rs b/crates/tantivy/src/query/range_query/range_query.rsindex 949b7f8e..c86fe8d4 100644--- a/crates/tantivy/src/query/range_query/range_query.rs+++ b/crates/tantivy/src/query/range_query/range_query.rs@@ -5,11 +5,11 @@ use std::ops::{Bound, Range};use crate::common::{BinarySerializable, BitSet};use super::map_bound;-use super::range_query_u64_fastfield::FastFieldRangeWeight;+use super::range_query_u64_columnfield::ColumnFieldRangeWeight;use crate::error::TantivyError;use crate::index::SegmentReader;use crate::query::explanation::does_not_match;-use crate::query::range_query::is_type_valid_for_fastfield_range_query;+use crate::query::range_query::is_type_valid_for_columnfield_range_query;use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};use crate::schema::{Field, IndexRecordOption, Term, Type};use crate::termdict::{TermDictionary, TermStreamer};@@ -27,9 +27,9 @@ use crate::{DateTime, DocId, Score};/// `TermInfo` from the inverted index (posting list) and put them into a `BitSet`./// Depending on the number of terms matched, this is a potentially expensive operation.///-/// ## IP fast field-/// For IP fast fields a custom variant is used, by scanning the fast field. Unlike the default-/// variant we can walk in a lazy fashion over it, since the fastfield is implicit orderered by+/// ## IP columnar field+/// For IP columnar fields a custom variant is used, by scanning the columnar field. Unlike the default+/// variant we can walk in a lazy fashion over it, since the columnfield is implicit orderered by/// DocId.//////@@ -317,8 +317,8 @@ impl RangeQuery {}}-/// Returns true if the type maps to a u64 fast field-pub(crate) fn maps_to_u64_fastfield(typ: Type) -> bool {+/// Returns true if the type maps to a u64 columnar field+pub(crate) fn maps_to_u64_columnfield(typ: Type) -> bool {match typ {Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,Type::IpAddr => false,@@ -342,17 +342,17 @@ impl Query for RangeQuery {return Err(TantivyError::SchemaError(err_msg));}- if field_type.is_fast() && is_type_valid_for_fastfield_range_query(self.value_type) {+ if field_type.is_columnar() && is_type_valid_for_columnfield_range_query(self.value_type) {// We run the range query on u64 value space for performance reasons and simpicity// assert the type maps to u64- assert!(maps_to_u64_fastfield(self.value_type));+ assert!(maps_to_u64_columnfield(self.value_type));let parse_from_bytes = |data: &Vec<u8>| {u64::from_be(BinarySerializable::deserialize(&mut &data[..]).unwrap())};let lower_bound = map_bound(&self.lower_bound, parse_from_bytes);let upper_bound = map_bound(&self.upper_bound, parse_from_bytes);- Ok(Box::new(FastFieldRangeWeight::new_u64_lenient(+ Ok(Box::new(ColumnFieldRangeWeight::new_u64_lenient(self.field.to_string(),lower_bound,upper_bound,diff --git a/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs b/crates/tantivy/src/query/range_query/range_query_u64_columnfield.rssimilarity index 90%rename from crates/tantivy/src/query/range_query/range_query_u64_fastfield.rsrename to crates/tantivy/src/query/range_query/range_query_u64_columnfield.rsindex 13c8ca4a..02809aa6 100644--- a/crates/tantivy/src/query/range_query/range_query_u64_fastfield.rs+++ b/crates/tantivy/src/query/range_query/range_query_u64_columnfield.rs@@ -1,27 +1,27 @@//! Fastfields support efficient scanning for range queries.-//! We use this variant only if the fastfield exists, otherwise the default in `range_query` is+//! We use this variant only if the columnfield exists, otherwise the default in `range_query` is//! used, which uses the term dictionary + postings.use std::ops::{Bound, RangeInclusive};use crate::columnar::{ColumnType, HasAssociatedColumnType, MonotonicallyMappableToU64};-use super::fast_field_range_query::RangeDocSet;+use super::column_field_range_query::RangeDocSet;use super::map_bound;use crate::query::{ConstScorer, EmptyScorer, Explanation, Query, Scorer, Weight};use crate::{DocId, DocSet, Score, SegmentReader, TantivyError};-/// `FastFieldRangeWeight` uses the fast field to execute range queries.+/// `ColumnFieldRangeWeight` uses the columnar field to execute range queries.#[derive(Clone, Debug)]-pub struct FastFieldRangeWeight {+pub struct ColumnFieldRangeWeight {field: String,lower_bound: Bound<u64>,upper_bound: Bound<u64>,column_type_opt: Option<ColumnType>,}-impl FastFieldRangeWeight {- /// Create a new FastFieldRangeWeight, using the u64 representation of any fast field.+impl ColumnFieldRangeWeight {+ /// Create a new ColumnFieldRangeWeight, using the u64 representation of any columnar field.pub(crate) fn new_u64_lenient(field: String,lower_bound: Bound<u64>,@@ -37,7 +37,7 @@ impl FastFieldRangeWeight {}}- /// Create a new `FastFieldRangeWeight` for a range of a u64-mappable type .+ /// Create a new `ColumnFieldRangeWeight` for a range of a u64-mappable type .pub fn new<T: HasAssociatedColumnType + MonotonicallyMappableToU64>(field: String,lower_bound: Bound<T>,@@ -54,7 +54,7 @@ impl FastFieldRangeWeight {}}-impl Query for FastFieldRangeWeight {+impl Query for ColumnFieldRangeWeight {fn weight(&self,_enable_scoring: crate::query::EnableScoring<'_>,@@ -63,16 +63,16 @@ impl Query for FastFieldRangeWeight {}}-impl Weight for FastFieldRangeWeight {+impl Weight for ColumnFieldRangeWeight {fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {- let fast_field_reader = reader.fast_fields();+ let column_field_reader = reader.column_fields();let column_type_opt: Option<[ColumnType; 1]> =self.column_type_opt.map(|column_type| [column_type]);let column_type_opt_ref: Option<&[ColumnType]> = column_type_opt.as_ref().map(|column_types| column_types.as_slice());let Some((column, _)) =- fast_field_reader.u64_lenient_for_type(column_type_opt_ref, &self.field)?+ column_field_reader.u64_lenient_for_type(column_type_opt_ref, &self.field)?else {return Ok(Box::new(EmptyScorer));};@@ -138,9 +138,9 @@ pub mod tests {use rand::SeedableRng;use crate::collector::Count;- use crate::query::range_query::range_query_u64_fastfield::FastFieldRangeWeight;+ use crate::query::range_query::range_query_u64_columnfield::ColumnFieldRangeWeight;use crate::query::{QueryParser, Weight};- use crate::schema::{Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING};+ use crate::schema::{Schema, SchemaBuilder, COLUMN, INDEXED, STORED, STRING};use crate::{Index, IndexWriter, TERMINATED};#[derive(Clone, Debug)]@@ -194,14 +194,14 @@ pub mod tests {#[test]fn test_range_regression_simplified() {let mut schema_builder = SchemaBuilder::new();- let field = schema_builder.add_u64_field("test_field", FAST);+ let field = schema_builder.add_u64_field("test_field", COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut writer: IndexWriter = index.writer_for_tests().unwrap();writer.add_document(doc!(field=>52_000u64)).unwrap();writer.commit().unwrap();let searcher = index.reader().unwrap().searcher();- let range_query = FastFieldRangeWeight::new_u64_lenient(+ let range_query = ColumnFieldRangeWeight::new_u64_lenient("test_field".to_string(),Bound::Included(50_000),Bound::Included(50_002),@@ -220,9 +220,9 @@ pub mod tests {pub fn create_index_from_docs(docs: &[Doc]) -> Index {let mut schema_builder = Schema::builder();- let id_u64_field = schema_builder.add_u64_field("id", INDEXED | STORED | FAST);- let id_f64_field = schema_builder.add_f64_field("id_f64", INDEXED | STORED | FAST);- let id_i64_field = schema_builder.add_i64_field("id_i64", INDEXED | STORED | FAST);+ let id_u64_field = schema_builder.add_u64_field("id", INDEXED | STORED | COLUMN);+ let id_f64_field = schema_builder.add_f64_field("id_f64", INDEXED | STORED | COLUMN);+ let id_i64_field = schema_builder.add_i64_field("id_i64", INDEXED | STORED | COLUMN);let text_field = schema_builder.add_text_field("id_name", STRING | STORED);let schema = schema_builder.build();diff --git a/crates/tantivy/src/schema/bytes_options.rs b/crates/tantivy/src/schema/bytes_options.rsindex e5c28ea1..142067be 100644--- a/crates/tantivy/src/schema/bytes_options.rs+++ b/crates/tantivy/src/schema/bytes_options.rs@@ -2,14 +2,14 @@ use std::ops::BitOr;use serde::{Deserialize, Serialize};-use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};+use super::flags::{ColumnarFlag, IndexedFlag, SchemaFlagList, StoredFlag};/// Define how a bytes field should be handled by tantivy.#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]#[serde(from = "BytesOptionsDeser")]pub struct BytesOptions {indexed: bool,fieldnorms: bool,- fast: bool,+ columnar: bool,stored: bool,}@@ -23,7 +23,7 @@ struct BytesOptionsDeser {indexed: bool,#[serde(default)]fieldnorms: Option<bool>,- fast: bool,+ columnar: bool,stored: bool,}@@ -32,7 +32,7 @@ impl From<BytesOptionsDeser> for BytesOptions {BytesOptions {indexed: deser.indexed,fieldnorms: deser.fieldnorms.unwrap_or(deser.indexed),- fast: deser.fast,+ columnar: deser.columnar,stored: deser.stored,}}@@ -51,10 +51,10 @@ impl BytesOptions {self.fieldnorms}- /// Returns true if the value is a fast field.+ /// Returns true if the value is a columnar field.#[inline]- pub fn is_fast(&self) -> bool {- self.fast+ pub fn is_columnar(&self) -> bool {+ self.columnar}/// Returns true if the value is stored.@@ -83,12 +83,12 @@ impl BytesOptions {self}- /// Set the field as a fast field.+ /// Set the field as a columnar field.////// Fast fields are designed for random access.#[must_use]- pub fn set_fast(mut self) -> BytesOptions {- self.fast = true;+ pub fn set_columnar(mut self) -> BytesOptions {+ self.columnar = true;self}@@ -112,7 +112,7 @@ impl<T: Into<BytesOptions>> BitOr<T> for BytesOptions {indexed: self.indexed | other.indexed,fieldnorms: self.fieldnorms | other.fieldnorms,stored: self.stored | other.stored,- fast: self.fast | other.fast,+ columnar: self.columnar | other.columnar,}}}@@ -123,13 +123,13 @@ impl From<()> for BytesOptions {}}-impl From<FastFlag> for BytesOptions {- fn from(_: FastFlag) -> Self {+impl From<ColumnarFlag> for BytesOptions {+ fn from(_: ColumnarFlag) -> Self {BytesOptions {indexed: false,fieldnorms: false,stored: false,- fast: true,+ columnar: true,}}}@@ -140,7 +140,7 @@ impl From<StoredFlag> for BytesOptions {indexed: false,fieldnorms: false,stored: true,- fast: false,+ columnar: false,}}}@@ -151,7 +151,7 @@ impl From<IndexedFlag> for BytesOptions {indexed: true,fieldnorms: true,stored: false,- fast: false,+ columnar: false,}}}@@ -169,11 +169,11 @@ where#[cfg(test)]mod tests {- use crate::schema::{BytesOptions, FAST, INDEXED, STORED};+ use crate::schema::{BytesOptions, COLUMN, INDEXED, STORED};#[test]- fn test_bytes_option_fast_flag() {- assert_eq!(BytesOptions::default().set_fast(), FAST.into());+ fn test_bytes_option_columnar_flag() {+ assert_eq!(BytesOptions::default().set_columnar(), COLUMN.into());assert_eq!(BytesOptions::default().set_indexed().set_fieldnorms(),INDEXED.into()@@ -181,17 +181,17 @@ mod tests {assert_eq!(BytesOptions::default().set_stored(), STORED.into());}#[test]- fn test_bytes_option_fast_flag_composition() {+ fn test_bytes_option_columnar_flag_composition() {assert_eq!(- BytesOptions::default().set_fast().set_stored(),- (FAST | STORED).into()+ BytesOptions::default().set_columnar().set_stored(),+ (COLUMN | STORED).into());assert_eq!(BytesOptions::default().set_indexed().set_fieldnorms()- .set_fast(),- (INDEXED | FAST).into()+ .set_columnar(),+ (INDEXED | COLUMN).into());assert_eq!(BytesOptions::default()@@ -203,13 +203,13 @@ mod tests {}#[test]- fn test_bytes_option_fast_() {+ fn test_bytes_option_columnar_() {assert!(!BytesOptions::default().is_stored());- assert!(!BytesOptions::default().is_fast());+ assert!(!BytesOptions::default().is_columnar());assert!(!BytesOptions::default().is_indexed());assert!(!BytesOptions::default().fieldnorms());assert!(BytesOptions::default().set_stored().is_stored());- assert!(BytesOptions::default().set_fast().is_fast());+ assert!(BytesOptions::default().set_columnar().is_columnar());assert!(BytesOptions::default().set_indexed().is_indexed());assert!(BytesOptions::default().set_fieldnorms().fieldnorms());}@@ -218,7 +218,7 @@ mod tests {fn test_bytes_options_deser_if_fieldnorm_missing_indexed_true() {let json = r#"{"indexed": true,- "fast": false,+ "columnar": false,"stored": false}"#;let bytes_options: BytesOptions = serde_json::from_str(json).unwrap();@@ -227,7 +227,7 @@ mod tests {&BytesOptions {indexed: true,fieldnorms: true,- fast: false,+ columnar: false,stored: false});@@ -238,7 +238,7 @@ mod tests {let json = r#"{"indexed": false,"stored": false,- "fast": false+ "columnar": false}"#;let bytes_options: BytesOptions = serde_json::from_str(json).unwrap();assert_eq!(@@ -246,7 +246,7 @@ mod tests {&BytesOptions {indexed: false,fieldnorms: false,- fast: false,+ columnar: false,stored: false});@@ -257,7 +257,7 @@ mod tests {let json = r#"{"indexed": true,"fieldnorms": false,- "fast": false,+ "columnar": false,"stored": false}"#;let bytes_options: BytesOptions = serde_json::from_str(json).unwrap();@@ -266,7 +266,7 @@ mod tests {&BytesOptions {indexed: true,fieldnorms: false,- fast: false,+ columnar: false,stored: false});@@ -278,7 +278,7 @@ mod tests {let json = r#"{"indexed": false,"fieldnorms": true,- "fast": false,+ "columnar": false,"stored": false}"#;let bytes_options: BytesOptions = serde_json::from_str(json).unwrap();@@ -287,7 +287,7 @@ mod tests {&BytesOptions {indexed: false,fieldnorms: true,- fast: false,+ columnar: false,stored: false});diff --git a/crates/tantivy/src/schema/date_time_options.rs b/crates/tantivy/src/schema/date_time_options.rsindex 355cfbb7..86639376 100644--- a/crates/tantivy/src/schema/date_time_options.rs+++ b/crates/tantivy/src/schema/date_time_options.rs@@ -3,7 +3,7 @@ use std::ops::BitOr;pub use crate::common::DateTimePrecision;use serde::{Deserialize, Serialize};-use crate::schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};+use crate::schema::flags::{ColumnarFlag, IndexedFlag, SchemaFlagList, StoredFlag};/// The precision of the indexed date/time values in the inverted index.pub const DATE_TIME_PRECISION_INDEXED: DateTimePrecision = DateTimePrecision::Seconds;@@ -15,10 +15,10 @@ pub struct DateOptions {// This boolean has no effect if the field is not marked as indexed true.fieldnorms: bool,#[serde(default)]- fast: bool,+ columnar: bool,stored: bool,// Internal storage precision, used to optimize storage- // compression on fast fields.+ // compression on columnar fields.#[serde(default)]precision: DateTimePrecision,}@@ -42,10 +42,10 @@ impl DateOptions {self.fieldnorms && self.indexed}- /// Returns true iff the value is a fast field.+ /// Returns true iff the value is a columnar field.#[inline]- pub fn is_fast(&self) -> bool {- self.fast+ pub fn is_columnar(&self) -> bool {+ self.columnar}/// Set the field as stored.@@ -80,20 +80,20 @@ impl DateOptions {self}- /// Set the field as a fast field.+ /// Set the field as a columnar field.///- /// Fast fields are designed for random access.+ /// Columnar fields are designed for random access.#[must_use]- pub fn set_fast(mut self) -> DateOptions {- self.fast = true;+ pub fn set_columnar(mut self) -> DateOptions {+ self.columnar = true;self}- /// Sets the precision for this DateTime field on the fast field.+ /// Sets the precision for this DateTime field on the columnar field./// Indexed precision is always [`DATE_TIME_PRECISION_INDEXED`].////// Internal storage precision, used to optimize storage- /// compression on fast fields.+ /// compression on columnar fields.pub fn set_precision(mut self, precision: DateTimePrecision) -> DateOptions {self.precision = precision;self@@ -102,7 +102,7 @@ impl DateOptions {/// Returns the storage precision for this DateTime field.////// Internal storage precision, used to optimize storage- /// compression on fast fields.+ /// compression on columnar fields.pub fn get_precision(&self) -> DateTimePrecision {self.precision}@@ -114,10 +114,10 @@ impl From<()> for DateOptions {}}-impl From<FastFlag> for DateOptions {- fn from(_: FastFlag) -> Self {+impl From<ColumnarFlag> for DateOptions {+ fn from(_: ColumnarFlag) -> Self {DateOptions {- fast: true,+ columnar: true,..Default::default()}}@@ -151,7 +151,7 @@ impl<T: Into<DateOptions>> BitOr<T> for DateOptions {indexed: self.indexed | other.indexed,fieldnorms: self.fieldnorms | other.fieldnorms,stored: self.stored | other.stored,- fast: self.fast | other.fast,+ columnar: self.columnar | other.columnar,precision: self.precision,}}@@ -204,7 +204,7 @@ mod tests {serde_json::json!({"precision": "milliseconds","indexed": true,- "fast": false,+ "columnar": false,"fieldnorms": false,"stored": false})diff --git a/crates/tantivy/src/schema/document/se.rs b/crates/tantivy/src/schema/document/se.rsindex a0bb2661..512bf880 100644--- a/crates/tantivy/src/schema/document/se.rs+++ b/crates/tantivy/src/schema/document/se.rs@@ -307,7 +307,7 @@ mod tests {use super::*;use crate::schema::document::existing_type_impls::JsonObjectIter;- use crate::schema::{Field, FAST, STORED, TEXT};+ use crate::schema::{Field, COLUMN, STORED, TEXT};use crate::tokenizer::PreTokenizedString;fn serialize_value<'a>(value: ReferenceValue<'a, &'a serde_json::Value>) -> Vec<u8> {@@ -698,7 +698,7 @@ mod tests {fn test_document_serialize() {let mut builder = Schema::builder();let name = builder.add_text_field("name", TEXT | STORED);- let age = builder.add_u64_field("age", FAST | STORED);+ let age = builder.add_u64_field("age", COLUMN | STORED);let schema = builder.build();let mut document = BTreeMap::new();@@ -720,7 +720,7 @@ mod tests {let mut builder = Schema::builder();let name = builder.add_text_field("name", TEXT | STORED);// This should be skipped when serializing.- let age = builder.add_u64_field("age", FAST);+ let age = builder.add_u64_field("age", COLUMN);let schema = builder.build();let mut document = BTreeMap::new();diff --git a/crates/tantivy/src/schema/field_entry.rs b/crates/tantivy/src/schema/field_entry.rsindex 67cc894d..721a8936 100644--- a/crates/tantivy/src/schema/field_entry.rs+++ b/crates/tantivy/src/schema/field_entry.rs@@ -97,9 +97,9 @@ impl FieldEntry {self.field_type.has_fieldnorms()}- /// Returns true if the field is a fast field- pub fn is_fast(&self) -> bool {- self.field_type.is_fast()+ /// Returns true if the field is a columnar field+ pub fn is_columnar(&self) -> bool {+ self.field_type.is_columnar()}/// Returns true if the field has the expand dots option set (for json fields)@@ -154,7 +154,7 @@ mod tests {"tokenizer": "default"},"stored": false,- "fast": false+ "columnar": false}}"#;let field_value_json = serde_json::to_string_pretty(&field_value).unwrap();diff --git a/crates/tantivy/src/schema/field_type.rs b/crates/tantivy/src/schema/field_type.rsindex 472ab4aa..e496b982 100644--- a/crates/tantivy/src/schema/field_type.rs+++ b/crates/tantivy/src/schema/field_type.rs@@ -231,18 +231,18 @@ impl FieldType {}}- /// returns true if the field is fast.- pub fn is_fast(&self) -> bool {+ /// returns true if the field is columnar,+ pub fn is_columnar(&self) -> bool {match *self {- FieldType::Bytes(ref bytes_options) => bytes_options.is_fast(),- FieldType::Str(ref text_options) => text_options.is_fast(),+ FieldType::Bytes(ref bytes_options) => bytes_options.is_columnar(),+ FieldType::Str(ref text_options) => text_options.is_columnar(),FieldType::U64(ref int_options)| FieldType::I64(ref int_options)| FieldType::F64(ref int_options)- | FieldType::Bool(ref int_options) => int_options.is_fast(),- FieldType::Date(ref date_options) => date_options.is_fast(),- FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(),- FieldType::JsonObject(ref json_object_options) => json_object_options.is_fast(),+ | FieldType::Bool(ref int_options) => int_options.is_columnar(),+ FieldType::Date(ref date_options) => date_options.is_columnar(),+ FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_columnar(),+ FieldType::JsonObject(ref json_object_options) => json_object_options.is_columnar(),}}diff --git a/crates/tantivy/src/schema/flags.rs b/crates/tantivy/src/schema/flags.rsindex 4f8caa61..92ab5cb8 100644--- a/crates/tantivy/src/schema/flags.rs+++ b/crates/tantivy/src/schema/flags.rs@@ -36,22 +36,22 @@ pub struct CoerceFlag;////// `COERCE` will try to convert values into its value type if they don't match.///-/// See [fast fields](`crate::fastfield`).+/// See [columnar fields](`crate::columnfield`).pub const COERCE: SchemaFlagList<CoerceFlag, ()> = SchemaFlagList {head: CoerceFlag,tail: (),};#[derive(Clone)]-pub struct FastFlag;-/// Flag to mark the field as a fast field (similar to Lucene's DocValues)+pub struct ColumnarFlag;+/// Flag to mark the field as a columnar field (similar to Lucene's DocValues)////// Fast fields can be random-accessed rapidly. Fields useful for scoring, filtering-/// or collection should be mark as fast fields.+/// or collection should be mark as columnar fields.///-/// See [fast fields](`crate::fastfield`).-pub const FAST: SchemaFlagList<FastFlag, ()> = SchemaFlagList {- head: FastFlag,+/// See [columnar fields](`crate::columnfield`).+pub const COLUMN: SchemaFlagList<ColumnarFlag, ()> = SchemaFlagList {+ head: ColumnarFlag,tail: (),};diff --git a/crates/tantivy/src/schema/ip_options.rs b/crates/tantivy/src/schema/ip_options.rsindex c8883059..faad1b65 100644--- a/crates/tantivy/src/schema/ip_options.rs+++ b/crates/tantivy/src/schema/ip_options.rs@@ -3,7 +3,7 @@ use std::ops::BitOr;use serde::{Deserialize, Serialize};-use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};+use super::flags::{ColumnarFlag, IndexedFlag, SchemaFlagList, StoredFlag};/// Trait to convert into an Ipv6Addr.pub trait IntoIpv6Addr {@@ -23,17 +23,17 @@ impl IntoIpv6Addr for IpAddr {/// Define how an ip field should be handled by tantivy.#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]pub struct IpAddrOptions {- fast: bool,+ columnar: bool,stored: bool,indexed: bool,fieldnorms: bool,}impl IpAddrOptions {- /// Returns true iff the value is a fast field.+ /// Returns true iff the value is a columnar field.#[inline]- pub fn is_fast(&self) -> bool {- self.fast+ pub fn is_columnar(&self) -> bool {+ self.columnar}/// Returns `true` if the ip address should be stored in the doc store.@@ -84,12 +84,12 @@ impl IpAddrOptions {self}- /// Set the field as a fast field.+ /// Set the field as a columnar field.///- /// Fast fields are designed for random access.+ /// Columnar fields are designed for random access.#[must_use]- pub fn set_fast(mut self) -> Self {- self.fast = true;+ pub fn set_columnar(mut self) -> Self {+ self.columnar = true;self}}@@ -100,13 +100,13 @@ impl From<()> for IpAddrOptions {}}-impl From<FastFlag> for IpAddrOptions {- fn from(_: FastFlag) -> Self {+impl From<ColumnarFlag> for IpAddrOptions {+ fn from(_: ColumnarFlag) -> Self {IpAddrOptions {fieldnorms: false,indexed: false,stored: false,- fast: true,+ columnar: true,}}}@@ -117,7 +117,7 @@ impl From<StoredFlag> for IpAddrOptions {fieldnorms: false,indexed: false,stored: true,- fast: false,+ columnar: false,}}}@@ -128,7 +128,7 @@ impl From<IndexedFlag> for IpAddrOptions {fieldnorms: true,indexed: true,stored: false,- fast: false,+ columnar: false,}}}@@ -142,7 +142,7 @@ impl<T: Into<IpAddrOptions>> BitOr<T> for IpAddrOptions {fieldnorms: self.fieldnorms | other.fieldnorms,indexed: self.indexed | other.indexed,stored: self.stored | other.stored,- fast: self.fast | other.fast,+ columnar: self.columnar | other.columnar,}}}diff --git a/crates/tantivy/src/schema/json_object_options.rs b/crates/tantivy/src/schema/json_object_options.rsindex 18b48ac9..63995ffe 100644--- a/crates/tantivy/src/schema/json_object_options.rs+++ b/crates/tantivy/src/schema/json_object_options.rs@@ -2,8 +2,8 @@ use std::ops::BitOr;use serde::{Deserialize, Serialize};-use super::text_options::{FastFieldTextOptions, TokenizerName};-use crate::schema::flags::{FastFlag, SchemaFlagList, StoredFlag};+use super::text_options::{ColumnFieldTextOptions, TokenizerName};+use crate::schema::flags::{ColumnarFlag, SchemaFlagList, StoredFlag};use crate::schema::{TextFieldIndexing, TextOptions};/// The `JsonObjectOptions` make it possible to@@ -14,12 +14,12 @@ pub struct JsonObjectOptions {// If set to some, int, date, f64 and text will be indexed.// Text will use the TextFieldIndexing setting for indexing.indexing: Option<TextFieldIndexing>,- // Store all field as fast fields with an optional tokenizer for text.- fast: FastFieldTextOptions,+ // Store all field as columnar fields with an optional tokenizer for text.+ columnar: ColumnFieldTextOptions,/// tantivy will generate pathes to the different nodes of the json object/// both in:/// - the inverted index (for the terms)- /// - fast fields (for the column names).+ /// - columnar fields (for the column names).////// These json path are encoded by concatenating the list of object keys that/// are visited from the root to the leaf.@@ -58,22 +58,24 @@ impl JsonObjectOptions {}/// Returns true if and only if the json object fields are- /// to be treated as fast fields.+ /// to be treated as columnar fields.#[inline]- pub fn is_fast(&self) -> bool {- matches!(self.fast, FastFieldTextOptions::IsEnabled(true))+ pub fn is_columnar(&self) -> bool {+ matches!(self.columnar, ColumnFieldTextOptions::IsEnabled(true))|| matches!(- &self.fast,- FastFieldTextOptions::EnabledWithTokenizer { with_tokenizer: _ }+ &self.columnar,+ ColumnFieldTextOptions::EnabledWithTokenizer { with_tokenizer: _ })}- /// Returns true if and only if the value is a fast field.+ /// Returns true if and only if the value is a columnar field.#[inline]- pub fn get_fast_field_tokenizer_name(&self) -> Option<&str> {- match &self.fast {- FastFieldTextOptions::IsEnabled(true) | FastFieldTextOptions::IsEnabled(false) => None,- FastFieldTextOptions::EnabledWithTokenizer {+ pub fn get_column_field_tokenizer_name(&self) -> Option<&str> {+ match &self.columnar {+ ColumnFieldTextOptions::IsEnabled(true) | ColumnFieldTextOptions::IsEnabled(false) => {+ None+ }+ ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,} => Some(tokenizer.name()),}@@ -120,30 +122,30 @@ impl JsonObjectOptions {self}- /// Set the field as a fast field.+ /// Set the field as a columnar field.///- /// Fast fields are designed for random access.+ /// Columnar fields are designed for random access./// Access time are similar to a random lookup in an array.- /// Text fast fields will have the term ids stored in the fast field.+ /// Text columnar fields will have the term ids stored in the columnar field.////// The effective cardinality depends on the tokenizer. Without a tokenizer, the text will be/// stored as is, which equals to the "raw" tokenizer. The tokenizer can be used to apply/// normalization like lower case.- /// The passed tokenizer_name must be available on the fast field tokenizer manager.- /// `Index::fast_field_tokenizer`.+ /// The passed tokenizer_name must be available on the columnar field tokenizer manager.+ /// `Index::column_field_tokenizer`.////// The original text can be retrieved via/// [`TermDictionary::ord_to_term()`](crate::termdict::TermDictionary::ord_to_term)/// from the dictionary.#[must_use]- pub fn set_fast(mut self, tokenizer_name: Option<&str>) -> Self {+ pub fn set_columnar(mut self, tokenizer_name: Option<&str>) -> Self {if let Some(tokenizer) = tokenizer_name {let tokenizer = TokenizerName::from_name(tokenizer);- self.fast = FastFieldTextOptions::EnabledWithTokenizer {+ self.columnar = ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,}} else {- self.fast = FastFieldTextOptions::IsEnabled(true);+ self.columnar = ColumnFieldTextOptions::IsEnabled(true);}self}@@ -161,18 +163,18 @@ impl From<StoredFlag> for JsonObjectOptions {JsonObjectOptions {stored: true,indexing: None,- fast: FastFieldTextOptions::default(),+ columnar: ColumnFieldTextOptions::default(),expand_dots_enabled: false,}}}-impl From<FastFlag> for JsonObjectOptions {- fn from(_fast_flag: FastFlag) -> Self {+impl From<ColumnarFlag> for JsonObjectOptions {+ fn from(_columnar_flag: ColumnarFlag) -> Self {JsonObjectOptions {stored: false,indexing: None,- fast: FastFieldTextOptions::IsEnabled(true),+ columnar: ColumnFieldTextOptions::IsEnabled(true),expand_dots_enabled: false,}}@@ -192,7 +194,7 @@ impl<T: Into<JsonObjectOptions>> BitOr<T> for JsonObjectOptions {JsonObjectOptions {indexing: self.indexing.or(other.indexing),stored: self.stored | other.stored,- fast: self.fast | other.fast,+ columnar: self.columnar | other.columnar,expand_dots_enabled: self.expand_dots_enabled | other.expand_dots_enabled,}}@@ -214,7 +216,7 @@ impl From<TextOptions> for JsonObjectOptions {JsonObjectOptions {stored: text_options.is_stored(),indexing: text_options.get_indexing_options().cloned(),- fast: text_options.fast,+ columnar: text_options.columnar,expand_dots_enabled: false,}}@@ -223,7 +225,7 @@ impl From<TextOptions> for JsonObjectOptions {#[cfg(test)]mod tests {use super::*;- use crate::schema::{FAST, STORED, TEXT};+ use crate::schema::{COLUMN, STORED, TEXT};#[test]fn test_json_options() {@@ -231,31 +233,31 @@ mod tests {let json_options: JsonObjectOptions = (STORED | TEXT).into();assert!(json_options.is_stored());assert!(json_options.is_indexed());- assert!(!json_options.is_fast());+ assert!(!json_options.is_columnar());}{let json_options: JsonObjectOptions = TEXT.into();assert!(!json_options.is_stored());assert!(json_options.is_indexed());- assert!(!json_options.is_fast());+ assert!(!json_options.is_columnar());}{let json_options: JsonObjectOptions = STORED.into();assert!(json_options.is_stored());assert!(!json_options.is_indexed());- assert!(!json_options.is_fast());+ assert!(!json_options.is_columnar());}{- let json_options: JsonObjectOptions = FAST.into();+ let json_options: JsonObjectOptions = COLUMN.into();assert!(!json_options.is_stored());assert!(!json_options.is_indexed());- assert!(json_options.is_fast());+ assert!(json_options.is_columnar());}{- let json_options: JsonObjectOptions = (FAST | STORED).into();+ let json_options: JsonObjectOptions = (COLUMN | STORED).into();assert!(json_options.is_stored());assert!(!json_options.is_indexed());- assert!(json_options.is_fast());+ assert!(json_options.is_columnar());}}}diff --git a/crates/tantivy/src/schema/mod.rs b/crates/tantivy/src/schema/mod.rsindex 2c6e98d8..41ce8e1e 100644--- a/crates/tantivy/src/schema/mod.rs+++ b/crates/tantivy/src/schema/mod.rs@@ -94,15 +94,15 @@//! ### Fast fields//! This functionality is somewhat similar to Lucene's `DocValues`.//!-//! Fields that are indexed as [`FAST`] will be stored in a special data structure that will+//! Fields that are indexed as [`COLUMN`] will be stored in a special data structure that will//! make it possible to access the value given the doc id rapidly. This is useful if the value//! of the field is required during scoring or collection for instance.//!//! ```//! use tantivy::schema::*;//! let mut schema_builder = Schema::builder();-//! schema_builder.add_u64_field("population", STORED | FAST);-//! schema_builder.add_text_field("zip_code", STRING | FAST);+//! schema_builder.add_u64_field("population", STORED | COLUMN);+//! schema_builder.add_text_field("zip_code", STRING | COLUMN);//! let schema = schema_builder.build();//! ```@@ -132,7 +132,7 @@ pub use self::document::{DocParsingError, Document, OwnedValue, TantivyDocument,pub use self::field::Field;pub use self::field_entry::FieldEntry;pub use self::field_type::{FieldType, Type};-pub use self::flags::{COERCE, FAST, INDEXED, STORED};+pub use self::flags::{COERCE, COLUMN, INDEXED, STORED};pub use self::index_record_option::IndexRecordOption;pub use self::ip_options::{IntoIpv6Addr, IpAddrOptions};pub use self::json_object_options::JsonObjectOptions;diff --git a/crates/tantivy/src/schema/numeric_options.rs b/crates/tantivy/src/schema/numeric_options.rsindex db36b523..d825c4c7 100644--- a/crates/tantivy/src/schema/numeric_options.rs+++ b/crates/tantivy/src/schema/numeric_options.rs@@ -3,7 +3,7 @@ use std::ops::BitOr;use serde::{Deserialize, Serialize};use super::flags::CoerceFlag;-use crate::schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};+use crate::schema::flags::{ColumnarFlag, IndexedFlag, SchemaFlagList, StoredFlag};/// Define how an `u64`, `i64`, or `f64` field should be handled by tantivy.#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]@@ -12,7 +12,7 @@ pub struct NumericOptions {indexed: bool,// This boolean has no effect if the field is not marked as indexed too.fieldnorms: bool, // This attribute only has an effect if indexed is true.- fast: bool,+ columnar: bool,stored: bool,#[serde(skip_serializing_if = "is_false")]coerce: bool,@@ -33,7 +33,7 @@ struct NumericOptionsDeser {#[serde(default)]fieldnorms: Option<bool>, // This attribute only has an effect if indexed is true.#[serde(default)]- fast: bool,+ columnar: bool,stored: bool,#[serde(default)]coerce: bool,@@ -44,7 +44,7 @@ impl From<NumericOptionsDeser> for NumericOptions {NumericOptions {indexed: deser.indexed,fieldnorms: deser.fieldnorms.unwrap_or(deser.indexed),- fast: deser.fast,+ columnar: deser.columnar,stored: deser.stored,coerce: deser.coerce,}@@ -70,10 +70,10 @@ impl NumericOptions {self.fieldnorms && self.indexed}- /// Returns true iff the value is a fast field.+ /// Returns true iff the value is a columnar field.#[inline]- pub fn is_fast(&self) -> bool {- self.fast+ pub fn is_columnar(&self) -> bool {+ self.columnar}/// Returns true if values should be coerced to numbers.@@ -121,12 +121,12 @@ impl NumericOptions {self}- /// Set the field as a fast field.+ /// Set the field as a columnar field.///- /// Fast fields are designed for random access.+ /// Columnar fields are designed for random access of a single column value for each document.#[must_use]- pub fn set_fast(mut self) -> NumericOptions {- self.fast = true;+ pub fn set_columnar(mut self) -> NumericOptions {+ self.columnar = true;self}}@@ -143,19 +143,19 @@ impl From<CoerceFlag> for NumericOptions {indexed: false,fieldnorms: false,stored: false,- fast: false,+ columnar: false,coerce: true,}}}-impl From<FastFlag> for NumericOptions {- fn from(_: FastFlag) -> Self {+impl From<ColumnarFlag> for NumericOptions {+ fn from(_: ColumnarFlag) -> Self {NumericOptions {indexed: false,fieldnorms: false,stored: false,- fast: true,+ columnar: true,coerce: false,}}@@ -167,7 +167,7 @@ impl From<StoredFlag> for NumericOptions {indexed: false,fieldnorms: false,stored: true,- fast: false,+ columnar: false,coerce: false,}}@@ -179,7 +179,7 @@ impl From<IndexedFlag> for NumericOptions {indexed: true,fieldnorms: true,stored: false,- fast: false,+ columnar: false,coerce: false,}}@@ -194,7 +194,7 @@ impl<T: Into<NumericOptions>> BitOr<T> for NumericOptions {indexed: self.indexed | other.indexed,fieldnorms: self.fieldnorms | other.fieldnorms,stored: self.stored | other.stored,- fast: self.fast | other.fast,+ columnar: self.columnar | other.columnar,coerce: self.coerce | other.coerce,}}@@ -227,7 +227,7 @@ mod tests {&NumericOptions {indexed: true,fieldnorms: true,- fast: false,+ columnar: false,stored: false,coerce: false,}@@ -246,7 +246,7 @@ mod tests {&NumericOptions {indexed: false,fieldnorms: false,- fast: false,+ columnar: false,stored: false,coerce: false,}@@ -266,7 +266,7 @@ mod tests {&NumericOptions {indexed: true,fieldnorms: false,- fast: false,+ columnar: false,stored: false,coerce: false,}@@ -287,7 +287,7 @@ mod tests {&NumericOptions {indexed: false,fieldnorms: true,- fast: false,+ columnar: false,stored: false,coerce: false,}@@ -309,7 +309,7 @@ mod tests {&NumericOptions {indexed: false,fieldnorms: true,- fast: false,+ columnar: false,stored: false,coerce: true,}diff --git a/crates/tantivy/src/schema/schema.rs b/crates/tantivy/src/schema/schema.rsindex 1c377dc8..035841c6 100644--- a/crates/tantivy/src/schema/schema.rs+++ b/crates/tantivy/src/schema/schema.rs@@ -155,10 +155,10 @@ impl SchemaBuilder {self.add_field(field_entry)}- /// Adds a fast bytes field to the schema.+ /// Adds a columnar field to the schema.////// Bytes field are not searchable and are only used- /// as fast field, to associate any kind of payload+ /// as columnar field, to associate any kind of payload/// to a document.////// For instance, learning-to-rank often requires to access@@ -334,7 +334,7 @@ impl Schema {None}- /// Transforms a user-supplied fast field name into a column name.+ /// Transforms a user-supplied columnar field name into a column name.////// This is similar to `.find_field` except it includes some fallback logic to/// a default json field. This functionality is used in Quickwit.@@ -438,13 +438,13 @@ mod tests {#[test]pub fn test_schema_serialization() {let mut schema_builder = Schema::builder();- let count_options = NumericOptions::default().set_stored().set_fast();- let popularity_options = NumericOptions::default().set_stored().set_fast();+ let count_options = NumericOptions::default().set_stored().set_columnar();+ let popularity_options = NumericOptions::default().set_stored().set_columnar();let score_options = NumericOptions::default().set_indexed().set_fieldnorm()- .set_fast();- let is_read_options = NumericOptions::default().set_stored().set_fast();+ .set_columnar();+ let is_read_options = NumericOptions::default().set_stored().set_columnar();schema_builder.add_text_field("title", TEXT);schema_builder.add_text_field("author",@@ -471,7 +471,7 @@ mod tests {"tokenizer": "default"},"stored": false,- "fast": false+ "columnar": false}},{@@ -484,7 +484,7 @@ mod tests {"tokenizer": "raw"},"stored": false,- "fast": false+ "columnar": false}},{@@ -493,7 +493,7 @@ mod tests {"options": {"indexed": false,"fieldnorms": false,- "fast": true,+ "columnar": true,"stored": true}},@@ -503,7 +503,7 @@ mod tests {"options": {"indexed": false,"fieldnorms": false,- "fast": true,+ "columnar": true,"stored": true}},@@ -513,7 +513,7 @@ mod tests {"options": {"indexed": true,"fieldnorms": true,- "fast": true,+ "columnar": true,"stored": false}},@@ -523,7 +523,7 @@ mod tests {"options": {"indexed": false,"fieldnorms": false,- "fast": true,+ "columnar": true,"stored": true}}@@ -569,12 +569,12 @@ mod tests {#[test]pub fn test_document_to_json() {let mut schema_builder = Schema::builder();- let count_options = NumericOptions::default().set_stored().set_fast();- let is_read_options = NumericOptions::default().set_stored().set_fast();+ let count_options = NumericOptions::default().set_stored().set_columnar();+ let is_read_options = NumericOptions::default().set_stored().set_columnar();schema_builder.add_text_field("title", TEXT);schema_builder.add_text_field("author", STRING);schema_builder.add_u64_field("count", count_options);- schema_builder.add_ip_addr_field("ip", FAST | STORED);+ schema_builder.add_ip_addr_field("ip", COLUMN | STORED);schema_builder.add_bool_field("is_read", is_read_options);let schema = schema_builder.build();let doc_json = r#"{@@ -593,7 +593,7 @@ mod tests {#[test]pub fn test_document_to_ipv4_json() {let mut schema_builder = Schema::builder();- schema_builder.add_ip_addr_field("ip", FAST | STORED);+ schema_builder.add_ip_addr_field("ip", COLUMN | STORED);let schema = schema_builder.build();// IpV4 loopback@@ -667,9 +667,9 @@ mod tests {#[test]pub fn test_parse_document() {let mut schema_builder = Schema::builder();- let count_options = NumericOptions::default().set_stored().set_fast();- let popularity_options = NumericOptions::default().set_stored().set_fast();- let score_options = NumericOptions::default().set_indexed().set_fast();+ let count_options = NumericOptions::default().set_stored().set_columnar();+ let popularity_options = NumericOptions::default().set_stored().set_columnar();+ let score_options = NumericOptions::default().set_indexed().set_columnar();let title_field = schema_builder.add_text_field("title", TEXT);let author_field = schema_builder.add_text_field("author", STRING);let count_field = schema_builder.add_u64_field("count", count_options);@@ -826,7 +826,7 @@ mod tests {.set_stored().set_indexed().set_fieldnorm()- .set_fast();+ .set_columnar();schema_builder.add_text_field("_id", id_options);schema_builder.add_date_field("_timestamp", timestamp_options);@@ -841,7 +841,7 @@ mod tests {"tokenizer": "default"},"stored": false,- "fast": false+ "columnar": false}},{@@ -850,7 +850,7 @@ mod tests {"options": {"indexed": false,"fieldnorms": false,- "fast": true,+ "columnar": true,"stored": true}}@@ -874,7 +874,7 @@ mod tests {"tokenizer": "raw"},"stored": true,- "fast": false+ "columnar": false}},{@@ -883,7 +883,7 @@ mod tests {"options": {"indexed": true,"fieldnorms": true,- "fast": true,+ "columnar": true,"stored": true,"precision": "seconds"}@@ -898,7 +898,7 @@ mod tests {"tokenizer": "default"},"stored": false,- "fast": false+ "columnar": false}},{@@ -907,7 +907,7 @@ mod tests {"options": {"indexed": false,"fieldnorms": false,- "fast": true,+ "columnar": true,"stored": true}}diff --git a/crates/tantivy/src/schema/term.rs b/crates/tantivy/src/schema/term.rsindex 21c1b1e4..07a89600 100644--- a/crates/tantivy/src/schema/term.rs+++ b/crates/tantivy/src/schema/term.rs@@ -8,7 +8,7 @@ use crate::common::JsonPathWriter;use super::date_time_options::DATE_TIME_PRECISION_INDEXED;use super::Field;-use crate::fastfield::FastValue;+use crate::columnfield::ColumnarValue;use crate::json_utils::split_json_path;use crate::schema::Type;use crate::DateTime;@@ -71,7 +71,7 @@ impl Term {term}- fn from_fast_value<T: FastValue>(field: Field, val: &T) -> Term {+ fn from_columnar_value<T: ColumnarValue>(field: Field, val: &T) -> Term {let mut term = Self::with_type_and_field(T::to_type(), field);term.set_u64(val.to_u64());term@@ -101,27 +101,27 @@ impl Term {/// Builds a term given a field, and a `u64`-valuepub fn from_field_u64(field: Field, val: u64) -> Term {- Term::from_fast_value(field, &val)+ Term::from_columnar_value(field, &val)}/// Builds a term given a field, and a `i64`-valuepub fn from_field_i64(field: Field, val: i64) -> Term {- Term::from_fast_value(field, &val)+ Term::from_columnar_value(field, &val)}/// Builds a term given a field, and a `f64`-valuepub fn from_field_f64(field: Field, val: f64) -> Term {- Term::from_fast_value(field, &val)+ Term::from_columnar_value(field, &val)}/// Builds a term given a field, and a `bool`-valuepub fn from_field_bool(field: Field, val: bool) -> Term {- Term::from_fast_value(field, &val)+ Term::from_columnar_value(field, &val)}/// Builds a term given a field, and a `DateTime` valuepub fn from_field_date(field: Field, val: DateTime) -> Term {- Term::from_fast_value(field, &val.truncate(DATE_TIME_PRECISION_INDEXED))+ Term::from_columnar_value(field, &val.truncate(DATE_TIME_PRECISION_INDEXED))}/// Builds a term given a field, and a string value@@ -153,38 +153,38 @@ impl Term {/// The use of BigEndian has the benefit of preserving/// the natural order of the values.pub fn set_u64(&mut self, val: u64) {- self.set_fast_value(val);+ self.set_columnar_value(val);}/// Sets a `i64` value in the term.pub fn set_i64(&mut self, val: i64) {- self.set_fast_value(val);+ self.set_columnar_value(val);}/// Sets a `DateTime` value in the term.pub fn set_date(&mut self, date: DateTime) {- self.set_fast_value(date);+ self.set_columnar_value(date);}/// Sets a `f64` value in the term.pub fn set_f64(&mut self, val: f64) {- self.set_fast_value(val);+ self.set_columnar_value(val);}/// Sets a `bool` value in the term.pub fn set_bool(&mut self, val: bool) {- self.set_fast_value(val);+ self.set_columnar_value(val);}- fn set_fast_value<T: FastValue>(&mut self, val: T) {+ fn set_columnar_value<T: ColumnarValue>(&mut self, val: T) {self.set_bytes(val.to_u64().to_be_bytes().as_ref());}- /// Append a type marker + fast value to a term.- /// This is used in JSON type to append a fast value after the path.+ /// Append a type marker + columnar value to a term.+ /// This is used in JSON type to append a columnar value after the path.////// It will not clear existing bytes.- pub fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {+ pub fn append_type_and_columnar_value<T: ColumnarValue>(&mut self, val: T) {self.0.push(T::to_type().to_code());let value = if T::to_type() == Type::Date {DateTime::from_u64(val.to_u64())@@ -340,10 +340,10 @@ where/// Returns `None` if the term is not of the u64 type, or if the term byte representation/// is invalid.pub fn as_u64(&self) -> Option<u64> {- self.get_fast_type::<u64>()+ self.get_columnar_type::<u64>()}- fn get_fast_type<T: FastValue>(&self) -> Option<T> {+ fn get_columnar_type<T: ColumnarValue>(&self) -> Option<T> {if self.typ() != T::to_type() {return None;}@@ -357,7 +357,7 @@ where/// Returns `None` if the term is not of the i64 type, or if the term byte representation/// is invalid.pub fn as_i64(&self) -> Option<i64> {- self.get_fast_type::<i64>()+ self.get_columnar_type::<i64>()}/// Returns the `f64` value stored in a term.@@ -365,7 +365,7 @@ where/// Returns `None` if the term is not of the f64 type, or if the term byte representation/// is invalid.pub fn as_f64(&self) -> Option<f64> {- self.get_fast_type::<f64>()+ self.get_columnar_type::<f64>()}/// Returns the `bool` value stored in a term.@@ -373,7 +373,7 @@ where/// Returns `None` if the term is not of the bool type, or if the term byte representation/// is invalid.pub fn as_bool(&self) -> Option<bool> {- self.get_fast_type::<bool>()+ self.get_columnar_type::<bool>()}/// Returns the `Date` value stored in a term.@@ -381,7 +381,7 @@ where/// Returns `None` if the term is not of the Date type, or if the term byte representation/// is invalid.pub fn as_date(&self) -> Option<DateTime> {- self.get_fast_type::<DateTime>()+ self.get_columnar_type::<DateTime>()}/// Returns the text associated with the term.@@ -584,7 +584,7 @@ mod tests {assert_eq!(term.value().as_str(), Some("test"))}- /// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term.+ /// Size (in bytes) of the buffer of a columnar value (u64, i64, f64, or date) term./// <field> + <type byte> + <value len>////// - <field> is a big endian encoded u32 field id@@ -594,7 +594,7 @@ mod tests {////// - <value> is, if this is not the json term, a binary representation specific to the type./// If it is a JSON Term, then it is prepended with the path that leads to this leaf value.- const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8;+ const COLUMN_VALUE_TERM_LEN: usize = 4 + 1 + 8;#[test]pub fn test_term_u64() {@@ -603,7 +603,7 @@ mod tests {let term = Term::from_field_u64(count_field, 983u64);assert_eq!(term.field(), count_field);assert_eq!(term.typ(), Type::U64);- assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN);+ assert_eq!(term.serialized_term().len(), COLUMN_VALUE_TERM_LEN);assert_eq!(term.value().as_u64(), Some(983u64))}@@ -614,7 +614,7 @@ mod tests {let term = Term::from_field_bool(bool_field, true);assert_eq!(term.field(), bool_field);assert_eq!(term.typ(), Type::Bool);- assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN);+ assert_eq!(term.serialized_term().len(), COLUMN_VALUE_TERM_LEN);assert_eq!(term.value().as_bool(), Some(true))}}diff --git a/crates/tantivy/src/schema/text_options.rs b/crates/tantivy/src/schema/text_options.rsindex 1b920248..3bf43a65 100644--- a/crates/tantivy/src/schema/text_options.rs+++ b/crates/tantivy/src/schema/text_options.rs@@ -3,7 +3,7 @@ use std::ops::BitOr;use serde::{Deserialize, Serialize};-use super::flags::{CoerceFlag, FastFlag};+use super::flags::{CoerceFlag, ColumnarFlag};use crate::schema::flags::{SchemaFlagList, StoredFlag};use crate::schema::IndexRecordOption;@@ -16,7 +16,7 @@ pub struct TextOptions {#[serde(default)]stored: bool,#[serde(default)]- pub(crate) fast: FastFieldTextOptions,+ pub(crate) columnar: ColumnFieldTextOptions,#[serde(default)]#[serde(skip_serializing_if = "is_false")]/// coerce values into string if they are not of type string@@ -25,43 +25,47 @@ pub struct TextOptions {#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]#[serde(untagged)]-/// Enum to control how the fast field setting of a text field.-pub(crate) enum FastFieldTextOptions {+/// Enum to control how the columnar field setting of a text field.+pub(crate) enum ColumnFieldTextOptions {/// Flag to enable/disableIsEnabled(bool),- /// Enable with tokenizer. The tokenizer must be available on the fast field tokenizer manager.- /// `Index::fast_field_tokenizer`.+ /// Enable with tokenizer. The tokenizer must be available on the columnar field tokenizer manager.+ /// `Index::column_field_tokenizer`.EnabledWithTokenizer { with_tokenizer: TokenizerName },}-impl Default for FastFieldTextOptions {+impl Default for ColumnFieldTextOptions {fn default() -> Self {- FastFieldTextOptions::IsEnabled(false)+ ColumnFieldTextOptions::IsEnabled(false)}}-impl BitOr<FastFieldTextOptions> for FastFieldTextOptions {- type Output = FastFieldTextOptions;+impl BitOr<ColumnFieldTextOptions> for ColumnFieldTextOptions {+ type Output = ColumnFieldTextOptions;- fn bitor(self, other: FastFieldTextOptions) -> FastFieldTextOptions {+ fn bitor(self, other: ColumnFieldTextOptions) -> ColumnFieldTextOptions {match (self, other) {(- FastFieldTextOptions::EnabledWithTokenizer {+ ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,},_,)| (_,- FastFieldTextOptions::EnabledWithTokenizer {+ ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,},- ) => FastFieldTextOptions::EnabledWithTokenizer {+ ) => ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,},- (FastFieldTextOptions::IsEnabled(true), _)- | (_, FastFieldTextOptions::IsEnabled(true)) => FastFieldTextOptions::IsEnabled(true),- (_, FastFieldTextOptions::IsEnabled(false)) => FastFieldTextOptions::IsEnabled(false),+ (ColumnFieldTextOptions::IsEnabled(true), _)+ | (_, ColumnFieldTextOptions::IsEnabled(true)) => {+ ColumnFieldTextOptions::IsEnabled(true)+ }+ (_, ColumnFieldTextOptions::IsEnabled(false)) => {+ ColumnFieldTextOptions::IsEnabled(false)+ }}}}@@ -83,22 +87,24 @@ impl TextOptions {self.stored}- /// Returns true if and only if the value is a fast field.+ /// Returns true if and only if the value is a columnar field.#[inline]- pub fn is_fast(&self) -> bool {- matches!(self.fast, FastFieldTextOptions::IsEnabled(true))+ pub fn is_columnar(&self) -> bool {+ matches!(self.columnar, ColumnFieldTextOptions::IsEnabled(true))|| matches!(- &self.fast,- FastFieldTextOptions::EnabledWithTokenizer { with_tokenizer: _ }+ &self.columnar,+ ColumnFieldTextOptions::EnabledWithTokenizer { with_tokenizer: _ })}- /// Returns true if and only if the value is a fast field.+ /// Returns true if and only if the value is a columnar field.#[inline]- pub fn get_fast_field_tokenizer_name(&self) -> Option<&str> {- match &self.fast {- FastFieldTextOptions::IsEnabled(true) | FastFieldTextOptions::IsEnabled(false) => None,- FastFieldTextOptions::EnabledWithTokenizer {+ pub fn get_column_field_tokenizer_name(&self) -> Option<&str> {+ match &self.columnar {+ ColumnFieldTextOptions::IsEnabled(true) | ColumnFieldTextOptions::IsEnabled(false) => {+ None+ }+ ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,} => Some(tokenizer.name()),}@@ -110,30 +116,30 @@ impl TextOptions {self.coerce}- /// Set the field as a fast field.+ /// Set the field as a columnar field.///- /// Fast fields are designed for random access.+ /// Columnar fields are designed for random access./// Access time are similar to a random lookup in an array.- /// Text fast fields will have the term ids stored in the fast field.+ /// Text columnar fields will have the term ids stored in the columnar field.////// The effective cardinality depends on the tokenizer. Without a tokenizer, the text will be/// stored as is, which equals to the "raw" tokenizer. The tokenizer can be used to apply/// normalization like lower case.- /// The passed tokenizer_name must be available on the fast field tokenizer manager.- /// `Index::fast_field_tokenizer`.+ /// The passed tokenizer_name must be available on the columnar field tokenizer manager.+ /// `Index::column_field_tokenizer`.////// The original text can be retrieved via/// [`TermDictionary::ord_to_term()`](crate::termdict::TermDictionary::ord_to_term)/// from the dictionary.#[must_use]- pub fn set_fast(mut self, tokenizer_name: Option<&str>) -> TextOptions {+ pub fn set_columnar(mut self, tokenizer_name: Option<&str>) -> TextOptions {if let Some(tokenizer) = tokenizer_name {let tokenizer = TokenizerName::from_name(tokenizer);- self.fast = FastFieldTextOptions::EnabledWithTokenizer {+ self.columnar = ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: tokenizer,}} else {- self.fast = FastFieldTextOptions::IsEnabled(true);+ self.columnar = ColumnFieldTextOptions::IsEnabled(true);}self}@@ -268,7 +274,7 @@ pub const STRING: TextOptions = TextOptions {record: IndexRecordOption::Basic,}),stored: false,- fast: FastFieldTextOptions::IsEnabled(false),+ columnar: ColumnFieldTextOptions::IsEnabled(false),coerce: false,};@@ -281,7 +287,7 @@ pub const TEXT: TextOptions = TextOptions {}),stored: false,coerce: false,- fast: FastFieldTextOptions::IsEnabled(false),+ columnar: ColumnFieldTextOptions::IsEnabled(false),};impl<T: Into<TextOptions>> BitOr<T> for TextOptions {@@ -292,7 +298,7 @@ impl<T: Into<TextOptions>> BitOr<T> for TextOptions {TextOptions {indexing: self.indexing.or(other.indexing),stored: self.stored | other.stored,- fast: self.fast | other.fast,+ columnar: self.columnar | other.columnar,coerce: self.coerce | other.coerce,}}@@ -309,7 +315,7 @@ impl From<StoredFlag> for TextOptions {TextOptions {indexing: None,stored: true,- fast: FastFieldTextOptions::default(),+ columnar: ColumnFieldTextOptions::default(),coerce: false,}}@@ -320,18 +326,18 @@ impl From<CoerceFlag> for TextOptions {TextOptions {indexing: None,stored: false,- fast: FastFieldTextOptions::default(),+ columnar: ColumnFieldTextOptions::default(),coerce: true,}}}-impl From<FastFlag> for TextOptions {- fn from(_: FastFlag) -> TextOptions {+impl From<ColumnarFlag> for TextOptions {+ fn from(_: ColumnarFlag) -> TextOptions {TextOptions {indexing: None,stored: false,- fast: FastFieldTextOptions::IsEnabled(true),+ columnar: ColumnFieldTextOptions::IsEnabled(true),coerce: false,}}@@ -350,7 +356,7 @@ where#[cfg(test)]mod tests {- use crate::schema::text_options::{FastFieldTextOptions, TokenizerName};+ use crate::schema::text_options::{ColumnFieldTextOptions, TokenizerName};use crate::schema::*;#[test]@@ -395,42 +401,42 @@ mod tests {}#[test]- fn serde_fast_field_tokenizer() {+ fn serde_column_field_tokenizer() {let json = r#" {- "fast": { "with_tokenizer": "default" }+ "columnar": { "with_tokenizer": "default" }} "#;let options: TextOptions = serde_json::from_str(json).unwrap();assert_eq!(- options.fast,- FastFieldTextOptions::EnabledWithTokenizer {+ options.columnar,+ ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: TokenizerName::from_static("default")});let options: TextOptions =serde_json::from_str(&serde_json::to_string(&options).unwrap()).unwrap();assert_eq!(- options.fast,- FastFieldTextOptions::EnabledWithTokenizer {+ options.columnar,+ ColumnFieldTextOptions::EnabledWithTokenizer {with_tokenizer: TokenizerName::from_static("default")});let json = r#" {- "fast": true+ "columnar": true} "#;let options: TextOptions = serde_json::from_str(json).unwrap();- assert_eq!(options.fast, FastFieldTextOptions::IsEnabled(true));+ assert_eq!(options.columnar, ColumnFieldTextOptions::IsEnabled(true));let options: TextOptions =serde_json::from_str(&serde_json::to_string(&options).unwrap()).unwrap();- assert_eq!(options.fast, FastFieldTextOptions::IsEnabled(true));+ assert_eq!(options.columnar, ColumnFieldTextOptions::IsEnabled(true));let json = r#" {- "fast": false+ "columnar": false} "#;let options: TextOptions = serde_json::from_str(json).unwrap();- assert_eq!(options.fast, FastFieldTextOptions::IsEnabled(false));+ assert_eq!(options.columnar, ColumnFieldTextOptions::IsEnabled(false));let options: TextOptions =serde_json::from_str(&serde_json::to_string(&options).unwrap()).unwrap();- assert_eq!(options.fast, FastFieldTextOptions::IsEnabled(false));+ assert_eq!(options.columnar, ColumnFieldTextOptions::IsEnabled(false));}}diff --git a/crates/tantivy/src/space_usage/mod.rs b/crates/tantivy/src/space_usage/mod.rsindex 81a93b86..17df8381 100644--- a/crates/tantivy/src/space_usage/mod.rs+++ b/crates/tantivy/src/space_usage/mod.rs@@ -67,7 +67,7 @@ pub struct SegmentSpaceUsage {termdict: PerFieldSpaceUsage,postings: PerFieldSpaceUsage,positions: PerFieldSpaceUsage,- fast_fields: PerFieldSpaceUsage,+ column_fields: PerFieldSpaceUsage,fieldnorms: PerFieldSpaceUsage,store: StoreSpaceUsage,@@ -82,14 +82,14 @@ impl SegmentSpaceUsage {termdict: PerFieldSpaceUsage,postings: PerFieldSpaceUsage,positions: PerFieldSpaceUsage,- fast_fields: PerFieldSpaceUsage,+ column_fields: PerFieldSpaceUsage,fieldnorms: PerFieldSpaceUsage,store: StoreSpaceUsage,) -> SegmentSpaceUsage {let total = termdict.total()+ postings.total()+ positions.total()- + fast_fields.total()+ + column_fields.total()+ fieldnorms.total()+ store.total();SegmentSpaceUsage {@@ -97,7 +97,7 @@ impl SegmentSpaceUsage {termdict,postings,positions,- fast_fields,+ column_fields,fieldnorms,store,total,@@ -114,7 +114,7 @@ impl SegmentSpaceUsage {match component {Postings => PerField(self.postings().clone()),Positions => PerField(self.positions().clone()),- FastFields => PerField(self.fast_fields().clone()),+ ColumnFields => PerField(self.column_fields().clone()),FieldNorms => PerField(self.fieldnorms().clone()),Terms => PerField(self.termdict().clone()),SegmentComponent::Store => ComponentSpaceUsage::Store(self.store().clone()),@@ -142,9 +142,9 @@ impl SegmentSpaceUsage {&self.positions}- /// Space usage for fast fields- pub fn fast_fields(&self) -> &PerFieldSpaceUsage {- &self.fast_fields+ /// Space usage for columnar fields+ pub fn column_fields(&self) -> &PerFieldSpaceUsage {+ &self.column_fields}/// Space usage for field norms@@ -280,7 +280,7 @@ impl FieldUsage {#[cfg(test)]mod test {use crate::index::Index;- use crate::schema::{Field, Schema, FAST, INDEXED, STORED, TEXT};+ use crate::schema::{Field, Schema, COLUMN, INDEXED, STORED, TEXT};use crate::space_usage::PerFieldSpaceUsage;use crate::IndexWriter;@@ -312,9 +312,9 @@ mod test {}#[test]- fn test_fast_indexed() -> crate::Result<()> {+ fn test_columnar_indexed() -> crate::Result<()> {let mut schema_builder = Schema::builder();- let name = schema_builder.add_u64_field("name", FAST | INDEXED);+ let name = schema_builder.add_u64_field("name", COLUMN | INDEXED);let schema = schema_builder.build();let index = Index::create_in_ram(schema);@@ -341,7 +341,7 @@ mod test {expect_single_field(segment.termdict(), &name, 1, 512);expect_single_field(segment.postings(), &name, 1, 512);assert_eq!(segment.positions().total(), 0);- expect_single_field(segment.fast_fields(), &name, 1, 512);+ expect_single_field(segment.column_fields(), &name, 1, 512);expect_single_field(segment.fieldnorms(), &name, 1, 512);// TODO: understand why the following fails// assert_eq!(0, segment.store().total());@@ -380,7 +380,7 @@ mod test {expect_single_field(segment.termdict(), &name, 1, 512);expect_single_field(segment.postings(), &name, 1, 512);expect_single_field(segment.positions(), &name, 1, 512);- assert_eq!(segment.fast_fields().total(), 0);+ assert_eq!(segment.column_fields().total(), 0);expect_single_field(segment.fieldnorms(), &name, 1, 512);// TODO: understand why the following fails// assert_eq!(0, segment.store().total());@@ -418,7 +418,7 @@ mod test {assert_eq!(segment.termdict().total(), 0);assert_eq!(segment.postings().total(), 0);assert_eq!(segment.positions().total(), 0);- assert_eq!(segment.fast_fields().total(), 0);+ assert_eq!(segment.column_fields().total(), 0);assert_eq!(segment.fieldnorms().total(), 0);assert!(segment.store().total() > 0);assert!(segment.store().total() < 512);@@ -455,7 +455,7 @@ mod test {expect_single_field(segment_space_usage.termdict(), &name, 1, 512);expect_single_field(segment_space_usage.postings(), &name, 1, 512);assert_eq!(segment_space_usage.positions().total(), 0u64);- assert_eq!(segment_space_usage.fast_fields().total(), 0u64);+ assert_eq!(segment_space_usage.column_fields().total(), 0u64);expect_single_field(segment_space_usage.fieldnorms(), &name, 1, 512);Ok(())}