web search done right
f64 and bool types in numerical fields
Mikkel Denkere44f34e2613c
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex 46b3f028..ee46d4f5 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -30,7 +30,7 @@ use crate::{inverted_index::{DocAddress, WebpagePointer},prehashed::Prehashed,ranking::initial::{InitialScoreTweaker, Score},- schema::{column_field, ColumnFieldEnum},+ schema::{numerical_field, NumericalFieldEnum},simhash,};@@ -122,7 +122,12 @@ pub struct TopSegmentCollector {}impl TopSegmentCollector {- fn get_hash(&self, doc: DocId, field1: ColumnFieldEnum, field2: ColumnFieldEnum) -> Prehashed {+ fn get_hash(+ &self,+ doc: DocId,+ field1: NumericalFieldEnum,+ field2: NumericalFieldEnum,+ ) -> Prehashed {let field_reader = self.columnfield_segment_reader.get_field_reader(doc);let hash = [@@ -152,7 +157,7 @@ impl TopSegmentCollector {let simhash: Option<u64> = self.columnfield_segment_reader.get_field_reader(doc)- .get(column_field::SimHash.into())+ .get(numerical_field::SimHash.into()).unwrap().into();@@ -160,23 +165,23 @@ impl TopSegmentCollector {hashes: Hashes {site: self.get_hash(doc,- column_field::SiteHash1.into(),- column_field::SiteHash2.into(),+ numerical_field::SiteHash1.into(),+ numerical_field::SiteHash2.into(),),title: self.get_hash(doc,- column_field::TitleHash1.into(),- column_field::TitleHash2.into(),+ numerical_field::TitleHash1.into(),+ numerical_field::TitleHash2.into(),),url: self.get_hash(doc,- column_field::UrlHash1.into(),- column_field::UrlHash2.into(),+ numerical_field::UrlHash1.into(),+ numerical_field::UrlHash2.into(),),url_without_tld: self.get_hash(doc,- column_field::UrlWithoutTldHash1.into(),- column_field::UrlWithoutTldHash2.into(),+ numerical_field::UrlWithoutTldHash1.into(),+ numerical_field::UrlWithoutTldHash2.into(),),simhash: simhash.unwrap(),},diff --git a/crates/core/src/columnfield_reader.rs b/crates/core/src/columnfield_reader.rsindex 25a07865..d69d6119 100644--- a/crates/core/src/columnfield_reader.rs+++ b/crates/core/src/columnfield_reader.rs@@ -20,7 +20,7 @@ use tantivy::{columnar::ColumnValues, index::SegmentId, DocId};use crate::{enum_map::EnumMap,- schema::{column_field::ColumnField, ColumnFieldEnum, DataType, Field},+ schema::{numerical_field::NumericalField, DataType, Field, NumericalFieldEnum},};#[derive(Default, Clone)]@@ -52,23 +52,28 @@ impl ColumnFieldReader {let mut u64s = EnumMap::new();let mut bytes = EnumMap::new();+ let mut bools = EnumMap::new();+ let mut f64s = EnumMap::new();- for field in Field::all().filter_map(|f| f.as_fast()) {+ for field in Field::all().filter_map(|f| f.as_numerical()) {match field.data_type() {DataType::U64 => {- let num_docs = reader.max_doc() as usize;- let mut data = vec![0; num_docs];- if let Ok(field_reader) = columnfield_readers.u64(field.name()) {- for (doc, elem) in data.iter_mut().enumerate() {- *elem = field_reader.values.get_val(doc as u32);- }+ if let Ok(reader) = columnfield_readers.u64(field.name()) {+ u64s.insert(field, reader.values);+ }+ }+ DataType::F64 => {+ if let Ok(reader) = columnfield_readers.f64(field.name()) {+ f64s.insert(field, reader.values);+ }+ }+ DataType::Bool => {+ if let Ok(reader) = columnfield_readers.bool(field.name()) {+ bools.insert(field, reader.values);}-- u64s.insert(field, data);}DataType::Bytes => {- if let Some(reader) = columnfield_readers.bytes(field.name()).ok().flatten()- {+ if let Ok(Some(reader)) = columnfield_readers.bytes(field.name()) {bytes.insert(field, reader);}}@@ -78,7 +83,12 @@ impl ColumnFieldReader {segments.insert(reader.segment_id(),Arc::new(SegmentReader {- field_readers: AllReaders { u64s, bytes },+ field_readers: AllReaders {+ u64s,+ bytes,+ bools,+ f64s,+ },}),);}@@ -90,13 +100,17 @@ impl ColumnFieldReader {}struct AllReaders {- u64s: EnumMap<ColumnFieldEnum, Vec<u64>>,- bytes: EnumMap<ColumnFieldEnum, tantivy::columnar::BytesColumn>,+ u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,+ f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,+ bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,+ bytes: EnumMap<NumericalFieldEnum, tantivy::columnar::BytesColumn>,}pub enum Value {U64(u64),+ F64(f64),Bytes(Vec<u8>),+ Bool(bool),}impl Value {@@ -107,12 +121,26 @@ impl Value {}}+ pub fn as_f64(&self) -> Option<f64> {+ match self {+ Value::F64(val) => Some(*val),+ _ => None,+ }+ }+pub fn as_bytes(&self) -> Option<&[u8]> {match self {Value::Bytes(val) => Some(val),_ => None,}}++ pub fn as_bool(&self) -> Option<bool> {+ match self {+ Value::Bool(val) => Some(*val),+ _ => None,+ }+ }}impl From<u64> for Value {@@ -121,12 +149,24 @@ impl From<u64> for Value {}}+impl From<f64> for Value {+ fn from(val: f64) -> Self {+ Value::F64(val)+ }+}+impl From<Vec<u8>> for Value {fn from(val: Vec<u8>) -> Self {Value::Bytes(val)}}+impl From<bool> for Value {+ fn from(val: bool) -> Self {+ Value::Bool(val)+ }+}+impl From<Value> for Option<u64> {fn from(val: Value) -> Self {val.as_u64()@@ -148,22 +188,25 @@ impl From<Value> for Option<Vec<u8>> {}}+impl From<Value> for Option<bool> {+ fn from(val: Value) -> Self {+ val.as_bool()+ }+}+pub struct FieldReader<'a> {readers: &'a AllReaders,doc: DocId,}impl<'a> FieldReader<'a> {- pub fn get(&self, field: ColumnFieldEnum) -> Option<Value> {+ pub fn get(&self, field: NumericalFieldEnum) -> Option<Value> {match field.data_type() {- DataType::U64 => Some(- self.readers- .u64s- .get(field)?- .get(self.doc as usize)- .copied()?- .into(),- ),+ DataType::U64 => Some(self.readers.u64s.get(field)?.get_val(self.doc).into()),++ DataType::F64 => Some(self.readers.f64s.get(field)?.get_val(self.doc).into()),++ DataType::Bool => Some(self.readers.bools.get(field)?.get_val(self.doc).into()),DataType::Bytes => {let reader = self.readers.bytes.get(field)?;diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 5b3aec9e..4f8043a7 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -46,7 +46,7 @@ use crate::config::SnippetConfig;use crate::ranking::initial::Score;use crate::schema::text_field::TextField;-use crate::schema::{column_field, text_field, ColumnFieldEnum, Field, TextFieldEnum};+use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};use crate::snippet::TextSnippet;use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Stemmed, TrigramTokenizer, UrlTokenizer,@@ -151,9 +151,11 @@ impl InvertedIndex {} else {let index_settings = tantivy::IndexSettings {sort_by_field: Some(tantivy::IndexSortByField {- field: Field::Columnar(ColumnFieldEnum::from(column_field::PreComputedScore))- .name()- .to_string(),+ field: Field::Numerical(NumericalFieldEnum::from(+ numerical_field::PreComputedScore,+ ))+ .name()+ .to_string(),order: tantivy::Order::Desc,}),..Default::default()@@ -277,7 +279,7 @@ impl From<TantivyDocument> for RetrievedWebpage {Some(Field::Text(TextFieldEnum::Url(_))) => {webpage.url = str_value(text_field::Url.name(), &value);}- Some(Field::Columnar(ColumnFieldEnum::LastUpdated(_))) => {+ Some(Field::Numerical(NumericalFieldEnum::LastUpdated(_))) => {webpage.updated_time = {let timestamp = value.as_u64().unwrap() as i64;if timestamp == 0 {@@ -290,7 +292,7 @@ impl From<TantivyDocument> for RetrievedWebpage {Some(Field::Text(TextFieldEnum::AllBody(_))) => {webpage.dirty_body = str_value(text_field::AllBody.name(), &value);}- Some(Field::Columnar(ColumnFieldEnum::Region(_))) => {+ Some(Field::Numerical(NumericalFieldEnum::Region(_))) => {webpage.region = {let id = value.as_u64().unwrap();Region::from_id(id)@@ -304,11 +306,11 @@ impl From<TantivyDocument> for RetrievedWebpage {let json = str_value(text_field::SchemaOrgJson.name(), &value);webpage.schema_org = serde_json::from_str(&json).unwrap_or_default();}- Some(Field::Columnar(ColumnFieldEnum::LikelyHasAds(_))) => {- webpage.likely_has_ads = value.as_u64().unwrap_or_default() != 0;+ Some(Field::Numerical(NumericalFieldEnum::LikelyHasAds(_))) => {+ webpage.likely_has_ads = value.as_bool().unwrap_or_default();}- Some(Field::Columnar(ColumnFieldEnum::LikelyHasPaywall(_))) => {- webpage.likely_has_paywall = value.as_u64().unwrap_or_default() != 0;+ Some(Field::Numerical(NumericalFieldEnum::LikelyHasPaywall(_))) => {+ webpage.likely_has_paywall = value.as_bool().unwrap_or_default();}Some(Field::Text(TextFieldEnum::RecipeFirstIngredientTagId(_))) => {let tag_id = str_value(text_field::RecipeFirstIngredientTagId.name(), &value);diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 89e8cc2b..f5ec6dfc 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -32,7 +32,7 @@ use crate::query::shortcircuit::ShortCircuitQuery;use crate::query::Query;use crate::ranking::pipeline::LocalRecallRankingWebpage;use crate::ranking::SignalComputer;-use crate::schema::{column_field, text_field, ColumnFieldEnum, Field, TextFieldEnum};+use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};use crate::search_ctx::Ctx;use crate::snippet;use crate::snippet::TextSnippet;@@ -156,7 +156,9 @@ impl InvertedIndex {let field = self.schema()- .get_field(Field::Columnar(ColumnFieldEnum::from(column_field::HostNodeID)).name())+ .get_field(+ Field::Numerical(NumericalFieldEnum::from(numerical_field::HostNodeID)).name(),+ ).unwrap();let id = doc.get_first(field).unwrap().as_u64().unwrap();diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex 459cb0b5..3f58d9a7 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -25,7 +25,7 @@ use tantivy::{use crate::{columnfield_reader::{self, ColumnFieldReader},query::intersection::Intersection,- schema::ColumnFieldEnum,+ schema::NumericalFieldEnum,};use super::SmallPatternPart;@@ -127,7 +127,7 @@ impl Scorer for AllScorer {pub struct EmptyFieldScorer {pub segment_reader: Arc<columnfield_reader::SegmentReader>,- pub num_tokens_columnfield: ColumnFieldEnum,+ pub num_tokens_columnfield: NumericalFieldEnum,pub all_scorer: AllScorer,}@@ -210,7 +210,7 @@ pub struct NormalPatternScorer {left: Vec<u32>,right: Vec<u32>,phrase_count: u32,- num_tokens_field: ColumnFieldEnum,+ num_tokens_field: NumericalFieldEnum,segment_reader: Arc<columnfield_reader::SegmentReader>,}@@ -219,7 +219,7 @@ impl NormalPatternScorer {term_postings_list: Vec<SegmentPostings>,pattern: Vec<SmallPatternPart>,segment: tantivy::index::SegmentId,- num_tokens_field: ColumnFieldEnum,+ num_tokens_field: NumericalFieldEnum,columnfield_reader: ColumnFieldReader,) -> Self {let num_query_terms = term_postings_list.len();diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rsindex e8f93aed..8d1cbc05 100644--- a/crates/core/src/query/pattern_query/weight.rs+++ b/crates/core/src/query/pattern_query/weight.rs@@ -24,7 +24,7 @@ use tantivy::{use crate::{columnfield_reader::ColumnFieldReader,schema::{- column_field,+ numerical_field,text_field::{self, TextField},Field, TextFieldEnum,},@@ -62,7 +62,7 @@ impl FastSiteDomainPatternWeight {let opt = match field_no_tokenizer {Field::Text(t) => t.record_option(),- Field::Columnar(_) => unreachable!(),+ Field::Numerical(_) => unreachable!(),};match reader@@ -127,23 +127,27 @@ impl PatternWeight {}let num_tokens_columnfield = match Field::get(self.field.field_id() as usize) {- Some(Field::Text(TextFieldEnum::Title(_))) => Ok(column_field::NumTitleTokens.into()),+ Some(Field::Text(TextFieldEnum::Title(_))) => {+ Ok(numerical_field::NumTitleTokens.into())+ }Some(Field::Text(TextFieldEnum::CleanBody(_))) => {- Ok(column_field::NumCleanBodyTokens.into())+ Ok(numerical_field::NumCleanBodyTokens.into())+ }+ Some(Field::Text(TextFieldEnum::Url(_))) => Ok(numerical_field::NumUrlTokens.into()),+ Some(Field::Text(TextFieldEnum::Domain(_))) => {+ Ok(numerical_field::NumDomainTokens.into())}- Some(Field::Text(TextFieldEnum::Url(_))) => Ok(column_field::NumUrlTokens.into()),- Some(Field::Text(TextFieldEnum::Domain(_))) => Ok(column_field::NumDomainTokens.into()),Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_))) => {- Ok(column_field::NumUrlForSiteOperatorTokens.into())+ Ok(numerical_field::NumUrlForSiteOperatorTokens.into())}Some(Field::Text(TextFieldEnum::Description(_))) => {- Ok(column_field::NumDescriptionTokens.into())+ Ok(numerical_field::NumDescriptionTokens.into())}Some(Field::Text(TextFieldEnum::MicroformatTags(_))) => {- Ok(column_field::NumMicroformatTagsTokens.into())+ Ok(numerical_field::NumMicroformatTagsTokens.into())}Some(Field::Text(TextFieldEnum::FlattenedSchemaOrgJson(_))) => {- Ok(column_field::NumFlattenedSchemaTokens.into())+ Ok(numerical_field::NumFlattenedSchemaTokens.into())}Some(field) => Err(TantivyError::InvalidArgument(format!("{} is not supported in pattern query",diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex a3c87562..d3672eb4 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -28,7 +28,7 @@ use crate::{pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},SignalComputer, SignalEnum,},- schema::column_field,+ schema::numerical_field,searcher::{api, SearchQuery},webgraph,};@@ -160,15 +160,15 @@ impl LocalRecallRankingWebpage {let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);let title_embedding: Option<Vec<u8>> = columnfields- .get(column_field::TitleEmbeddings.into())+ .get(numerical_field::TitleEmbeddings.into()).and_then(|v| v.into());let keyword_embedding: Option<Vec<u8>> = columnfields- .get(column_field::KeywordEmbeddings.into())+ .get(numerical_field::KeywordEmbeddings.into()).and_then(|v| v.into());let host_id = columnfields- .get(column_field::HostNodeID.into())+ .get(numerical_field::HostNodeID.into()).unwrap().as_u64().unwrap()diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex 18228b2d..84167f8e 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -18,7 +18,7 @@ use tantivy::DocId;use super::{Signal, SignalComputer};use crate::{- schema::{self, Field, FLOAT_SCALING},+ schema::{self, Field},webpage::Webpage,};@@ -105,7 +105,9 @@ impl Signal for HostCentrality {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::HostCentrality.into()))+ Some(Field::Numerical(+ schema::numerical_field::HostCentrality.into(),+ ))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -118,9 +120,9 @@ impl Signal for HostCentrality {let val = columnfield_reader.get(self.as_columnfield().unwrap())- .and_then(|v| v.as_u64())+ .and_then(|v| v.as_f64()).unwrap();- Some(val as f64 / FLOAT_SCALING as f64)+ Some(val)}}@@ -143,8 +145,8 @@ impl Signal for HostCentralityRank {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(- schema::column_field::HostCentralityRank.into(),+ Some(Field::Numerical(+ schema::numerical_field::HostCentralityRank.into(),))}@@ -183,7 +185,9 @@ impl Signal for PageCentrality {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::PageCentrality.into()))+ Some(Field::Numerical(+ schema::numerical_field::PageCentrality.into(),+ ))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -196,9 +200,9 @@ impl Signal for PageCentrality {let val = columnfield_reader.get(self.as_columnfield().unwrap())- .and_then(|v| v.as_u64())+ .and_then(|v| v.as_f64()).unwrap();- Some(val as f64 / FLOAT_SCALING as f64)+ Some(val)}}@@ -221,8 +225,8 @@ impl Signal for PageCentralityRank {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(- schema::column_field::PageCentralityRank.into(),+ Some(Field::Numerical(+ schema::numerical_field::PageCentralityRank.into(),))}@@ -261,7 +265,7 @@ impl Signal for IsHomepage {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::IsHomepage.into()))+ Some(Field::Numerical(schema::numerical_field::IsHomepage.into()))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -274,9 +278,14 @@ impl Signal for IsHomepage {let val = columnfield_reader.get(self.as_columnfield().unwrap())- .and_then(|v| v.as_u64())+ .and_then(|v| v.as_bool()).unwrap();- Some(val as f64)++ if val {+ Some(1.0)+ } else {+ Some(0.0)+ }}}@@ -299,7 +308,9 @@ impl Signal for FetchTimeMs {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::FetchTimeMs.into()))+ Some(Field::Numerical(+ schema::numerical_field::FetchTimeMs.into(),+ ))}fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {@@ -347,7 +358,9 @@ impl Signal for UpdateTimestamp {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::LastUpdated.into()))+ Some(Field::Numerical(+ schema::numerical_field::LastUpdated.into(),+ ))}fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {@@ -392,7 +405,9 @@ impl Signal for TrackerScore {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::TrackerScore.into()))+ Some(Field::Numerical(+ schema::numerical_field::TrackerScore.into(),+ ))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -431,7 +446,7 @@ impl Signal for Region {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::Region.into()))+ Some(Field::Numerical(schema::numerical_field::Region.into()))}fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {@@ -553,8 +568,8 @@ impl Signal for UrlDigits {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(- schema::column_field::NumPathAndQueryDigits.into(),+ Some(Field::Numerical(+ schema::numerical_field::NumPathAndQueryDigits.into(),))}@@ -609,8 +624,8 @@ impl Signal for UrlSlashes {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(- schema::column_field::NumPathAndQuerySlashes.into(),+ Some(Field::Numerical(+ schema::numerical_field::NumPathAndQuerySlashes.into(),))}@@ -656,7 +671,9 @@ impl Signal for LinkDensity {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(schema::column_field::LinkDensity.into()))+ Some(Field::Numerical(+ schema::numerical_field::LinkDensity.into(),+ ))}fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {@@ -670,8 +687,8 @@ impl Signal for LinkDensity {let val = columnfield_reader.get(self.as_columnfield().unwrap())- .and_then(|v| v.as_u64())+ .and_then(|v| v.as_f64()).unwrap();- Some(score_link_density(val as f64 / FLOAT_SCALING as f64))+ Some(score_link_density(val))}}diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rsindex 260e04e0..9de46936 100644--- a/crates/core/src/ranking/signal/prelude.rs+++ b/crates/core/src/ranking/signal/prelude.rs@@ -18,7 +18,7 @@ use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};use crate::schema::Field;use crate::{- schema::{ColumnFieldEnum, TextFieldEnum},+ schema::{NumericalFieldEnum, TextFieldEnum},webpage::Webpage,};use enum_dispatch::enum_dispatch;@@ -46,8 +46,8 @@ pub trait Signal:self.as_field().and_then(|field| field.as_text())}- fn as_columnfield(&self) -> Option<ColumnFieldEnum> {- self.as_field().and_then(|field| field.as_fast())+ fn as_columnfield(&self) -> Option<NumericalFieldEnum> {+ self.as_field().and_then(|field| field.as_numerical())}}diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rsindex ce1eedff..f9e9884f 100644--- a/crates/core/src/ranking/signal/text.rs+++ b/crates/core/src/ranking/signal/text.rs@@ -806,8 +806,8 @@ impl Signal for TitleEmbeddingSimilarity {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(- schema::column_field::TitleEmbeddings.into(),+ Some(Field::Numerical(+ schema::numerical_field::TitleEmbeddings.into(),))}@@ -835,8 +835,8 @@ impl Signal for KeywordEmbeddingSimilarity {}fn as_field(&self) -> Option<Field> {- Some(Field::Columnar(- schema::column_field::KeywordEmbeddings.into(),+ Some(Field::Numerical(+ schema::numerical_field::KeywordEmbeddings.into(),))}diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rsindex ba34fa75..aa6842e0 100644--- a/crates/core/src/schema/mod.rs+++ b/crates/core/src/schema/mod.rs@@ -14,21 +14,19 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub mod column_field;+pub mod numerical_field;pub mod text_field;use tantivy::schema::{BytesOptions, DateOptions, NumericOptions, TextOptions};-pub use column_field::{ColumnFieldEnum, DataType};+pub use numerical_field::{DataType, NumericalFieldEnum};pub use text_field::TextFieldEnum;-use self::{column_field::ColumnField, text_field::TextField};--pub const FLOAT_SCALING: u64 = 1_000_000_000;+use self::{numerical_field::NumericalField, text_field::TextField};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Field {- Columnar(ColumnFieldEnum),+ Numerical(NumericalFieldEnum),Text(TextFieldEnum),}@@ -40,10 +38,10 @@ impl Field {}let field_id = field_id - TextFieldEnum::num_variants();- if field_id < ColumnFieldEnum::num_variants() {- return Some(Field::Columnar(ColumnFieldEnum::get(field_id).unwrap()));+ if field_id < NumericalFieldEnum::num_variants() {+ return Some(Field::Numerical(NumericalFieldEnum::get(field_id).unwrap()));}- let _field_id = field_id - ColumnFieldEnum::num_variants();+ let _field_id = field_id - NumericalFieldEnum::num_variants();None}@@ -52,12 +50,12 @@ impl Field {pub fn all() -> impl Iterator<Item = Field> {TextFieldEnum::all().map(Field::Text)- .chain(ColumnFieldEnum::all().map(Field::Columnar))+ .chain(NumericalFieldEnum::all().map(Field::Numerical))}pub fn has_pos(&self) -> bool {match self {- Field::Columnar(_) => false,+ Field::Numerical(_) => false,Field::Text(text) => text.has_pos(),}}@@ -65,34 +63,34 @@ impl Field {pub fn indexing_option(&self) -> IndexingOption {match self {Field::Text(f) => f.indexing_option(),- Field::Columnar(f) => f.indexing_option(),+ Field::Numerical(f) => f.indexing_option(),}}pub fn name(&self) -> &str {match self {Field::Text(f) => f.name(),- Field::Columnar(f) => f.name(),+ Field::Numerical(f) => f.name(),}}pub fn is_searchable(&self) -> bool {match self {Field::Text(f) => f.is_searchable(),- Field::Columnar(_) => false,+ Field::Numerical(_) => false,}}pub fn as_text(&self) -> Option<TextFieldEnum> {match self {- Field::Columnar(_) => None,+ Field::Numerical(_) => None,Field::Text(field) => Some(*field),}}- pub fn as_fast(&self) -> Option<ColumnFieldEnum> {+ pub fn as_numerical(&self) -> Option<NumericalFieldEnum> {match self {- Field::Columnar(field) => Some(*field),+ Field::Numerical(field) => Some(*field),Field::Text(_) => None,}}@@ -104,7 +102,17 @@ pub fn create_schema() -> tantivy::schema::Schema {for field in Field::all() {match field.indexing_option() {IndexingOption::Text(options) => builder.add_text_field(field.name(), options),- IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),+ IndexingOption::Integer(options) => {+ let fast = field.as_numerical().expect("Expected fast field");+ match fast.data_type() {+ DataType::U64 => builder.add_u64_field(field.name(), options),+ DataType::F64 => builder.add_f64_field(field.name(), options),+ DataType::Bool => builder.add_bool_field(field.name(), options),+ DataType::Bytes => {+ panic!("bytes field should have a `Bytes` variant as indexing option")+ }+ }+ }IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),};diff --git a/crates/core/src/schema/column_field.rs b/crates/core/src/schema/numerical_field.rssimilarity index 85%rename from crates/core/src/schema/column_field.rsrename to crates/core/src/schema/numerical_field.rsindex a31ff09b..87bff767 100644--- a/crates/core/src/schema/column_field.rs+++ b/crates/core/src/schema/numerical_field.rs@@ -29,10 +29,12 @@ use crate::{Result,};-use super::{IndexingOption, FLOAT_SCALING};+use super::IndexingOption;#[enum_dispatch]-pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {+pub trait NumericalField:+ Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash+{fn name(&self) -> &str;fn add_html_tantivy(&self,@@ -64,19 +66,30 @@ pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::ha}fn indexing_option(&self) -> IndexingOption {- debug_assert!(matches!(self.data_type(), DataType::U64));+ match self.data_type() {+ DataType::U64 | DataType::Bool | DataType::F64 => {+ let mut opt = NumericOptions::default().set_columnar();- let mut opt = NumericOptions::default().set_columnar();+ if self.is_stored() {+ opt = opt.set_stored();+ }- if self.is_stored() {- opt = opt.set_stored();- }+ if self.is_indexed() {+ opt = opt.set_indexed();+ }- if self.is_indexed() {- opt = opt.set_indexed();- }+ IndexingOption::Integer(opt)+ }+ DataType::Bytes => {+ let mut opt = BytesOptions::default().set_columnar().set_indexed();- IndexingOption::Integer(opt)+ if self.is_stored() {+ opt = opt.set_stored();+ }++ IndexingOption::Bytes(opt)+ }+ }}fn tantivy_field(&self, schema: &tantivy::schema::Schema) -> tantivy::schema::Field {@@ -86,10 +99,10 @@ pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::ha}}-#[enum_dispatch(ColumnField)]+#[enum_dispatch(NumericalField)]#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants)]#[strum_discriminants(derive(VariantArray))]-pub enum ColumnFieldEnum {+pub enum NumericalFieldEnum {IsHomepage,HostCentrality,HostCentralityRank,@@ -131,7 +144,7 @@ pub enum ColumnFieldEnum {KeywordEmbeddings,}-enum_dispatch_from_discriminant!(ColumnFieldEnumDiscriminants => ColumnFieldEnum,+enum_dispatch_from_discriminant!(NumericalFieldEnumDiscriminants => NumericalFieldEnum,[IsHomepage,HostCentrality,@@ -174,44 +187,50 @@ enum_dispatch_from_discriminant!(ColumnFieldEnumDiscriminants => ColumnFieldEnumKeywordEmbeddings,]);-impl ColumnFieldEnum {- pub fn all() -> impl Iterator<Item = ColumnFieldEnum> {- ColumnFieldEnumDiscriminants::VARIANTS+impl NumericalFieldEnum {+ pub fn all() -> impl Iterator<Item = NumericalFieldEnum> {+ NumericalFieldEnumDiscriminants::VARIANTS.iter().copied().map(|v| v.into())}- pub fn get(field_id: usize) -> Option<ColumnFieldEnum> {- ColumnFieldEnumDiscriminants::VARIANTS+ pub fn get(field_id: usize) -> Option<NumericalFieldEnum> {+ NumericalFieldEnumDiscriminants::VARIANTS.get(field_id).copied()- .map(ColumnFieldEnum::from)+ .map(NumericalFieldEnum::from)}pub fn num_variants() -> usize {- ColumnFieldEnumDiscriminants::VARIANTS.len()+ NumericalFieldEnumDiscriminants::VARIANTS.len()}}pub enum DataType {U64,+ F64,+ Bool,Bytes,}-impl InsertEnumMapKey for ColumnFieldEnum {+impl InsertEnumMapKey for NumericalFieldEnum {fn into_usize(self) -> usize {- ColumnFieldEnumDiscriminants::from(self) as usize+ NumericalFieldEnumDiscriminants::from(self) as usize}}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct IsHomepage;-impl ColumnField for IsHomepage {+impl NumericalField for IsHomepage {fn name(&self) -> &str {"is_homepage"}+ fn data_type(&self) -> DataType {+ DataType::Bool+ }+fn add_html_tantivy(&self,html: &Html,@@ -219,7 +238,7 @@ impl ColumnField for IsHomepage {doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), (html.is_homepage()).into());+ doc.add_bool(self.tantivy_field(schema), html.is_homepage());Ok(())}@@ -227,7 +246,7 @@ impl ColumnField for IsHomepage {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct HostCentrality;-impl ColumnField for HostCentrality {+impl NumericalField for HostCentrality {fn name(&self) -> &str {"host_centrality"}@@ -242,16 +261,17 @@ impl ColumnField for HostCentrality {Ok(())}+ fn data_type(&self) -> DataType {+ DataType::F64+ }+fn add_webpage_tantivy(&self,webpage: &Webpage,doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(- self.tantivy_field(schema),- (webpage.host_centrality * FLOAT_SCALING as f64) as u64,- );+ doc.add_f64(self.tantivy_field(schema), webpage.host_centrality);Ok(())}@@ -259,7 +279,7 @@ impl ColumnField for HostCentrality {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct HostCentralityRank;-impl ColumnField for HostCentralityRank {+impl NumericalField for HostCentralityRank {fn name(&self) -> &str {"host_centrality_rank"}@@ -288,7 +308,7 @@ impl ColumnField for HostCentralityRank {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct PageCentrality;-impl ColumnField for PageCentrality {+impl NumericalField for PageCentrality {fn name(&self) -> &str {"page_centrality"}@@ -302,16 +322,17 @@ impl ColumnField for PageCentrality {Ok(())}+ fn data_type(&self) -> DataType {+ DataType::F64+ }+fn add_webpage_tantivy(&self,webpage: &Webpage,doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(- self.tantivy_field(schema),- (webpage.page_centrality * FLOAT_SCALING as f64) as u64,- );+ doc.add_f64(self.tantivy_field(schema), webpage.page_centrality);Ok(())}@@ -319,7 +340,7 @@ impl ColumnField for PageCentrality {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct PageCentralityRank;-impl ColumnField for PageCentralityRank {+impl NumericalField for PageCentralityRank {fn name(&self) -> &str {"page_centrality_rank"}@@ -347,7 +368,7 @@ impl ColumnField for PageCentralityRank {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct FetchTimeMs;-impl ColumnField for FetchTimeMs {+impl NumericalField for FetchTimeMs {fn name(&self) -> &str {"fetch_time_ms"}@@ -375,7 +396,7 @@ impl ColumnField for FetchTimeMs {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LastUpdated;-impl ColumnField for LastUpdated {+impl NumericalField for LastUpdated {fn name(&self) -> &str {"last_updated"}@@ -403,7 +424,7 @@ impl ColumnField for LastUpdated {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TrackerScore;-impl ColumnField for TrackerScore {+impl NumericalField for TrackerScore {fn name(&self) -> &str {"tracker_score"}@@ -423,7 +444,7 @@ impl ColumnField for TrackerScore {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct Region;-impl ColumnField for Region {+impl NumericalField for Region {fn name(&self) -> &str {"region"}@@ -464,7 +485,7 @@ impl ColumnField for Region {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumUrlTokens;-impl ColumnField for NumUrlTokens {+impl NumericalField for NumUrlTokens {fn name(&self) -> &str {"num_url_tokens"}@@ -487,7 +508,7 @@ impl ColumnField for NumUrlTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumTitleTokens;-impl ColumnField for NumTitleTokens {+impl NumericalField for NumTitleTokens {fn name(&self) -> &str {"num_title_tokens"}@@ -514,7 +535,7 @@ impl ColumnField for NumTitleTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumCleanBodyTokens;-impl ColumnField for NumCleanBodyTokens {+impl NumericalField for NumCleanBodyTokens {fn name(&self) -> &str {"num_clean_body_tokens"}@@ -536,7 +557,7 @@ impl ColumnField for NumCleanBodyTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumDescriptionTokens;-impl ColumnField for NumDescriptionTokens {+impl NumericalField for NumDescriptionTokens {fn name(&self) -> &str {"num_description_tokens"}@@ -559,7 +580,7 @@ impl ColumnField for NumDescriptionTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumUrlForSiteOperatorTokens;-impl ColumnField for NumUrlForSiteOperatorTokens {+impl NumericalField for NumUrlForSiteOperatorTokens {fn name(&self) -> &str {"num_url_for_site_operator_tokens"}@@ -582,7 +603,7 @@ impl ColumnField for NumUrlForSiteOperatorTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumDomainTokens;-impl ColumnField for NumDomainTokens {+impl NumericalField for NumDomainTokens {fn name(&self) -> &str {"num_domain_tokens"}@@ -605,7 +626,7 @@ impl ColumnField for NumDomainTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumMicroformatTagsTokens;-impl ColumnField for NumMicroformatTagsTokens {+impl NumericalField for NumMicroformatTagsTokens {fn name(&self) -> &str {"num_microformat_tags_tokens"}@@ -628,7 +649,7 @@ impl ColumnField for NumMicroformatTagsTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct SiteHash1;-impl ColumnField for SiteHash1 {+impl NumericalField for SiteHash1 {fn name(&self) -> &str {"site_hash1"}@@ -648,7 +669,7 @@ impl ColumnField for SiteHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct SiteHash2;-impl ColumnField for SiteHash2 {+impl NumericalField for SiteHash2 {fn name(&self) -> &str {"site_hash2"}@@ -668,7 +689,7 @@ impl ColumnField for SiteHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutQueryHash1;-impl ColumnField for UrlWithoutQueryHash1 {+impl NumericalField for UrlWithoutQueryHash1 {fn name(&self) -> &str {"url_without_query_hash1"}@@ -691,7 +712,7 @@ impl ColumnField for UrlWithoutQueryHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutQueryHash2;-impl ColumnField for UrlWithoutQueryHash2 {+impl NumericalField for UrlWithoutQueryHash2 {fn name(&self) -> &str {"url_without_query_hash2"}@@ -714,7 +735,7 @@ impl ColumnField for UrlWithoutQueryHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TitleHash1;-impl ColumnField for TitleHash1 {+impl NumericalField for TitleHash1 {fn name(&self) -> &str {"title_hash1"}@@ -734,7 +755,7 @@ impl ColumnField for TitleHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TitleHash2;-impl ColumnField for TitleHash2 {+impl NumericalField for TitleHash2 {fn name(&self) -> &str {"title_hash2"}@@ -754,7 +775,7 @@ impl ColumnField for TitleHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlHash1;-impl ColumnField for UrlHash1 {+impl NumericalField for UrlHash1 {fn name(&self) -> &str {"url_hash1"}@@ -774,7 +795,7 @@ impl ColumnField for UrlHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlHash2;-impl ColumnField for UrlHash2 {+impl NumericalField for UrlHash2 {fn name(&self) -> &str {"url_hash2"}@@ -794,7 +815,7 @@ impl ColumnField for UrlHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct DomainHash1;-impl ColumnField for DomainHash1 {+impl NumericalField for DomainHash1 {fn name(&self) -> &str {"domain_hash1"}@@ -814,7 +835,7 @@ impl ColumnField for DomainHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct DomainHash2;-impl ColumnField for DomainHash2 {+impl NumericalField for DomainHash2 {fn name(&self) -> &str {"domain_hash2"}@@ -834,7 +855,7 @@ impl ColumnField for DomainHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutTldHash1;-impl ColumnField for UrlWithoutTldHash1 {+impl NumericalField for UrlWithoutTldHash1 {fn name(&self) -> &str {"url_without_tld_hash1"}@@ -854,7 +875,7 @@ impl ColumnField for UrlWithoutTldHash1 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct UrlWithoutTldHash2;-impl ColumnField for UrlWithoutTldHash2 {+impl NumericalField for UrlWithoutTldHash2 {fn name(&self) -> &str {"url_without_tld_hash2"}@@ -874,7 +895,7 @@ impl ColumnField for UrlWithoutTldHash2 {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct PreComputedScore;-impl ColumnField for PreComputedScore {+impl NumericalField for PreComputedScore {fn name(&self) -> &str {"pre_computed_score"}@@ -893,16 +914,17 @@ impl ColumnField for PreComputedScore {Ok(())}+ fn data_type(&self) -> DataType {+ DataType::F64+ }+fn add_webpage_tantivy(&self,webpage: &Webpage,doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(- self.tantivy_field(schema),- (webpage.pre_computed_score * FLOAT_SCALING as f64) as u64,- );+ doc.add_f64(self.tantivy_field(schema), webpage.pre_computed_score);Ok(())}@@ -910,7 +932,7 @@ impl ColumnField for PreComputedScore {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct HostNodeID;-impl ColumnField for HostNodeID {+impl NumericalField for HostNodeID {fn name(&self) -> &str {"host_node_id"}@@ -950,7 +972,7 @@ impl ColumnField for HostNodeID {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct SimHash;-impl ColumnField for SimHash {+impl NumericalField for SimHash {fn name(&self) -> &str {"sim_hash"}@@ -981,7 +1003,7 @@ impl ColumnField for SimHash {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumFlattenedSchemaTokens;-impl ColumnField for NumFlattenedSchemaTokens {+impl NumericalField for NumFlattenedSchemaTokens {fn name(&self) -> &str {"num_flattened_schema_tokens"}@@ -1004,7 +1026,7 @@ impl ColumnField for NumFlattenedSchemaTokens {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumPathAndQuerySlashes;-impl ColumnField for NumPathAndQuerySlashes {+impl NumericalField for NumPathAndQuerySlashes {fn name(&self) -> &str {"num_path_and_query_slashes"}@@ -1034,7 +1056,7 @@ impl ColumnField for NumPathAndQuerySlashes {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct NumPathAndQueryDigits;-impl ColumnField for NumPathAndQueryDigits {+impl NumericalField for NumPathAndQueryDigits {fn name(&self) -> &str {"num_path_and_query_digits"}@@ -1072,7 +1094,7 @@ impl ColumnField for NumPathAndQueryDigits {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LikelyHasAds;-impl ColumnField for LikelyHasAds {+impl NumericalField for LikelyHasAds {fn name(&self) -> &str {"likely_has_ads"}@@ -1081,6 +1103,10 @@ impl ColumnField for LikelyHasAds {true}+ fn data_type(&self) -> DataType {+ DataType::Bool+ }+fn add_html_tantivy(&self,html: &Html,@@ -1088,7 +1114,7 @@ impl ColumnField for LikelyHasAds {doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), html.likely_has_ads() as u64);+ doc.add_bool(self.tantivy_field(schema), html.likely_has_ads());Ok(())}@@ -1096,7 +1122,7 @@ impl ColumnField for LikelyHasAds {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LikelyHasPaywall;-impl ColumnField for LikelyHasPaywall {+impl NumericalField for LikelyHasPaywall {fn name(&self) -> &str {"likely_has_paywall"}@@ -1105,6 +1131,10 @@ impl ColumnField for LikelyHasPaywall {true}+ fn data_type(&self) -> DataType {+ DataType::Bool+ }+fn add_html_tantivy(&self,html: &Html,@@ -1112,7 +1142,7 @@ impl ColumnField for LikelyHasPaywall {doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), html.likely_has_paywall() as u64);+ doc.add_bool(self.tantivy_field(schema), html.likely_has_paywall());Ok(())}@@ -1120,7 +1150,7 @@ impl ColumnField for LikelyHasPaywall {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct LinkDensity;-impl ColumnField for LinkDensity {+impl NumericalField for LinkDensity {fn name(&self) -> &str {"link_density"}@@ -1129,6 +1159,10 @@ impl ColumnField for LinkDensity {true}+ fn data_type(&self) -> DataType {+ DataType::F64+ }+fn add_html_tantivy(&self,html: &Html,@@ -1136,10 +1170,7 @@ impl ColumnField for LinkDensity {doc: &mut TantivyDocument,schema: &tantivy::schema::Schema,) -> Result<()> {- doc.add_u64(- self.tantivy_field(schema),- (html.link_density() * FLOAT_SCALING as f64) as u64,- );+ doc.add_f64(self.tantivy_field(schema), html.link_density());Ok(())}@@ -1147,7 +1178,7 @@ impl ColumnField for LinkDensity {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct TitleEmbeddings;-impl ColumnField for TitleEmbeddings {+impl NumericalField for TitleEmbeddings {fn name(&self) -> &str {"title_embeddings"}@@ -1156,8 +1187,8 @@ impl ColumnField for TitleEmbeddings {DataType::Bytes}- fn indexing_option(&self) -> IndexingOption {- IndexingOption::Bytes(BytesOptions::default().set_columnar().set_stored())+ fn is_stored(&self) -> bool {+ true}fn add_html_tantivy(@@ -1191,7 +1222,7 @@ impl ColumnField for TitleEmbeddings {#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct KeywordEmbeddings;-impl ColumnField for KeywordEmbeddings {+impl NumericalField for KeywordEmbeddings {fn name(&self) -> &str {"keyword_embeddings"}@@ -1200,8 +1231,8 @@ impl ColumnField for KeywordEmbeddings {DataType::Bytes}- fn indexing_option(&self) -> IndexingOption {- IndexingOption::Bytes(BytesOptions::default().set_columnar().set_stored())+ fn is_stored(&self) -> bool {+ true}fn add_html_tantivy(diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex bfaa0235..bcd4a0c9 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -19,7 +19,7 @@ use crate::{prehashed::hash,rake::RakeModel,schema::{- column_field::ColumnField,+ numerical_field::NumericalField,text_field::{self, TextField},TextFieldEnum,},@@ -201,7 +201,7 @@ impl Html {{match field {Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,- Field::Columnar(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,+ Field::Numerical(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,}}diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex 3f88b2ab..c05c6b40 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- schema::{column_field::ColumnField, text_field::TextField, Field},+ schema::{numerical_field::NumericalField, text_field::TextField, Field},webgraph::NodeID,Result,};@@ -134,7 +134,7 @@ impl Webpage {for field in Field::all() {match field {- Field::Columnar(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,+ Field::Numerical(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,}}