Skip to content

web search done right

f64 and bool types in numerical fields

Mikkel Denkere44f34e2613c
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index 46b3f028..ee46d4f5 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -30,7 +30,7 @@ use crate::{
inverted_index::{DocAddress, WebpagePointer},
prehashed::Prehashed,
ranking::initial::{InitialScoreTweaker, Score},
- schema::{column_field, ColumnFieldEnum},
+ schema::{numerical_field, NumericalFieldEnum},
simhash,
};
@@ -122,7 +122,12 @@ pub struct TopSegmentCollector {
}
impl TopSegmentCollector {
- fn get_hash(&self, doc: DocId, field1: ColumnFieldEnum, field2: ColumnFieldEnum) -> Prehashed {
+ fn get_hash(
+ &self,
+ doc: DocId,
+ field1: NumericalFieldEnum,
+ field2: NumericalFieldEnum,
+ ) -> Prehashed {
let field_reader = self.columnfield_segment_reader.get_field_reader(doc);
let hash = [
@@ -152,7 +157,7 @@ impl TopSegmentCollector {
let simhash: Option<u64> = self
.columnfield_segment_reader
.get_field_reader(doc)
- .get(column_field::SimHash.into())
+ .get(numerical_field::SimHash.into())
.unwrap()
.into();
@@ -160,23 +165,23 @@ impl TopSegmentCollector {
hashes: Hashes {
site: self.get_hash(
doc,
- column_field::SiteHash1.into(),
- column_field::SiteHash2.into(),
+ numerical_field::SiteHash1.into(),
+ numerical_field::SiteHash2.into(),
),
title: self.get_hash(
doc,
- column_field::TitleHash1.into(),
- column_field::TitleHash2.into(),
+ numerical_field::TitleHash1.into(),
+ numerical_field::TitleHash2.into(),
),
url: self.get_hash(
doc,
- column_field::UrlHash1.into(),
- column_field::UrlHash2.into(),
+ numerical_field::UrlHash1.into(),
+ numerical_field::UrlHash2.into(),
),
url_without_tld: self.get_hash(
doc,
- column_field::UrlWithoutTldHash1.into(),
- column_field::UrlWithoutTldHash2.into(),
+ numerical_field::UrlWithoutTldHash1.into(),
+ numerical_field::UrlWithoutTldHash2.into(),
),
simhash: simhash.unwrap(),
},
diff --git a/crates/core/src/columnfield_reader.rs b/crates/core/src/columnfield_reader.rs
index 25a07865..d69d6119 100644
--- a/crates/core/src/columnfield_reader.rs
+++ b/crates/core/src/columnfield_reader.rs
@@ -20,7 +20,7 @@ use tantivy::{columnar::ColumnValues, index::SegmentId, DocId};
use crate::{
enum_map::EnumMap,
- schema::{column_field::ColumnField, ColumnFieldEnum, DataType, Field},
+ schema::{numerical_field::NumericalField, DataType, Field, NumericalFieldEnum},
};
#[derive(Default, Clone)]
@@ -52,23 +52,28 @@ impl ColumnFieldReader {
let mut u64s = EnumMap::new();
let mut bytes = EnumMap::new();
+ let mut bools = EnumMap::new();
+ let mut f64s = EnumMap::new();
- for field in Field::all().filter_map(|f| f.as_fast()) {
+ for field in Field::all().filter_map(|f| f.as_numerical()) {
match field.data_type() {
DataType::U64 => {
- let num_docs = reader.max_doc() as usize;
- let mut data = vec![0; num_docs];
- if let Ok(field_reader) = columnfield_readers.u64(field.name()) {
- for (doc, elem) in data.iter_mut().enumerate() {
- *elem = field_reader.values.get_val(doc as u32);
- }
+ if let Ok(reader) = columnfield_readers.u64(field.name()) {
+ u64s.insert(field, reader.values);
+ }
+ }
+ DataType::F64 => {
+ if let Ok(reader) = columnfield_readers.f64(field.name()) {
+ f64s.insert(field, reader.values);
+ }
+ }
+ DataType::Bool => {
+ if let Ok(reader) = columnfield_readers.bool(field.name()) {
+ bools.insert(field, reader.values);
}
-
- u64s.insert(field, data);
}
DataType::Bytes => {
- if let Some(reader) = columnfield_readers.bytes(field.name()).ok().flatten()
- {
+ if let Ok(Some(reader)) = columnfield_readers.bytes(field.name()) {
bytes.insert(field, reader);
}
}
@@ -78,7 +83,12 @@ impl ColumnFieldReader {
segments.insert(
reader.segment_id(),
Arc::new(SegmentReader {
- field_readers: AllReaders { u64s, bytes },
+ field_readers: AllReaders {
+ u64s,
+ bytes,
+ bools,
+ f64s,
+ },
}),
);
}
@@ -90,13 +100,17 @@ impl ColumnFieldReader {
}
struct AllReaders {
- u64s: EnumMap<ColumnFieldEnum, Vec<u64>>,
- bytes: EnumMap<ColumnFieldEnum, tantivy::columnar::BytesColumn>,
+ u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,
+ f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,
+ bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,
+ bytes: EnumMap<NumericalFieldEnum, tantivy::columnar::BytesColumn>,
}
pub enum Value {
U64(u64),
+ F64(f64),
Bytes(Vec<u8>),
+ Bool(bool),
}
impl Value {
@@ -107,12 +121,26 @@ impl Value {
}
}
+ pub fn as_f64(&self) -> Option<f64> {
+ match self {
+ Value::F64(val) => Some(*val),
+ _ => None,
+ }
+ }
+
pub fn as_bytes(&self) -> Option<&[u8]> {
match self {
Value::Bytes(val) => Some(val),
_ => None,
}
}
+
+ pub fn as_bool(&self) -> Option<bool> {
+ match self {
+ Value::Bool(val) => Some(*val),
+ _ => None,
+ }
+ }
}
impl From<u64> for Value {
@@ -121,12 +149,24 @@ impl From<u64> for Value {
}
}
+impl From<f64> for Value {
+ fn from(val: f64) -> Self {
+ Value::F64(val)
+ }
+}
+
impl From<Vec<u8>> for Value {
fn from(val: Vec<u8>) -> Self {
Value::Bytes(val)
}
}
+impl From<bool> for Value {
+ fn from(val: bool) -> Self {
+ Value::Bool(val)
+ }
+}
+
impl From<Value> for Option<u64> {
fn from(val: Value) -> Self {
val.as_u64()
@@ -148,22 +188,25 @@ impl From<Value> for Option<Vec<u8>> {
}
}
+impl From<Value> for Option<bool> {
+ fn from(val: Value) -> Self {
+ val.as_bool()
+ }
+}
+
pub struct FieldReader<'a> {
readers: &'a AllReaders,
doc: DocId,
}
impl<'a> FieldReader<'a> {
- pub fn get(&self, field: ColumnFieldEnum) -> Option<Value> {
+ pub fn get(&self, field: NumericalFieldEnum) -> Option<Value> {
match field.data_type() {
- DataType::U64 => Some(
- self.readers
- .u64s
- .get(field)?
- .get(self.doc as usize)
- .copied()?
- .into(),
- ),
+ DataType::U64 => Some(self.readers.u64s.get(field)?.get_val(self.doc).into()),
+
+ DataType::F64 => Some(self.readers.f64s.get(field)?.get_val(self.doc).into()),
+
+ DataType::Bool => Some(self.readers.bools.get(field)?.get_val(self.doc).into()),
DataType::Bytes => {
let reader = self.readers.bytes.get(field)?;
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 5b3aec9e..4f8043a7 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -46,7 +46,7 @@ use crate::config::SnippetConfig;
use crate::ranking::initial::Score;
use crate::schema::text_field::TextField;
-use crate::schema::{column_field, text_field, ColumnFieldEnum, Field, TextFieldEnum};
+use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};
use crate::snippet::TextSnippet;
use crate::tokenizer::{
BigramTokenizer, Identity, JsonField, Stemmed, TrigramTokenizer, UrlTokenizer,
@@ -151,9 +151,11 @@ impl InvertedIndex {
} else {
let index_settings = tantivy::IndexSettings {
sort_by_field: Some(tantivy::IndexSortByField {
- field: Field::Columnar(ColumnFieldEnum::from(column_field::PreComputedScore))
- .name()
- .to_string(),
+ field: Field::Numerical(NumericalFieldEnum::from(
+ numerical_field::PreComputedScore,
+ ))
+ .name()
+ .to_string(),
order: tantivy::Order::Desc,
}),
..Default::default()
@@ -277,7 +279,7 @@ impl From<TantivyDocument> for RetrievedWebpage {
Some(Field::Text(TextFieldEnum::Url(_))) => {
webpage.url = str_value(text_field::Url.name(), &value);
}
- Some(Field::Columnar(ColumnFieldEnum::LastUpdated(_))) => {
+ Some(Field::Numerical(NumericalFieldEnum::LastUpdated(_))) => {
webpage.updated_time = {
let timestamp = value.as_u64().unwrap() as i64;
if timestamp == 0 {
@@ -290,7 +292,7 @@ impl From<TantivyDocument> for RetrievedWebpage {
Some(Field::Text(TextFieldEnum::AllBody(_))) => {
webpage.dirty_body = str_value(text_field::AllBody.name(), &value);
}
- Some(Field::Columnar(ColumnFieldEnum::Region(_))) => {
+ Some(Field::Numerical(NumericalFieldEnum::Region(_))) => {
webpage.region = {
let id = value.as_u64().unwrap();
Region::from_id(id)
@@ -304,11 +306,11 @@ impl From<TantivyDocument> for RetrievedWebpage {
let json = str_value(text_field::SchemaOrgJson.name(), &value);
webpage.schema_org = serde_json::from_str(&json).unwrap_or_default();
}
- Some(Field::Columnar(ColumnFieldEnum::LikelyHasAds(_))) => {
- webpage.likely_has_ads = value.as_u64().unwrap_or_default() != 0;
+ Some(Field::Numerical(NumericalFieldEnum::LikelyHasAds(_))) => {
+ webpage.likely_has_ads = value.as_bool().unwrap_or_default();
}
- Some(Field::Columnar(ColumnFieldEnum::LikelyHasPaywall(_))) => {
- webpage.likely_has_paywall = value.as_u64().unwrap_or_default() != 0;
+ Some(Field::Numerical(NumericalFieldEnum::LikelyHasPaywall(_))) => {
+ webpage.likely_has_paywall = value.as_bool().unwrap_or_default();
}
Some(Field::Text(TextFieldEnum::RecipeFirstIngredientTagId(_))) => {
let tag_id = str_value(text_field::RecipeFirstIngredientTagId.name(), &value);
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 89e8cc2b..f5ec6dfc 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -32,7 +32,7 @@ use crate::query::shortcircuit::ShortCircuitQuery;
use crate::query::Query;
use crate::ranking::pipeline::LocalRecallRankingWebpage;
use crate::ranking::SignalComputer;
-use crate::schema::{column_field, text_field, ColumnFieldEnum, Field, TextFieldEnum};
+use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};
use crate::search_ctx::Ctx;
use crate::snippet;
use crate::snippet::TextSnippet;
@@ -156,7 +156,9 @@ impl InvertedIndex {
let field = self
.schema()
- .get_field(Field::Columnar(ColumnFieldEnum::from(column_field::HostNodeID)).name())
+ .get_field(
+ Field::Numerical(NumericalFieldEnum::from(numerical_field::HostNodeID)).name(),
+ )
.unwrap();
let id = doc.get_first(field).unwrap().as_u64().unwrap();
diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rs
index 459cb0b5..3f58d9a7 100644
--- a/crates/core/src/query/pattern_query/scorer.rs
+++ b/crates/core/src/query/pattern_query/scorer.rs
@@ -25,7 +25,7 @@ use tantivy::{
use crate::{
columnfield_reader::{self, ColumnFieldReader},
query::intersection::Intersection,
- schema::ColumnFieldEnum,
+ schema::NumericalFieldEnum,
};
use super::SmallPatternPart;
@@ -127,7 +127,7 @@ impl Scorer for AllScorer {
pub struct EmptyFieldScorer {
pub segment_reader: Arc<columnfield_reader::SegmentReader>,
- pub num_tokens_columnfield: ColumnFieldEnum,
+ pub num_tokens_columnfield: NumericalFieldEnum,
pub all_scorer: AllScorer,
}
@@ -210,7 +210,7 @@ pub struct NormalPatternScorer {
left: Vec<u32>,
right: Vec<u32>,
phrase_count: u32,
- num_tokens_field: ColumnFieldEnum,
+ num_tokens_field: NumericalFieldEnum,
segment_reader: Arc<columnfield_reader::SegmentReader>,
}
@@ -219,7 +219,7 @@ impl NormalPatternScorer {
term_postings_list: Vec<SegmentPostings>,
pattern: Vec<SmallPatternPart>,
segment: tantivy::index::SegmentId,
- num_tokens_field: ColumnFieldEnum,
+ num_tokens_field: NumericalFieldEnum,
columnfield_reader: ColumnFieldReader,
) -> Self {
let num_query_terms = term_postings_list.len();
diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rs
index e8f93aed..8d1cbc05 100644
--- a/crates/core/src/query/pattern_query/weight.rs
+++ b/crates/core/src/query/pattern_query/weight.rs
@@ -24,7 +24,7 @@ use tantivy::{
use crate::{
columnfield_reader::ColumnFieldReader,
schema::{
- column_field,
+ numerical_field,
text_field::{self, TextField},
Field, TextFieldEnum,
},
@@ -62,7 +62,7 @@ impl FastSiteDomainPatternWeight {
let opt = match field_no_tokenizer {
Field::Text(t) => t.record_option(),
- Field::Columnar(_) => unreachable!(),
+ Field::Numerical(_) => unreachable!(),
};
match reader
@@ -127,23 +127,27 @@ impl PatternWeight {
}
let num_tokens_columnfield = match Field::get(self.field.field_id() as usize) {
- Some(Field::Text(TextFieldEnum::Title(_))) => Ok(column_field::NumTitleTokens.into()),
+ Some(Field::Text(TextFieldEnum::Title(_))) => {
+ Ok(numerical_field::NumTitleTokens.into())
+ }
Some(Field::Text(TextFieldEnum::CleanBody(_))) => {
- Ok(column_field::NumCleanBodyTokens.into())
+ Ok(numerical_field::NumCleanBodyTokens.into())
+ }
+ Some(Field::Text(TextFieldEnum::Url(_))) => Ok(numerical_field::NumUrlTokens.into()),
+ Some(Field::Text(TextFieldEnum::Domain(_))) => {
+ Ok(numerical_field::NumDomainTokens.into())
}
- Some(Field::Text(TextFieldEnum::Url(_))) => Ok(column_field::NumUrlTokens.into()),
- Some(Field::Text(TextFieldEnum::Domain(_))) => Ok(column_field::NumDomainTokens.into()),
Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_))) => {
- Ok(column_field::NumUrlForSiteOperatorTokens.into())
+ Ok(numerical_field::NumUrlForSiteOperatorTokens.into())
}
Some(Field::Text(TextFieldEnum::Description(_))) => {
- Ok(column_field::NumDescriptionTokens.into())
+ Ok(numerical_field::NumDescriptionTokens.into())
}
Some(Field::Text(TextFieldEnum::MicroformatTags(_))) => {
- Ok(column_field::NumMicroformatTagsTokens.into())
+ Ok(numerical_field::NumMicroformatTagsTokens.into())
}
Some(Field::Text(TextFieldEnum::FlattenedSchemaOrgJson(_))) => {
- Ok(column_field::NumFlattenedSchemaTokens.into())
+ Ok(numerical_field::NumFlattenedSchemaTokens.into())
}
Some(field) => Err(TantivyError::InvalidArgument(format!(
"{} is not supported in pattern query",
diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rs
index a3c87562..d3672eb4 100644
--- a/crates/core/src/ranking/pipeline/stages/recall.rs
+++ b/crates/core/src/ranking/pipeline/stages/recall.rs
@@ -28,7 +28,7 @@ use crate::{
pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},
SignalComputer, SignalEnum,
},
- schema::column_field,
+ schema::numerical_field,
searcher::{api, SearchQuery},
webgraph,
};
@@ -160,15 +160,15 @@ impl LocalRecallRankingWebpage {
let columnfields = columnfield_reader.get_field_reader(pointer.address.doc_id);
let title_embedding: Option<Vec<u8>> = columnfields
- .get(column_field::TitleEmbeddings.into())
+ .get(numerical_field::TitleEmbeddings.into())
.and_then(|v| v.into());
let keyword_embedding: Option<Vec<u8>> = columnfields
- .get(column_field::KeywordEmbeddings.into())
+ .get(numerical_field::KeywordEmbeddings.into())
.and_then(|v| v.into());
let host_id = columnfields
- .get(column_field::HostNodeID.into())
+ .get(numerical_field::HostNodeID.into())
.unwrap()
.as_u64()
.unwrap()
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rs
index 18228b2d..84167f8e 100644
--- a/crates/core/src/ranking/signal/non_text.rs
+++ b/crates/core/src/ranking/signal/non_text.rs
@@ -18,7 +18,7 @@ use tantivy::DocId;
use super::{Signal, SignalComputer};
use crate::{
- schema::{self, Field, FLOAT_SCALING},
+ schema::{self, Field},
webpage::Webpage,
};
@@ -105,7 +105,9 @@ impl Signal for HostCentrality {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::HostCentrality.into()))
+ Some(Field::Numerical(
+ schema::numerical_field::HostCentrality.into(),
+ ))
}
fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
@@ -118,9 +120,9 @@ impl Signal for HostCentrality {
let val = columnfield_reader
.get(self.as_columnfield().unwrap())
- .and_then(|v| v.as_u64())
+ .and_then(|v| v.as_f64())
.unwrap();
- Some(val as f64 / FLOAT_SCALING as f64)
+ Some(val)
}
}
@@ -143,8 +145,8 @@ impl Signal for HostCentralityRank {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(
- schema::column_field::HostCentralityRank.into(),
+ Some(Field::Numerical(
+ schema::numerical_field::HostCentralityRank.into(),
))
}
@@ -183,7 +185,9 @@ impl Signal for PageCentrality {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::PageCentrality.into()))
+ Some(Field::Numerical(
+ schema::numerical_field::PageCentrality.into(),
+ ))
}
fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
@@ -196,9 +200,9 @@ impl Signal for PageCentrality {
let val = columnfield_reader
.get(self.as_columnfield().unwrap())
- .and_then(|v| v.as_u64())
+ .and_then(|v| v.as_f64())
.unwrap();
- Some(val as f64 / FLOAT_SCALING as f64)
+ Some(val)
}
}
@@ -221,8 +225,8 @@ impl Signal for PageCentralityRank {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(
- schema::column_field::PageCentralityRank.into(),
+ Some(Field::Numerical(
+ schema::numerical_field::PageCentralityRank.into(),
))
}
@@ -261,7 +265,7 @@ impl Signal for IsHomepage {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::IsHomepage.into()))
+ Some(Field::Numerical(schema::numerical_field::IsHomepage.into()))
}
fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
@@ -274,9 +278,14 @@ impl Signal for IsHomepage {
let val = columnfield_reader
.get(self.as_columnfield().unwrap())
- .and_then(|v| v.as_u64())
+ .and_then(|v| v.as_bool())
.unwrap();
- Some(val as f64)
+
+ if val {
+ Some(1.0)
+ } else {
+ Some(0.0)
+ }
}
}
@@ -299,7 +308,9 @@ impl Signal for FetchTimeMs {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::FetchTimeMs.into()))
+ Some(Field::Numerical(
+ schema::numerical_field::FetchTimeMs.into(),
+ ))
}
fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {
@@ -347,7 +358,9 @@ impl Signal for UpdateTimestamp {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::LastUpdated.into()))
+ Some(Field::Numerical(
+ schema::numerical_field::LastUpdated.into(),
+ ))
}
fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {
@@ -392,7 +405,9 @@ impl Signal for TrackerScore {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::TrackerScore.into()))
+ Some(Field::Numerical(
+ schema::numerical_field::TrackerScore.into(),
+ ))
}
fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
@@ -431,7 +446,7 @@ impl Signal for Region {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::Region.into()))
+ Some(Field::Numerical(schema::numerical_field::Region.into()))
}
fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {
@@ -553,8 +568,8 @@ impl Signal for UrlDigits {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(
- schema::column_field::NumPathAndQueryDigits.into(),
+ Some(Field::Numerical(
+ schema::numerical_field::NumPathAndQueryDigits.into(),
))
}
@@ -609,8 +624,8 @@ impl Signal for UrlSlashes {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(
- schema::column_field::NumPathAndQuerySlashes.into(),
+ Some(Field::Numerical(
+ schema::numerical_field::NumPathAndQuerySlashes.into(),
))
}
@@ -656,7 +671,9 @@ impl Signal for LinkDensity {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(schema::column_field::LinkDensity.into()))
+ Some(Field::Numerical(
+ schema::numerical_field::LinkDensity.into(),
+ ))
}
fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
@@ -670,8 +687,8 @@ impl Signal for LinkDensity {
let val = columnfield_reader
.get(self.as_columnfield().unwrap())
- .and_then(|v| v.as_u64())
+ .and_then(|v| v.as_f64())
.unwrap();
- Some(score_link_density(val as f64 / FLOAT_SCALING as f64))
+ Some(score_link_density(val))
}
}
diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rs
index 260e04e0..9de46936 100644
--- a/crates/core/src/ranking/signal/prelude.rs
+++ b/crates/core/src/ranking/signal/prelude.rs
@@ -18,7 +18,7 @@ use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};
use crate::schema::Field;
use crate::{
- schema::{ColumnFieldEnum, TextFieldEnum},
+ schema::{NumericalFieldEnum, TextFieldEnum},
webpage::Webpage,
};
use enum_dispatch::enum_dispatch;
@@ -46,8 +46,8 @@ pub trait Signal:
self.as_field().and_then(|field| field.as_text())
}
- fn as_columnfield(&self) -> Option<ColumnFieldEnum> {
- self.as_field().and_then(|field| field.as_fast())
+ fn as_columnfield(&self) -> Option<NumericalFieldEnum> {
+ self.as_field().and_then(|field| field.as_numerical())
}
}
diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rs
index ce1eedff..f9e9884f 100644
--- a/crates/core/src/ranking/signal/text.rs
+++ b/crates/core/src/ranking/signal/text.rs
@@ -806,8 +806,8 @@ impl Signal for TitleEmbeddingSimilarity {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(
- schema::column_field::TitleEmbeddings.into(),
+ Some(Field::Numerical(
+ schema::numerical_field::TitleEmbeddings.into(),
))
}
@@ -835,8 +835,8 @@ impl Signal for KeywordEmbeddingSimilarity {
}
fn as_field(&self) -> Option<Field> {
- Some(Field::Columnar(
- schema::column_field::KeywordEmbeddings.into(),
+ Some(Field::Numerical(
+ schema::numerical_field::KeywordEmbeddings.into(),
))
}
diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rs
index ba34fa75..aa6842e0 100644
--- a/crates/core/src/schema/mod.rs
+++ b/crates/core/src/schema/mod.rs
@@ -14,21 +14,19 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-pub mod column_field;
+pub mod numerical_field;
pub mod text_field;
use tantivy::schema::{BytesOptions, DateOptions, NumericOptions, TextOptions};
-pub use column_field::{ColumnFieldEnum, DataType};
+pub use numerical_field::{DataType, NumericalFieldEnum};
pub use text_field::TextFieldEnum;
-use self::{column_field::ColumnField, text_field::TextField};
-
-pub const FLOAT_SCALING: u64 = 1_000_000_000;
+use self::{numerical_field::NumericalField, text_field::TextField};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Field {
- Columnar(ColumnFieldEnum),
+ Numerical(NumericalFieldEnum),
Text(TextFieldEnum),
}
@@ -40,10 +38,10 @@ impl Field {
}
let field_id = field_id - TextFieldEnum::num_variants();
- if field_id < ColumnFieldEnum::num_variants() {
- return Some(Field::Columnar(ColumnFieldEnum::get(field_id).unwrap()));
+ if field_id < NumericalFieldEnum::num_variants() {
+ return Some(Field::Numerical(NumericalFieldEnum::get(field_id).unwrap()));
}
- let _field_id = field_id - ColumnFieldEnum::num_variants();
+ let _field_id = field_id - NumericalFieldEnum::num_variants();
None
}
@@ -52,12 +50,12 @@ impl Field {
pub fn all() -> impl Iterator<Item = Field> {
TextFieldEnum::all()
.map(Field::Text)
- .chain(ColumnFieldEnum::all().map(Field::Columnar))
+ .chain(NumericalFieldEnum::all().map(Field::Numerical))
}
pub fn has_pos(&self) -> bool {
match self {
- Field::Columnar(_) => false,
+ Field::Numerical(_) => false,
Field::Text(text) => text.has_pos(),
}
}
@@ -65,34 +63,34 @@ impl Field {
pub fn indexing_option(&self) -> IndexingOption {
match self {
Field::Text(f) => f.indexing_option(),
- Field::Columnar(f) => f.indexing_option(),
+ Field::Numerical(f) => f.indexing_option(),
}
}
pub fn name(&self) -> &str {
match self {
Field::Text(f) => f.name(),
- Field::Columnar(f) => f.name(),
+ Field::Numerical(f) => f.name(),
}
}
pub fn is_searchable(&self) -> bool {
match self {
Field::Text(f) => f.is_searchable(),
- Field::Columnar(_) => false,
+ Field::Numerical(_) => false,
}
}
pub fn as_text(&self) -> Option<TextFieldEnum> {
match self {
- Field::Columnar(_) => None,
+ Field::Numerical(_) => None,
Field::Text(field) => Some(*field),
}
}
- pub fn as_fast(&self) -> Option<ColumnFieldEnum> {
+ pub fn as_numerical(&self) -> Option<NumericalFieldEnum> {
match self {
- Field::Columnar(field) => Some(*field),
+ Field::Numerical(field) => Some(*field),
Field::Text(_) => None,
}
}
@@ -104,7 +102,17 @@ pub fn create_schema() -> tantivy::schema::Schema {
for field in Field::all() {
match field.indexing_option() {
IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
- IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
+ IndexingOption::Integer(options) => {
+ let fast = field.as_numerical().expect("Expected fast field");
+ match fast.data_type() {
+ DataType::U64 => builder.add_u64_field(field.name(), options),
+ DataType::F64 => builder.add_f64_field(field.name(), options),
+ DataType::Bool => builder.add_bool_field(field.name(), options),
+ DataType::Bytes => {
+ panic!("bytes field should have a `Bytes` variant as indexing option")
+ }
+ }
+ }
IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),
IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),
};
diff --git a/crates/core/src/schema/column_field.rs b/crates/core/src/schema/numerical_field.rs
similarity index 85%
rename from crates/core/src/schema/column_field.rs
rename to crates/core/src/schema/numerical_field.rs
index a31ff09b..87bff767 100644
--- a/crates/core/src/schema/column_field.rs
+++ b/crates/core/src/schema/numerical_field.rs
@@ -29,10 +29,12 @@ use crate::{
Result,
};
-use super::{IndexingOption, FLOAT_SCALING};
+use super::IndexingOption;
#[enum_dispatch]
-pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {
+pub trait NumericalField:
+ Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash
+{
fn name(&self) -> &str;
fn add_html_tantivy(
&self,
@@ -64,19 +66,30 @@ pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::ha
}
fn indexing_option(&self) -> IndexingOption {
- debug_assert!(matches!(self.data_type(), DataType::U64));
+ match self.data_type() {
+ DataType::U64 | DataType::Bool | DataType::F64 => {
+ let mut opt = NumericOptions::default().set_columnar();
- let mut opt = NumericOptions::default().set_columnar();
+ if self.is_stored() {
+ opt = opt.set_stored();
+ }
- if self.is_stored() {
- opt = opt.set_stored();
- }
+ if self.is_indexed() {
+ opt = opt.set_indexed();
+ }
- if self.is_indexed() {
- opt = opt.set_indexed();
- }
+ IndexingOption::Integer(opt)
+ }
+ DataType::Bytes => {
+ let mut opt = BytesOptions::default().set_columnar().set_indexed();
- IndexingOption::Integer(opt)
+ if self.is_stored() {
+ opt = opt.set_stored();
+ }
+
+ IndexingOption::Bytes(opt)
+ }
+ }
}
fn tantivy_field(&self, schema: &tantivy::schema::Schema) -> tantivy::schema::Field {
@@ -86,10 +99,10 @@ pub trait ColumnField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::ha
}
}
-#[enum_dispatch(ColumnField)]
+#[enum_dispatch(NumericalField)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants)]
#[strum_discriminants(derive(VariantArray))]
-pub enum ColumnFieldEnum {
+pub enum NumericalFieldEnum {
IsHomepage,
HostCentrality,
HostCentralityRank,
@@ -131,7 +144,7 @@ pub enum ColumnFieldEnum {
KeywordEmbeddings,
}
-enum_dispatch_from_discriminant!(ColumnFieldEnumDiscriminants => ColumnFieldEnum,
+enum_dispatch_from_discriminant!(NumericalFieldEnumDiscriminants => NumericalFieldEnum,
[
IsHomepage,
HostCentrality,
@@ -174,44 +187,50 @@ enum_dispatch_from_discriminant!(ColumnFieldEnumDiscriminants => ColumnFieldEnum
KeywordEmbeddings,
]);
-impl ColumnFieldEnum {
- pub fn all() -> impl Iterator<Item = ColumnFieldEnum> {
- ColumnFieldEnumDiscriminants::VARIANTS
+impl NumericalFieldEnum {
+ pub fn all() -> impl Iterator<Item = NumericalFieldEnum> {
+ NumericalFieldEnumDiscriminants::VARIANTS
.iter()
.copied()
.map(|v| v.into())
}
- pub fn get(field_id: usize) -> Option<ColumnFieldEnum> {
- ColumnFieldEnumDiscriminants::VARIANTS
+ pub fn get(field_id: usize) -> Option<NumericalFieldEnum> {
+ NumericalFieldEnumDiscriminants::VARIANTS
.get(field_id)
.copied()
- .map(ColumnFieldEnum::from)
+ .map(NumericalFieldEnum::from)
}
pub fn num_variants() -> usize {
- ColumnFieldEnumDiscriminants::VARIANTS.len()
+ NumericalFieldEnumDiscriminants::VARIANTS.len()
}
}
pub enum DataType {
U64,
+ F64,
+ Bool,
Bytes,
}
-impl InsertEnumMapKey for ColumnFieldEnum {
+impl InsertEnumMapKey for NumericalFieldEnum {
fn into_usize(self) -> usize {
- ColumnFieldEnumDiscriminants::from(self) as usize
+ NumericalFieldEnumDiscriminants::from(self) as usize
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct IsHomepage;
-impl ColumnField for IsHomepage {
+impl NumericalField for IsHomepage {
fn name(&self) -> &str {
"is_homepage"
}
+ fn data_type(&self) -> DataType {
+ DataType::Bool
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
@@ -219,7 +238,7 @@ impl ColumnField for IsHomepage {
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), (html.is_homepage()).into());
+ doc.add_bool(self.tantivy_field(schema), html.is_homepage());
Ok(())
}
@@ -227,7 +246,7 @@ impl ColumnField for IsHomepage {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct HostCentrality;
-impl ColumnField for HostCentrality {
+impl NumericalField for HostCentrality {
fn name(&self) -> &str {
"host_centrality"
}
@@ -242,16 +261,17 @@ impl ColumnField for HostCentrality {
Ok(())
}
+ fn data_type(&self) -> DataType {
+ DataType::F64
+ }
+
fn add_webpage_tantivy(
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(
- self.tantivy_field(schema),
- (webpage.host_centrality * FLOAT_SCALING as f64) as u64,
- );
+ doc.add_f64(self.tantivy_field(schema), webpage.host_centrality);
Ok(())
}
@@ -259,7 +279,7 @@ impl ColumnField for HostCentrality {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct HostCentralityRank;
-impl ColumnField for HostCentralityRank {
+impl NumericalField for HostCentralityRank {
fn name(&self) -> &str {
"host_centrality_rank"
}
@@ -288,7 +308,7 @@ impl ColumnField for HostCentralityRank {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PageCentrality;
-impl ColumnField for PageCentrality {
+impl NumericalField for PageCentrality {
fn name(&self) -> &str {
"page_centrality"
}
@@ -302,16 +322,17 @@ impl ColumnField for PageCentrality {
Ok(())
}
+ fn data_type(&self) -> DataType {
+ DataType::F64
+ }
+
fn add_webpage_tantivy(
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(
- self.tantivy_field(schema),
- (webpage.page_centrality * FLOAT_SCALING as f64) as u64,
- );
+ doc.add_f64(self.tantivy_field(schema), webpage.page_centrality);
Ok(())
}
@@ -319,7 +340,7 @@ impl ColumnField for PageCentrality {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PageCentralityRank;
-impl ColumnField for PageCentralityRank {
+impl NumericalField for PageCentralityRank {
fn name(&self) -> &str {
"page_centrality_rank"
}
@@ -347,7 +368,7 @@ impl ColumnField for PageCentralityRank {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct FetchTimeMs;
-impl ColumnField for FetchTimeMs {
+impl NumericalField for FetchTimeMs {
fn name(&self) -> &str {
"fetch_time_ms"
}
@@ -375,7 +396,7 @@ impl ColumnField for FetchTimeMs {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct LastUpdated;
-impl ColumnField for LastUpdated {
+impl NumericalField for LastUpdated {
fn name(&self) -> &str {
"last_updated"
}
@@ -403,7 +424,7 @@ impl ColumnField for LastUpdated {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct TrackerScore;
-impl ColumnField for TrackerScore {
+impl NumericalField for TrackerScore {
fn name(&self) -> &str {
"tracker_score"
}
@@ -423,7 +444,7 @@ impl ColumnField for TrackerScore {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Region;
-impl ColumnField for Region {
+impl NumericalField for Region {
fn name(&self) -> &str {
"region"
}
@@ -464,7 +485,7 @@ impl ColumnField for Region {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumUrlTokens;
-impl ColumnField for NumUrlTokens {
+impl NumericalField for NumUrlTokens {
fn name(&self) -> &str {
"num_url_tokens"
}
@@ -487,7 +508,7 @@ impl ColumnField for NumUrlTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumTitleTokens;
-impl ColumnField for NumTitleTokens {
+impl NumericalField for NumTitleTokens {
fn name(&self) -> &str {
"num_title_tokens"
}
@@ -514,7 +535,7 @@ impl ColumnField for NumTitleTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumCleanBodyTokens;
-impl ColumnField for NumCleanBodyTokens {
+impl NumericalField for NumCleanBodyTokens {
fn name(&self) -> &str {
"num_clean_body_tokens"
}
@@ -536,7 +557,7 @@ impl ColumnField for NumCleanBodyTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumDescriptionTokens;
-impl ColumnField for NumDescriptionTokens {
+impl NumericalField for NumDescriptionTokens {
fn name(&self) -> &str {
"num_description_tokens"
}
@@ -559,7 +580,7 @@ impl ColumnField for NumDescriptionTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumUrlForSiteOperatorTokens;
-impl ColumnField for NumUrlForSiteOperatorTokens {
+impl NumericalField for NumUrlForSiteOperatorTokens {
fn name(&self) -> &str {
"num_url_for_site_operator_tokens"
}
@@ -582,7 +603,7 @@ impl ColumnField for NumUrlForSiteOperatorTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumDomainTokens;
-impl ColumnField for NumDomainTokens {
+impl NumericalField for NumDomainTokens {
fn name(&self) -> &str {
"num_domain_tokens"
}
@@ -605,7 +626,7 @@ impl ColumnField for NumDomainTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumMicroformatTagsTokens;
-impl ColumnField for NumMicroformatTagsTokens {
+impl NumericalField for NumMicroformatTagsTokens {
fn name(&self) -> &str {
"num_microformat_tags_tokens"
}
@@ -628,7 +649,7 @@ impl ColumnField for NumMicroformatTagsTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SiteHash1;
-impl ColumnField for SiteHash1 {
+impl NumericalField for SiteHash1 {
fn name(&self) -> &str {
"site_hash1"
}
@@ -648,7 +669,7 @@ impl ColumnField for SiteHash1 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SiteHash2;
-impl ColumnField for SiteHash2 {
+impl NumericalField for SiteHash2 {
fn name(&self) -> &str {
"site_hash2"
}
@@ -668,7 +689,7 @@ impl ColumnField for SiteHash2 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UrlWithoutQueryHash1;
-impl ColumnField for UrlWithoutQueryHash1 {
+impl NumericalField for UrlWithoutQueryHash1 {
fn name(&self) -> &str {
"url_without_query_hash1"
}
@@ -691,7 +712,7 @@ impl ColumnField for UrlWithoutQueryHash1 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UrlWithoutQueryHash2;
-impl ColumnField for UrlWithoutQueryHash2 {
+impl NumericalField for UrlWithoutQueryHash2 {
fn name(&self) -> &str {
"url_without_query_hash2"
}
@@ -714,7 +735,7 @@ impl ColumnField for UrlWithoutQueryHash2 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct TitleHash1;
-impl ColumnField for TitleHash1 {
+impl NumericalField for TitleHash1 {
fn name(&self) -> &str {
"title_hash1"
}
@@ -734,7 +755,7 @@ impl ColumnField for TitleHash1 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct TitleHash2;
-impl ColumnField for TitleHash2 {
+impl NumericalField for TitleHash2 {
fn name(&self) -> &str {
"title_hash2"
}
@@ -754,7 +775,7 @@ impl ColumnField for TitleHash2 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UrlHash1;
-impl ColumnField for UrlHash1 {
+impl NumericalField for UrlHash1 {
fn name(&self) -> &str {
"url_hash1"
}
@@ -774,7 +795,7 @@ impl ColumnField for UrlHash1 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UrlHash2;
-impl ColumnField for UrlHash2 {
+impl NumericalField for UrlHash2 {
fn name(&self) -> &str {
"url_hash2"
}
@@ -794,7 +815,7 @@ impl ColumnField for UrlHash2 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DomainHash1;
-impl ColumnField for DomainHash1 {
+impl NumericalField for DomainHash1 {
fn name(&self) -> &str {
"domain_hash1"
}
@@ -814,7 +835,7 @@ impl ColumnField for DomainHash1 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DomainHash2;
-impl ColumnField for DomainHash2 {
+impl NumericalField for DomainHash2 {
fn name(&self) -> &str {
"domain_hash2"
}
@@ -834,7 +855,7 @@ impl ColumnField for DomainHash2 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UrlWithoutTldHash1;
-impl ColumnField for UrlWithoutTldHash1 {
+impl NumericalField for UrlWithoutTldHash1 {
fn name(&self) -> &str {
"url_without_tld_hash1"
}
@@ -854,7 +875,7 @@ impl ColumnField for UrlWithoutTldHash1 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct UrlWithoutTldHash2;
-impl ColumnField for UrlWithoutTldHash2 {
+impl NumericalField for UrlWithoutTldHash2 {
fn name(&self) -> &str {
"url_without_tld_hash2"
}
@@ -874,7 +895,7 @@ impl ColumnField for UrlWithoutTldHash2 {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PreComputedScore;
-impl ColumnField for PreComputedScore {
+impl NumericalField for PreComputedScore {
fn name(&self) -> &str {
"pre_computed_score"
}
@@ -893,16 +914,17 @@ impl ColumnField for PreComputedScore {
Ok(())
}
+ fn data_type(&self) -> DataType {
+ DataType::F64
+ }
+
fn add_webpage_tantivy(
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(
- self.tantivy_field(schema),
- (webpage.pre_computed_score * FLOAT_SCALING as f64) as u64,
- );
+ doc.add_f64(self.tantivy_field(schema), webpage.pre_computed_score);
Ok(())
}
@@ -910,7 +932,7 @@ impl ColumnField for PreComputedScore {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct HostNodeID;
-impl ColumnField for HostNodeID {
+impl NumericalField for HostNodeID {
fn name(&self) -> &str {
"host_node_id"
}
@@ -950,7 +972,7 @@ impl ColumnField for HostNodeID {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SimHash;
-impl ColumnField for SimHash {
+impl NumericalField for SimHash {
fn name(&self) -> &str {
"sim_hash"
}
@@ -981,7 +1003,7 @@ impl ColumnField for SimHash {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumFlattenedSchemaTokens;
-impl ColumnField for NumFlattenedSchemaTokens {
+impl NumericalField for NumFlattenedSchemaTokens {
fn name(&self) -> &str {
"num_flattened_schema_tokens"
}
@@ -1004,7 +1026,7 @@ impl ColumnField for NumFlattenedSchemaTokens {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumPathAndQuerySlashes;
-impl ColumnField for NumPathAndQuerySlashes {
+impl NumericalField for NumPathAndQuerySlashes {
fn name(&self) -> &str {
"num_path_and_query_slashes"
}
@@ -1034,7 +1056,7 @@ impl ColumnField for NumPathAndQuerySlashes {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NumPathAndQueryDigits;
-impl ColumnField for NumPathAndQueryDigits {
+impl NumericalField for NumPathAndQueryDigits {
fn name(&self) -> &str {
"num_path_and_query_digits"
}
@@ -1072,7 +1094,7 @@ impl ColumnField for NumPathAndQueryDigits {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct LikelyHasAds;
-impl ColumnField for LikelyHasAds {
+impl NumericalField for LikelyHasAds {
fn name(&self) -> &str {
"likely_has_ads"
}
@@ -1081,6 +1103,10 @@ impl ColumnField for LikelyHasAds {
true
}
+ fn data_type(&self) -> DataType {
+ DataType::Bool
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
@@ -1088,7 +1114,7 @@ impl ColumnField for LikelyHasAds {
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), html.likely_has_ads() as u64);
+ doc.add_bool(self.tantivy_field(schema), html.likely_has_ads());
Ok(())
}
@@ -1096,7 +1122,7 @@ impl ColumnField for LikelyHasAds {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct LikelyHasPaywall;
-impl ColumnField for LikelyHasPaywall {
+impl NumericalField for LikelyHasPaywall {
fn name(&self) -> &str {
"likely_has_paywall"
}
@@ -1105,6 +1131,10 @@ impl ColumnField for LikelyHasPaywall {
true
}
+ fn data_type(&self) -> DataType {
+ DataType::Bool
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
@@ -1112,7 +1142,7 @@ impl ColumnField for LikelyHasPaywall {
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), html.likely_has_paywall() as u64);
+ doc.add_bool(self.tantivy_field(schema), html.likely_has_paywall());
Ok(())
}
@@ -1120,7 +1150,7 @@ impl ColumnField for LikelyHasPaywall {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct LinkDensity;
-impl ColumnField for LinkDensity {
+impl NumericalField for LinkDensity {
fn name(&self) -> &str {
"link_density"
}
@@ -1129,6 +1159,10 @@ impl ColumnField for LinkDensity {
true
}
+ fn data_type(&self) -> DataType {
+ DataType::F64
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
@@ -1136,10 +1170,7 @@ impl ColumnField for LinkDensity {
doc: &mut TantivyDocument,
schema: &tantivy::schema::Schema,
) -> Result<()> {
- doc.add_u64(
- self.tantivy_field(schema),
- (html.link_density() * FLOAT_SCALING as f64) as u64,
- );
+ doc.add_f64(self.tantivy_field(schema), html.link_density());
Ok(())
}
@@ -1147,7 +1178,7 @@ impl ColumnField for LinkDensity {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct TitleEmbeddings;
-impl ColumnField for TitleEmbeddings {
+impl NumericalField for TitleEmbeddings {
fn name(&self) -> &str {
"title_embeddings"
}
@@ -1156,8 +1187,8 @@ impl ColumnField for TitleEmbeddings {
DataType::Bytes
}
- fn indexing_option(&self) -> IndexingOption {
- IndexingOption::Bytes(BytesOptions::default().set_columnar().set_stored())
+ fn is_stored(&self) -> bool {
+ true
}
fn add_html_tantivy(
@@ -1191,7 +1222,7 @@ impl ColumnField for TitleEmbeddings {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct KeywordEmbeddings;
-impl ColumnField for KeywordEmbeddings {
+impl NumericalField for KeywordEmbeddings {
fn name(&self) -> &str {
"keyword_embeddings"
}
@@ -1200,8 +1231,8 @@ impl ColumnField for KeywordEmbeddings {
DataType::Bytes
}
- fn indexing_option(&self) -> IndexingOption {
- IndexingOption::Bytes(BytesOptions::default().set_columnar().set_stored())
+ fn is_stored(&self) -> bool {
+ true
}
fn add_html_tantivy(
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index bfaa0235..bcd4a0c9 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -19,7 +19,7 @@ use crate::{
prehashed::hash,
rake::RakeModel,
schema::{
- column_field::ColumnField,
+ numerical_field::NumericalField,
text_field::{self, TextField},
TextFieldEnum,
},
@@ -201,7 +201,7 @@ impl Html {
{
match field {
Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
- Field::Columnar(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
+ Field::Numerical(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
}
}
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index 3f88b2ab..c05c6b40 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- schema::{column_field::ColumnField, text_field::TextField, Field},
+ schema::{numerical_field::NumericalField, text_field::TextField, Field},
webgraph::NodeID,
Result,
};
@@ -134,7 +134,7 @@ impl Webpage {
for field in Field::all() {
match field {
- Field::Columnar(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
+ Field::Numerical(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
}
}