Skip to content

web search done right

fastfield cache

Mikkel Denker7a65ebbe00f0
diff --git a/src/collector.rs b/src/collector.rs
index 833c4969..741ff4c3 100644
--- a/src/collector.rs
+++ b/src/collector.rs
@@ -24,7 +24,7 @@ use tantivy::{
use crate::{
inverted_index::{DocAddress, WebsitePointer},
prehashed::{combine_u64s, PrehashMap, Prehashed},
- schema::Field,
+ schema::{FastField, Field},
};
fn adjust_score(num_taken: usize, original_score: f64) -> f64 {
@@ -109,7 +109,7 @@ impl Collector for TopDocs {
let key_reader = segment.fast_fields().u64s(
segment
.schema()
- .get_field(Field::SiteHash.as_str())
+ .get_field(Field::Fast(FastField::SiteHash).name())
.unwrap(),
)?;
diff --git a/src/fastfield_cache.rs b/src/fastfield_cache.rs
new file mode 100644
index 00000000..57b39917
--- /dev/null
+++ b/src/fastfield_cache.rs
@@ -0,0 +1,143 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ collections::HashMap,
+ sync::{Arc, RwLock},
+};
+
+use tantivy::{fastfield::Column, DocId, SegmentId};
+
+use crate::schema::{DataType, FastField, Field, ALL_FIELDS};
+
+#[derive(Default)]
+pub struct FastFieldCache {
+ segments: RwLock<HashMap<SegmentId, Arc<SegmentCache>>>,
+}
+
+impl FastFieldCache {
+ pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentCache> {
+ Arc::clone(self.segments.read().unwrap().get(segment).unwrap())
+ }
+}
+
+impl tantivy::Warmer for FastFieldCache {
+ fn warm(&self, searcher: &tantivy::Searcher) -> tantivy::Result<()> {
+ let mut segments_wrt = self.segments.write().unwrap();
+
+ let mut fast_fields: Vec<_> = ALL_FIELDS
+ .into_iter()
+ .filter_map(|field| field.as_fast())
+ .collect();
+ fast_fields.sort_by_key(|field| *field as usize);
+
+ let schema = searcher.schema();
+
+ for reader in searcher.segment_readers() {
+ let fastfield_readers = reader.fast_fields();
+ let max_doc = reader.max_doc();
+
+ let mut doc_caches = Vec::new();
+
+ for field in &fast_fields {
+ let tv_field = schema.get_field(Field::Fast(*field).name()).unwrap();
+
+ let doc_cache = match field.data_type() {
+ DataType::U64 => {
+ let reader = fastfield_readers.u64(tv_field).unwrap();
+ let mut output = vec![0; max_doc as usize];
+ reader.get_range(0, &mut output);
+
+ let docs = output.into_iter().map(FieldValue::U64).collect();
+
+ DocCache { docs }
+ }
+ DataType::F64 => {
+ let reader = fastfield_readers.f64(tv_field).unwrap();
+ let mut output = vec![0.0; max_doc as usize];
+ reader.get_range(0, &mut output);
+
+ let docs = output.into_iter().map(FieldValue::F64).collect();
+ DocCache { docs }
+ }
+ DataType::U64s => {
+ let reader = fastfield_readers.u64s(tv_field).unwrap();
+ let mut docs = Vec::new();
+
+ for doc_id in 0..max_doc {
+ let mut output = Vec::new();
+ reader.get_vals(doc_id, &mut output);
+ docs.push(output);
+ }
+
+ let docs = docs.into_iter().map(FieldValue::U64s).collect();
+
+ DocCache { docs }
+ }
+ };
+
+ doc_caches.push((field, doc_cache));
+ }
+
+ doc_caches.sort_by_key(|(field, _)| **field as usize);
+
+ segments_wrt.insert(
+ reader.segment_id(),
+ Arc::new(SegmentCache {
+ doc_caches: doc_caches.into_iter().map(|(_, cache)| cache).collect(),
+ }),
+ );
+ }
+
+ Ok(())
+ }
+
+ fn garbage_collect(&self, _live_generations: &[&tantivy::SearcherGeneration]) {}
+}
+
+pub enum FieldValue {
+ U64(u64),
+ U64s(Vec<u64>),
+ F64(f64),
+}
+
+#[derive(Default)]
+pub struct DocCache {
+ docs: Vec<FieldValue>,
+}
+
+impl DocCache {
+ pub fn get(&self, doc: &DocId) -> &FieldValue {
+ self.docs.get(*doc as usize).unwrap()
+ }
+
+ pub fn get_u64(&self, doc: &DocId) -> Option<u64> {
+ match self.get(doc) {
+ FieldValue::U64(res) => Some(*res),
+ _ => None,
+ }
+ }
+}
+
+pub struct SegmentCache {
+ doc_caches: Vec<DocCache>, // fast_field -> doc_cache
+}
+
+impl SegmentCache {
+ pub fn get_doc_cache(&self, field: &FastField) -> &DocCache {
+ self.doc_caches.get(*field as usize).unwrap()
+ }
+}
diff --git a/src/index.rs b/src/index.rs
index 71c35664..22b9a2ad 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -154,6 +154,9 @@ impl Index {
self.region_count.merge(other.region_count);
+ self.subdomain_counter.merge(other.subdomain_counter);
+ drop(self.subdomain_counter);
+
Self::open(&self.path).expect("failed to open index")
}
@@ -363,7 +366,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.inverted_index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 5b8846aa..9d35a2b3 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -23,10 +23,11 @@ use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
+use crate::fastfield_cache::FastFieldCache;
use crate::image_store::Image;
use crate::prehashed::Prehashed;
use crate::query::Query;
-use crate::schema::{Field, ALL_FIELDS};
+use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
use crate::snippet;
use crate::tokenizer::Identity;
use crate::webpage::region::Region;
@@ -35,7 +36,7 @@ use crate::Result;
use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::fs;
use std::path::Path;
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
#[derive(Debug, Serialize, Deserialize)]
pub struct InitialSearchResult {
@@ -84,6 +85,7 @@ pub struct InvertedIndex {
tantivy_index: tantivy::Index,
writer: IndexWriter,
reader: IndexReader,
+ fastfield_cache: Arc<FastFieldCache>,
schema: Arc<Schema>,
}
@@ -97,7 +99,7 @@ impl InvertedIndex {
} else {
let index_settings = tantivy::IndexSettings {
sort_by_field: Some(tantivy::IndexSortByField {
- field: Field::PreComputedScore.as_str().to_string(),
+ field: Field::Fast(FastField::PreComputedScore).name().to_string(),
order: tantivy::Order::Desc,
}),
..Default::default()
@@ -130,15 +132,27 @@ impl InvertedIndex {
let merge_policy = NoMergePolicy::default();
writer.set_merge_policy(Box::new(merge_policy));
+ let fastfield_cache = Arc::new(FastFieldCache::default());
+
+ let warmers: Vec<Weak<dyn tantivy::Warmer>> = vec![Arc::downgrade(
+ &(Arc::clone(&fastfield_cache) as Arc<dyn tantivy::Warmer>),
+ )];
+ let reader = tantivy_index.reader_builder().warmers(warmers).try_into()?;
+
Ok(InvertedIndex {
writer,
- reader: tantivy_index.reader()?,
+ reader,
schema: Arc::new(schema),
path: path.as_ref().to_str().unwrap().to_string(),
+ fastfield_cache,
tantivy_index,
})
}
+ pub fn fastfield_cache(&self) -> Arc<FastFieldCache> {
+ Arc::clone(&self.fastfield_cache)
+ }
+
pub fn tokenizers(&self) -> &TokenizerManager {
self.tantivy_index.tokenizers()
}
@@ -379,21 +393,21 @@ impl From<Document> for RetrievedWebpage {
for value in doc.field_values() {
match ALL_FIELDS[value.field.field_id() as usize] {
- Field::Title => {
+ Field::Text(TextField::Title) => {
webpage.title = value
.value
.as_text()
.expect("Title field should be text")
.to_string()
}
- Field::StemmedCleanBody => {
+ Field::Text(TextField::StemmedCleanBody) => {
webpage.body = value
.value
.as_text()
.expect("Body field should be text")
.to_string()
}
- Field::Description => {
+ Field::Text(TextField::Description) => {
let desc = value
.value
.as_text()
@@ -402,14 +416,14 @@ impl From<Document> for RetrievedWebpage {
webpage.description = if desc.is_empty() { None } else { Some(desc) }
}
- Field::Url => {
+ Field::Text(TextField::Url) => {
webpage.url = value
.value
.as_text()
.expect("Url field should be text")
.to_string()
}
- Field::PrimaryImage => {
+ Field::Text(TextField::PrimaryImage) => {
webpage.primary_image = {
let bytes = value
.value
@@ -419,7 +433,7 @@ impl From<Document> for RetrievedWebpage {
bincode::deserialize(bytes).unwrap()
}
}
- Field::LastUpdated => {
+ Field::Fast(FastField::LastUpdated) => {
webpage.updated_time = {
let timestamp = value.value.as_u64().unwrap() as i64;
if timestamp == 0 {
@@ -429,44 +443,20 @@ impl From<Document> for RetrievedWebpage {
}
}
}
- Field::AllBody => {
+ Field::Text(TextField::AllBody) => {
webpage.dirty_body = value
.value
.as_text()
.expect("All body field should be text")
.to_string()
}
- Field::Region => {
+ Field::Fast(FastField::Region) => {
webpage.region = {
let id = value.value.as_u64().unwrap();
Region::from_id(id)
}
}
- Field::BacklinkText
- | Field::HostCentrality
- | Field::PageCentrality
- | Field::Site
- | Field::Domain
- | Field::SiteNoTokenizer
- | Field::DomainNoTokenizer
- | Field::DomainNameIfHomepageNoTokenizer
- | Field::StemmedTitle
- | Field::CleanBody
- | Field::DomainIfHomepage
- | Field::TitleIfHomepage
- | Field::IsHomepage
- | Field::SiteHash
- | Field::UrlWithoutQueryHash
- | Field::TitleHash
- | Field::UrlHash
- | Field::DomainHash
- | Field::NumTrackers
- | Field::PreComputedScore
- | Field::NumCleanBodyTokens
- | Field::NumDescriptionTokens
- | Field::NumTitleTokens
- | Field::NumUrlTokens
- | Field::FetchTimeMs => {}
+ _ => {}
}
}
@@ -497,7 +487,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
@@ -542,7 +536,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
index
.insert(Webpage::new(
@@ -580,7 +578,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
index
.insert(Webpage::new(
@@ -618,7 +620,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
index
.insert(Webpage::new(
@@ -656,7 +662,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
index
.insert(Webpage::new(
@@ -728,7 +738,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
for _ in 0..100 {
index
@@ -768,7 +782,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
index
.insert(Webpage::new(
@@ -848,7 +866,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
@@ -869,7 +891,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
@@ -907,7 +933,11 @@ mod tests {
#[test]
fn only_show_primary_images_when_relevant() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let mut webpage = Webpage::new(
&format!(
@@ -985,7 +1015,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
diff --git a/src/lib.rs b/src/lib.rs
index 70b572ff..42868b36 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -38,6 +38,7 @@ mod collector;
mod directory;
mod entity_index;
mod exponential_backoff;
+mod fastfield_cache;
mod frontend;
mod image_downloader;
mod image_store;
diff --git a/src/query/mod.rs b/src/query/mod.rs
index ea9520d9..7ae521b5 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -14,7 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{ranking::goggles::Goggle, ranking::SignalAggregator, schema::Field, Result};
+use crate::{
+ ranking::goggles::Goggle,
+ ranking::SignalAggregator,
+ schema::{Field, TextField},
+ Result,
+};
use std::{collections::HashMap, sync::Arc};
use tantivy::{
query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},
@@ -44,10 +49,13 @@ fn proximity_queries(
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();
- let proxmity_fields = [Field::Title, Field::CleanBody];
+ let proxmity_fields = [
+ Field::Text(TextField::Title),
+ Field::Text(TextField::CleanBody),
+ ];
for field in &proxmity_fields {
- let tantivy_field = schema.get_field(field.as_str()).unwrap();
+ let tantivy_field = schema.get_field(field.name()).unwrap();
let tantivy_entry = schema.get_field_entry(tantivy_field);
for (boost, slop) in [(6, 0), (5, 1), (4, 2), (3, 4), (2, 16), (1, 32)] {
@@ -296,7 +304,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
index
.insert(Webpage::new(
@@ -381,7 +393,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -396,7 +412,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -411,7 +431,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -463,7 +487,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -515,7 +543,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -597,7 +629,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -611,7 +647,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -626,7 +666,11 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
diff --git a/src/query/parser.rs b/src/query/parser.rs
index 56714e1f..f2d7c4a3 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -23,7 +23,7 @@ use tantivy::{
use crate::{
bangs::BANG_PREFIX,
ranking::FieldBoost,
- schema::{Field, ALL_FIELDS},
+ schema::{Field, TextField, ALL_FIELDS},
};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -59,7 +59,12 @@ fn simple_into_tantivy(
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
let (backlink_field, backlink_field_entry) = fields
.iter()
- .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText))
+ .find(|(field, _)| {
+ matches!(
+ ALL_FIELDS[field.field_id() as usize],
+ Field::Text(TextField::BacklinkText)
+ )
+ })
.unwrap();
vec![
@@ -115,7 +120,10 @@ impl Term {
let (field, entry) = fields
.iter()
.find(|(field, _)| {
- matches!(ALL_FIELDS[field.field_id() as usize], Field::Title)
+ matches!(
+ ALL_FIELDS[field.field_id() as usize],
+ Field::Text(TextField::Title)
+ )
})
.unwrap();
vec![(
@@ -127,7 +135,10 @@ impl Term {
let (field, entry) = fields
.iter()
.find(|(field, _)| {
- matches!(ALL_FIELDS[field.field_id() as usize], Field::AllBody)
+ matches!(
+ ALL_FIELDS[field.field_id() as usize],
+ Field::Text(TextField::AllBody)
+ )
})
.unwrap();
vec![(
@@ -138,7 +149,12 @@ impl Term {
Term::Url(url) => {
let (field, entry) = fields
.iter()
- .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::Url))
+ .find(|(field, _)| {
+ matches!(
+ ALL_FIELDS[field.field_id() as usize],
+ Field::Text(TextField::Url)
+ )
+ })
.unwrap();
vec![(
Occur::Must,
@@ -185,7 +201,8 @@ impl Term {
.filter(|(field, _)| {
matches!(
ALL_FIELDS[field.field_id() as usize],
- Field::DomainNoTokenizer | Field::SiteNoTokenizer
+ Field::Text(TextField::DomainNoTokenizer)
+ | Field::Text(TextField::SiteNoTokenizer)
)
})
.into_iter()
@@ -218,7 +235,8 @@ impl Term {
))
};
- let boost = field_boost.get(&ALL_FIELDS[field.field_id() as usize]) as f32;
+ let boost =
+ field_boost.get(&ALL_FIELDS[field.field_id() as usize].as_text().unwrap()) as f32;
Box::new(BoostQuery::new(processed_query, boost))
}
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index d10cc10f..b3662a67 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -22,7 +22,11 @@ mod pattern_query;
use std::convert::TryFrom;
-use crate::{query::union::UnionQuery, schema::Field, Result};
+use crate::{
+ query::union::UnionQuery,
+ schema::{Field, TextField},
+ Result,
+};
use itertools::Itertools;
use tantivy::{
query::{BooleanQuery, BoostQuery, Occur, QueryClone, TermQuery},
@@ -203,10 +207,18 @@ impl Instruction {
for option in &self.options {
match option {
PatternOption::Site(site) if field.is_none() => {
- field = Some(schema.get_field(Field::Site.as_str()).unwrap());
-
- let domain_field = schema.get_field(Field::DomainNoTokenizer.as_str()).unwrap();
- let site_field = schema.get_field(Field::SiteNoTokenizer.as_str()).unwrap();
+ field = Some(
+ schema
+ .get_field(Field::Text(TextField::Site).name())
+ .unwrap(),
+ );
+
+ let domain_field = schema
+ .get_field(Field::Text(TextField::DomainNoTokenizer).name())
+ .unwrap();
+ let site_field = schema
+ .get_field(Field::Text(TextField::SiteNoTokenizer).name())
+ .unwrap();
subqueries.push((
Occur::Must,
@@ -218,16 +230,32 @@ impl Instruction {
));
}
PatternOption::InUrl if field.is_none() => {
- field = Some(schema.get_field(Field::Url.as_str()).unwrap())
+ field = Some(
+ schema
+ .get_field(Field::Text(TextField::Url).name())
+ .unwrap(),
+ )
}
PatternOption::InTitle if field.is_none() => {
- field = Some(schema.get_field(Field::Title.as_str()).unwrap())
+ field = Some(
+ schema
+ .get_field(Field::Text(TextField::Title).name())
+ .unwrap(),
+ )
}
PatternOption::InDescription if field.is_none() => {
- field = Some(schema.get_field(Field::Description.as_str()).unwrap())
+ field = Some(
+ schema
+ .get_field(Field::Text(TextField::Description).name())
+ .unwrap(),
+ )
}
PatternOption::InContent if field.is_none() => {
- field = Some(schema.get_field(Field::CleanBody.as_str()).unwrap())
+ field = Some(
+ schema
+ .get_field(Field::Text(TextField::CleanBody).name())
+ .unwrap(),
+ )
}
PatternOption::Action(pattern_action) if action.is_none() => {
action = Some(*pattern_action)
@@ -237,7 +265,11 @@ impl Instruction {
}
let action = action.unwrap_or(Action::Boost(1));
- let field = field.unwrap_or_else(|| schema.get_field(Field::Url.as_str()).unwrap());
+ let field = field.unwrap_or_else(|| {
+ schema
+ .get_field(Field::Text(TextField::Url).name())
+ .unwrap()
+ });
if !self.patterns.is_empty() {
let query = self.pattern_query(field);
diff --git a/src/ranking/goggles/pattern_query.rs b/src/ranking/goggles/pattern_query.rs
index c35d450e..2aeec39c 100644
--- a/src/ranking/goggles/pattern_query.rs
+++ b/src/ranking/goggles/pattern_query.rs
@@ -27,7 +27,7 @@ use tantivy::{
use crate::{
query::intersection::Intersection,
ranking::bm25::Bm25Weight,
- schema::{Field, ALL_FIELDS},
+ schema::{FastField, Field, TextField, ALL_FIELDS},
tokenizer,
};
@@ -149,33 +149,33 @@ impl PatternWeight {
.collect();
let num_tokens_reader = match &ALL_FIELDS[self.field.field_id() as usize] {
- Field::Title => reader.fast_fields().u64(
+ Field::Text(TextField::Title) => reader.fast_fields().u64(
reader
.schema()
- .get_field(Field::NumTitleTokens.as_str())
+ .get_field(Field::Fast(FastField::NumTitleTokens).name())
.unwrap(),
),
- Field::CleanBody => reader.fast_fields().u64(
+ Field::Text(TextField::CleanBody) => reader.fast_fields().u64(
reader
.schema()
- .get_field(Field::NumCleanBodyTokens.as_str())
+ .get_field(Field::Fast(FastField::NumCleanBodyTokens).name())
.unwrap(),
),
- Field::Url => reader.fast_fields().u64(
+ Field::Text(TextField::Url) => reader.fast_fields().u64(
reader
.schema()
- .get_field(Field::NumUrlTokens.as_str())
+ .get_field(Field::Fast(FastField::NumUrlTokens).name())
.unwrap(),
),
- Field::Description => reader.fast_fields().u64(
+ Field::Text(TextField::Description) => reader.fast_fields().u64(
reader
.schema()
- .get_field(Field::NumDescriptionTokens.as_str())
+ .get_field(Field::Fast(FastField::NumDescriptionTokens).name())
.unwrap(),
),
field => Err(TantivyError::InvalidArgument(format!(
"{} is not supported in pattern query",
- field.as_str()
+ field.name()
))),
}?;
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 2ec08a5a..838e3065 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -16,6 +16,7 @@
use std::sync::Arc;
+use crate::fastfield_cache::FastFieldCache;
use crate::webpage::region::{Region, RegionCount};
use chrono::Utc;
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
@@ -26,6 +27,7 @@ use super::SignalAggregator;
pub(crate) struct InitialScoreTweaker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
+ fastfield_cache: Arc<FastFieldCache>,
aggregator: SignalAggregator,
}
@@ -34,11 +36,13 @@ impl InitialScoreTweaker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
aggregator: SignalAggregator,
+ fastfield_cache: Arc<FastFieldCache>,
) -> Self {
Self {
region_count,
selected_region,
aggregator,
+ fastfield_cache,
}
}
}
@@ -54,8 +58,13 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
type Child = InitialSegmentScoreTweaker;
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
- let mut aggregator = SignalAggregator::new_like(&self.aggregator);
- aggregator.register_readers(segment_reader);
+ let mut aggregator = self.aggregator.clone();
+
+ let fastfield_cache_segment = self
+ .fastfield_cache
+ .get_segment(&segment_reader.segment_id());
+
+ aggregator.register_segment(fastfield_cache_segment);
let current_timestamp = Utc::now().timestamp() as usize;
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index d34a5776..eda19d72 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -28,6 +28,7 @@ use tantivy::collector::Collector;
use crate::{
collector::{MaxDocsConsidered, TopDocs},
+ fastfield_cache::FastFieldCache,
inverted_index,
searcher::NUM_RESULTS_PER_PAGE,
webpage::region::{Region, RegionCount},
@@ -41,11 +42,16 @@ pub struct Ranker {
max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
aggregator: SignalAggregator,
+ fastfield_cache: Arc<FastFieldCache>,
de_rank_similar: bool,
}
impl Ranker {
- pub fn new(region_count: RegionCount, aggregator: SignalAggregator) -> Self {
+ pub fn new(
+ region_count: RegionCount,
+ aggregator: SignalAggregator,
+ fastfield_cache: Arc<FastFieldCache>,
+ ) -> Self {
Ranker {
region_count: Arc::new(region_count),
selected_region: None,
@@ -53,6 +59,7 @@ impl Ranker {
aggregator,
max_docs: None,
de_rank_similar: true,
+ fastfield_cache,
}
}
@@ -82,7 +89,8 @@ impl Ranker {
let score_tweaker = InitialScoreTweaker::new(
Arc::clone(&self.region_count),
self.selected_region,
- SignalAggregator::new_like(&self.aggregator),
+ self.aggregator.clone(),
+ Arc::clone(&self.fastfield_cache),
);
let mut collector = TopDocs::with_limit(NUM_RESULTS_PER_PAGE);
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index 6e0f2ddb..f22ae4ee 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -14,17 +14,19 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{webpage::Webpage, Result};
+use crate::{
+ fastfield_cache,
+ schema::{FastField, TextField},
+ webpage::Webpage,
+ Result,
+};
use std::{array, convert::TryFrom, ops::Deref, sync::Arc};
use chrono::Utc;
-use tantivy::{
- fastfield::{Column, DynamicFastFieldReader},
- DocId, Score, SegmentReader,
-};
+use tantivy::{DocId, Score};
use crate::{
- schema::{Field, ALL_FIELDS, CENTRALITY_SCALING},
+ schema::{Field, CENTRALITY_SCALING},
webpage::region::{Region, RegionCount},
};
@@ -42,7 +44,7 @@ pub enum Signal {
Region,
}
-const ALL_SIGNALS: [Signal; 8] = [
+pub const ALL_SIGNALS: [Signal; 8] = [
Signal::Bm25,
Signal::HostCentrality,
Signal::PageCentrality,
@@ -54,23 +56,6 @@ const ALL_SIGNALS: [Signal; 8] = [
];
impl Signal {
- fn from_field(field: &Field) -> Option<Self> {
- match field {
- Field::IsHomepage => Some(Signal::IsHomepage),
- Field::HostCentrality => Some(Signal::HostCentrality),
- Field::PageCentrality => Some(Signal::PageCentrality),
- Field::FetchTimeMs => Some(Signal::FetchTimeMs),
- Field::LastUpdated => Some(Signal::UpdateTimestamp),
- Field::NumTrackers => Some(Signal::NumTrackers),
- Field::Region => Some(Signal::Region),
- _ => None,
- }
- }
-
- fn has_fast_reader(&self) -> bool {
- !matches!(self, Signal::Bm25)
- }
-
fn is_computable_before_search(&self) -> bool {
!matches!(self, Signal::Bm25)
}
@@ -153,18 +138,19 @@ impl Signal {
_ => None,
}
}
-}
-
-fn fastfield_reader(segment_reader: &SegmentReader, field: &Field) -> DynamicFastFieldReader<u64> {
- let tv_field = segment_reader
- .schema()
- .get_field(field.as_str())
- .unwrap_or_else(|| panic!("Faild to load {} field", field.as_str()));
- segment_reader
- .fast_fields()
- .u64(tv_field)
- .unwrap_or_else(|_| panic!("Failed to get {} fast-field reader", field.as_str()))
+ fn as_fastfield(&self) -> Option<FastField> {
+ match self {
+ Signal::Bm25 => None,
+ Signal::HostCentrality => Some(FastField::HostCentrality),
+ Signal::PageCentrality => Some(FastField::PageCentrality),
+ Signal::IsHomepage => Some(FastField::IsHomepage),
+ Signal::FetchTimeMs => Some(FastField::FetchTimeMs),
+ Signal::UpdateTimestamp => Some(FastField::LastUpdated),
+ Signal::NumTrackers => Some(FastField::NumTrackers),
+ Signal::Region => Some(FastField::Region),
+ }
+ }
}
#[derive(Debug, Clone)]
@@ -182,34 +168,34 @@ impl SignalCoefficient {
.unwrap_or_else(|| signal.default_coefficient())
}
- pub fn new(scores: impl Iterator<Item = (Signal, f64)>) -> Self {
- let mut fast_scores = Vec::new();
+ pub fn new(coefficients: impl Iterator<Item = (Signal, f64)>) -> Self {
+ let mut fast_coefficients = Vec::new();
- for (signal, score) in scores {
+ for (signal, coefficient) in coefficients {
let idx = signal as usize;
- while idx >= fast_scores.len() {
- fast_scores.push(None);
+ while idx >= fast_coefficients.len() {
+ fast_coefficients.push(None);
}
- fast_scores[idx] = Some(score);
+ fast_coefficients[idx] = Some(coefficient);
}
- Self(fast_scores)
+ Self(fast_coefficients)
}
}
impl FieldBoost {
- pub fn get(&self, field: &Field) -> f64 {
+ pub fn get(&self, field: &TextField) -> f64 {
self.0
.get((*field) as usize)
.copied()
.flatten()
- .or_else(|| field.boost().map(|s| s as f64))
+ .or_else(|| Field::Text(*field).boost().map(|s| s as f64))
.unwrap_or(1.0)
}
- pub fn new(scores: impl Iterator<Item = (Field, f64)>) -> Self {
+ pub fn new(scores: impl Iterator<Item = (TextField, f64)>) -> Self {
let mut fast_scores = Vec::new();
for (field, score) in scores {
@@ -228,7 +214,7 @@ impl FieldBoost {
#[derive(Clone)]
pub struct SignalAggregator {
- readers: Vec<Option<DynamicFastFieldReader<u64>>>,
+ fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,
signal_coefficients: SignalCoefficient,
field_boost: FieldBoost,
fetch_time_ms_cache: [f64; 1000],
@@ -253,7 +239,7 @@ impl Default for SignalAggregator {
impl SignalAggregator {
pub fn new(
coefficients: impl Iterator<Item = (Signal, f64)>,
- boosts: impl Iterator<Item = (Field, f64)>,
+ boosts: impl Iterator<Item = (TextField, f64)>,
) -> Self {
let signal_coefficients = SignalCoefficient::new(coefficients);
let field_boost = FieldBoost::new(boosts);
@@ -265,7 +251,7 @@ impl SignalAggregator {
.collect();
Self {
- readers: Vec::new(),
+ fastfield_cache: None,
signal_coefficients,
field_boost,
fetch_time_ms_cache,
@@ -273,32 +259,8 @@ impl SignalAggregator {
}
}
- pub fn new_like(other: &SignalAggregator) -> Self {
- Self {
- readers: Vec::new(),
- signal_coefficients: other.signal_coefficients.clone(),
- field_boost: other.field_boost.clone(),
- fetch_time_ms_cache: other.fetch_time_ms_cache,
- update_time_cache: other.update_time_cache.clone(),
- }
- }
-
- pub fn register_readers(&mut self, segment_reader: &SegmentReader) {
- self.readers.clear();
-
- for field in &ALL_FIELDS {
- if let Some(signal) = Signal::from_field(field) {
- if signal.has_fast_reader() {
- let idx = signal as usize;
-
- while idx >= self.readers.len() {
- self.readers.push(None);
- }
-
- self.readers[idx] = Some(fastfield_reader(segment_reader, field))
- }
- }
- }
+ pub fn register_segment(&mut self, cache: Arc<fastfield_cache::SegmentCache>) {
+ self.fastfield_cache = Some(cache);
}
pub fn score(
@@ -312,10 +274,11 @@ impl SignalAggregator {
ALL_SIGNALS
.into_iter()
.map(|signal| {
- let fastfield_value = match self.readers.get(signal as usize) {
- Some(Some(reader)) => Some(reader.get_val(doc as u64)),
- _ => None,
- };
+ let fastfield_value = signal.as_fastfield().and_then(|field| {
+ self.fastfield_cache
+ .as_ref()
+ .and_then(|cache| cache.get_doc_cache(&field).get_u64(&doc))
+ });
self.coefficients().get(&signal)
* signal.value(
@@ -410,8 +373,10 @@ impl TryFrom<Vec<RawAlteration>> for SignalAggregator {
}
}
Target::Field(name) => {
- if let Some(field) = Field::from_string(name) {
- boosts.push((field, alteration.score));
+ if let Some(field) = Field::from_name(name) {
+ if let Some(text_field) = field.as_text() {
+ boosts.push((text_field, alteration.score));
+ }
}
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 47c2a535..e2b9e544 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -22,8 +22,8 @@ use crate::tokenizer::{Identity, Normal, Stemmed};
pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum Field {
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum TextField {
Title,
CleanBody,
StemmedTitle,
@@ -40,14 +40,18 @@ pub enum Field {
DomainNameIfHomepageNoTokenizer,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
TitleIfHomepage,
- IsHomepage,
BacklinkText,
+ PrimaryImage,
+ Description,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum FastField {
+ IsHomepage,
HostCentrality,
PageCentrality,
FetchTimeMs,
- PrimaryImage,
LastUpdated,
- Description,
NumTrackers,
Region,
NumUrlTokens,
@@ -62,40 +66,47 @@ pub enum Field {
PreComputedScore,
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Field {
+ Fast(FastField),
+ Text(TextField),
+}
+
pub static ALL_FIELDS: [Field; 33] = [
- Field::Title,
- Field::CleanBody,
- Field::StemmedTitle,
- Field::StemmedCleanBody,
- Field::AllBody,
- Field::Url,
- Field::Site,
- Field::Domain,
- Field::SiteNoTokenizer,
- Field::DomainNoTokenizer,
- Field::DomainIfHomepage,
- Field::DomainNameIfHomepageNoTokenizer,
- Field::TitleIfHomepage,
- Field::IsHomepage,
- Field::BacklinkText,
- Field::HostCentrality,
- Field::PageCentrality,
- Field::FetchTimeMs,
- Field::PrimaryImage,
- Field::LastUpdated,
- Field::Description,
- Field::NumTrackers,
- Field::Region,
- Field::NumUrlTokens,
- Field::NumTitleTokens,
- Field::NumCleanBodyTokens,
- Field::NumDescriptionTokens,
- Field::SiteHash,
- Field::UrlWithoutQueryHash,
- Field::TitleHash,
- Field::UrlHash,
- Field::DomainHash,
- Field::PreComputedScore,
+ Field::Text(TextField::Title),
+ Field::Text(TextField::CleanBody),
+ Field::Text(TextField::StemmedTitle),
+ Field::Text(TextField::StemmedCleanBody),
+ Field::Text(TextField::AllBody),
+ Field::Text(TextField::Url),
+ Field::Text(TextField::Site),
+ Field::Text(TextField::Domain),
+ Field::Text(TextField::SiteNoTokenizer),
+ Field::Text(TextField::DomainNoTokenizer),
+ Field::Text(TextField::DomainIfHomepage),
+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer),
+ Field::Text(TextField::TitleIfHomepage),
+ Field::Text(TextField::BacklinkText),
+ Field::Text(TextField::PrimaryImage),
+ Field::Text(TextField::Description),
+ // FAST FIELDS
+ Field::Fast(FastField::IsHomepage),
+ Field::Fast(FastField::HostCentrality),
+ Field::Fast(FastField::PageCentrality),
+ Field::Fast(FastField::FetchTimeMs),
+ Field::Fast(FastField::LastUpdated),
+ Field::Fast(FastField::NumTrackers),
+ Field::Fast(FastField::Region),
+ Field::Fast(FastField::NumUrlTokens),
+ Field::Fast(FastField::NumTitleTokens),
+ Field::Fast(FastField::NumCleanBodyTokens),
+ Field::Fast(FastField::NumDescriptionTokens),
+ Field::Fast(FastField::SiteHash),
+ Field::Fast(FastField::UrlWithoutQueryHash),
+ Field::Fast(FastField::TitleHash),
+ Field::Fast(FastField::UrlHash),
+ Field::Fast(FastField::DomainHash),
+ Field::Fast(FastField::PreComputedScore),
];
impl Field {
@@ -116,106 +127,122 @@ impl Field {
pub fn options(&self) -> IndexingOption {
match self {
- Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::CleanBody => IndexingOption::Text(self.default_text_options()),
- Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::Site => IndexingOption::Text(self.default_text_options()),
- Field::Domain => IndexingOption::Text(self.default_text_options()),
- Field::SiteNoTokenizer => {
+ Field::Text(TextField::Title) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
+ Field::Text(TextField::CleanBody) => IndexingOption::Text(self.default_text_options()),
+ Field::Text(TextField::Url) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
+ Field::Text(TextField::Site) => IndexingOption::Text(self.default_text_options()),
+ Field::Text(TextField::Domain) => IndexingOption::Text(self.default_text_options()),
+ Field::Text(TextField::SiteNoTokenizer) => {
IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))
}
- Field::DomainNoTokenizer => {
+ Field::Text(TextField::DomainNoTokenizer) => {
IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))
}
- Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
- Field::TitleIfHomepage => IndexingOption::Text(self.default_text_options()),
- Field::DomainNameIfHomepageNoTokenizer => {
+ Field::Text(TextField::AllBody) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
+ Field::Text(TextField::DomainIfHomepage) => {
+ IndexingOption::Text(self.default_text_options())
+ }
+ Field::Text(TextField::TitleIfHomepage) => {
+ IndexingOption::Text(self.default_text_options())
+ }
+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))
}
- Field::IsHomepage => IndexingOption::Integer(
+ Field::Text(TextField::BacklinkText) => {
+ IndexingOption::Text(self.default_text_options())
+ }
+ Field::Text(TextField::StemmedTitle) => {
+ IndexingOption::Text(self.default_text_options_with_tokenizer(Stemmed::as_str()))
+ }
+ Field::Text(TextField::StemmedCleanBody) => IndexingOption::Text(
+ self.default_text_options_with_tokenizer(Stemmed::as_str())
+ .set_stored(),
+ ),
+ Field::Text(TextField::PrimaryImage) => {
+ IndexingOption::Bytes(BytesOptions::default().set_stored())
+ }
+ Field::Text(TextField::Description) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
+ Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
- Field::HostCentrality => IndexingOption::Integer(
+ Field::Fast(FastField::HostCentrality) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::PageCentrality => IndexingOption::Integer(
+ Field::Fast(FastField::PageCentrality) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::StemmedTitle => {
- IndexingOption::Text(self.default_text_options_with_tokenizer(Stemmed::as_str()))
- }
- Field::StemmedCleanBody => IndexingOption::Text(
- self.default_text_options_with_tokenizer(Stemmed::as_str())
- .set_stored(),
- ),
- Field::FetchTimeMs => IndexingOption::Integer(
+ Field::Fast(FastField::FetchTimeMs) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumTrackers => IndexingOption::Integer(
+ Field::Fast(FastField::NumTrackers) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::PrimaryImage => IndexingOption::Bytes(BytesOptions::default().set_stored()),
- Field::LastUpdated => IndexingOption::Integer(
+ Field::Fast(FastField::LastUpdated) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_stored()
.set_indexed(),
),
- Field::Description => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::Region => IndexingOption::Integer(
+ Field::Fast(FastField::Region) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_stored()
.set_indexed(),
),
- Field::NumCleanBodyTokens => IndexingOption::Integer(
+ Field::Fast(FastField::NumCleanBodyTokens) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumDescriptionTokens => IndexingOption::Integer(
+ Field::Fast(FastField::NumDescriptionTokens) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumTitleTokens => IndexingOption::Integer(
+ Field::Fast(FastField::NumTitleTokens) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumUrlTokens => IndexingOption::Integer(
+ Field::Fast(FastField::NumUrlTokens) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::SiteHash => IndexingOption::Integer(
+ Field::Fast(FastField::SiteHash) => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
- Field::UrlWithoutQueryHash => IndexingOption::Integer(
+ Field::Fast(FastField::UrlWithoutQueryHash) => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
- Field::UrlHash => IndexingOption::Integer(
+ Field::Fast(FastField::UrlHash) => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
- Field::DomainHash => IndexingOption::Integer(
+ Field::Fast(FastField::DomainHash) => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
- Field::TitleHash => IndexingOption::Integer(
+ Field::Fast(FastField::TitleHash) => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
- Field::PreComputedScore => IndexingOption::Float(
+ Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed()
@@ -224,139 +251,128 @@ impl Field {
}
}
- pub fn as_str(&self) -> &str {
+ pub fn name(&self) -> &str {
match self {
- Field::Title => "title",
- Field::CleanBody => "body",
- Field::Url => "url",
- Field::Site => "site",
- Field::Domain => "domain",
- Field::SiteNoTokenizer => "site_no_tokenizer",
- Field::DomainNoTokenizer => "domain_no_tokenizer",
- Field::BacklinkText => "backlink_text",
- Field::HostCentrality => "host_centrality",
- Field::PageCentrality => "page_centrality",
- Field::StemmedTitle => "stemmed_title",
- Field::StemmedCleanBody => "stemmed_body",
- Field::DomainIfHomepage => "domain_if_homepage",
- Field::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",
- Field::IsHomepage => "is_homepage",
- Field::FetchTimeMs => "fetch_time_ms",
- Field::PrimaryImage => "primary_image_uuid",
- Field::LastUpdated => "last_updated",
- Field::Description => "description",
- Field::AllBody => "all_body",
- Field::NumTrackers => "num_trackers",
- Field::Region => "region",
- Field::NumUrlTokens => "num_url_tokens",
- Field::NumTitleTokens => "num_title_tokens",
- Field::NumCleanBodyTokens => "num_clean_body_tokens",
- Field::NumDescriptionTokens => "num_description_tokens",
- Field::SiteHash => "site_hash",
- Field::UrlWithoutQueryHash => "url_without_query_hash",
- Field::PreComputedScore => "pre_computed_score",
- Field::TitleIfHomepage => "title_if_homepage",
- Field::TitleHash => "title_hash",
- Field::UrlHash => "url_hash",
- Field::DomainHash => "domain_hash",
+ Field::Text(TextField::Title) => "title",
+ Field::Text(TextField::CleanBody) => "body",
+ Field::Text(TextField::Url) => "url",
+ Field::Text(TextField::Site) => "site",
+ Field::Text(TextField::Domain) => "domain",
+ Field::Text(TextField::SiteNoTokenizer) => "site_no_tokenizer",
+ Field::Text(TextField::DomainNoTokenizer) => "domain_no_tokenizer",
+ Field::Text(TextField::BacklinkText) => "backlink_text",
+ Field::Text(TextField::StemmedTitle) => "stemmed_title",
+ Field::Text(TextField::StemmedCleanBody) => "stemmed_body",
+ Field::Text(TextField::DomainIfHomepage) => "domain_if_homepage",
+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
+ "domain_name_if_homepage_no_tokenizer"
+ }
+ Field::Text(TextField::Description) => "description",
+ Field::Text(TextField::PrimaryImage) => "primary_image_uuid",
+ Field::Text(TextField::TitleIfHomepage) => "title_if_homepage",
+ Field::Text(TextField::AllBody) => "all_body",
+ Field::Fast(FastField::HostCentrality) => "host_centrality",
+ Field::Fast(FastField::PageCentrality) => "page_centrality",
+ Field::Fast(FastField::IsHomepage) => "is_homepage",
+ Field::Fast(FastField::FetchTimeMs) => "fetch_time_ms",
+ Field::Fast(FastField::LastUpdated) => "last_updated",
+ Field::Fast(FastField::NumTrackers) => "num_trackers",
+ Field::Fast(FastField::Region) => "region",
+ Field::Fast(FastField::NumUrlTokens) => "num_url_tokens",
+ Field::Fast(FastField::NumTitleTokens) => "num_title_tokens",
+ Field::Fast(FastField::NumCleanBodyTokens) => "num_clean_body_tokens",
+ Field::Fast(FastField::NumDescriptionTokens) => "num_description_tokens",
+ Field::Fast(FastField::SiteHash) => "site_hash",
+ Field::Fast(FastField::UrlWithoutQueryHash) => "url_without_query_hash",
+ Field::Fast(FastField::PreComputedScore) => "pre_computed_score",
+ Field::Fast(FastField::TitleHash) => "title_hash",
+ Field::Fast(FastField::UrlHash) => "url_hash",
+ Field::Fast(FastField::DomainHash) => "domain_hash",
}
}
pub fn boost(&self) -> Option<f32> {
match self {
- Field::Site => Some(3.0),
- Field::TitleIfHomepage => Some(2.0),
- Field::DomainIfHomepage => Some(10.0),
- Field::DomainNameIfHomepageNoTokenizer => Some(30.0),
- Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),
- Field::CleanBody => Some(4.0),
- Field::Title => Some(10.0),
- Field::Url => Some(1.0),
- Field::Domain => Some(1.0),
- Field::AllBody => Some(0.01),
- Field::BacklinkText => Some(4.0),
- Field::HostCentrality
- | Field::PageCentrality
- | Field::SiteHash
- | Field::TitleHash
- | Field::UrlHash
- | Field::UrlWithoutQueryHash
- | Field::SiteNoTokenizer
- | Field::DomainNoTokenizer
- | Field::IsHomepage
- | Field::PrimaryImage
- | Field::FetchTimeMs
- | Field::Description
- | Field::NumTrackers
- | Field::NumUrlTokens
- | Field::DomainHash
- | Field::NumTitleTokens
- | Field::NumCleanBodyTokens
- | Field::NumDescriptionTokens
- | Field::Region
- | Field::PreComputedScore
- | Field::LastUpdated => None,
+ Field::Text(TextField::Site) => Some(3.0),
+ Field::Text(TextField::TitleIfHomepage) => Some(2.0),
+ Field::Text(TextField::DomainIfHomepage) => Some(10.0),
+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => Some(30.0),
+ Field::Text(TextField::StemmedCleanBody) | Field::Text(TextField::StemmedTitle) => {
+ Some(0.1)
+ }
+ Field::Text(TextField::CleanBody) => Some(4.0),
+ Field::Text(TextField::Title) => Some(10.0),
+ Field::Text(TextField::Url) => Some(1.0),
+ Field::Text(TextField::Domain) => Some(1.0),
+ Field::Text(TextField::AllBody) => Some(0.01),
+ Field::Text(TextField::BacklinkText) => Some(4.0),
+ Field::Text(TextField::SiteNoTokenizer)
+ | Field::Text(TextField::DomainNoTokenizer)
+ | Field::Text(TextField::Description)
+ | Field::Text(TextField::PrimaryImage) => None,
+ Field::Fast(_) => None,
}
}
pub fn is_searchable(&self) -> bool {
- !matches!(self, Field::PrimaryImage | Field::BacklinkText) && !self.is_fast()
+ !matches!(
+ self,
+ Field::Text(TextField::PrimaryImage) | Field::Text(TextField::BacklinkText)
+ ) && !self.is_fast()
}
pub fn is_fast(&self) -> bool {
- matches!(
- self,
- Field::IsHomepage
- | Field::HostCentrality
- | Field::FetchTimeMs
- | Field::LastUpdated
- | Field::NumTrackers
- | Field::Region
- | Field::NumUrlTokens
- | Field::NumTitleTokens
- | Field::NumCleanBodyTokens
- | Field::NumDescriptionTokens
- | Field::SiteHash
- | Field::UrlWithoutQueryHash
- | Field::TitleHash
- | Field::UrlHash
- | Field::DomainHash
- | Field::PreComputedScore
- )
+ matches!(self, Field::Fast(_))
}
- pub fn from_string(name: String) -> Option<Field> {
+ pub fn from_name(name: String) -> Option<Field> {
match name.as_str() {
- "title" => Some(Field::Title),
- "body" => Some(Field::CleanBody),
- "url" => Some(Field::Url),
- "site" => Some(Field::Site),
- "backlink_text" => Some(Field::BacklinkText),
- "host_centrality" => Some(Field::HostCentrality),
- "page_centrality" => Some(Field::PageCentrality),
- "stemmed_title" => Some(Field::StemmedTitle),
- "stemmed_body" => Some(Field::StemmedCleanBody),
- "domain" => Some(Field::Domain),
- "domain_if_homepage" => Some(Field::DomainIfHomepage),
- "is_homepage" => Some(Field::IsHomepage),
- "fetch_time_ms" => Some(Field::FetchTimeMs),
- "primary_image_uuid" => Some(Field::PrimaryImage),
- "last_updated" => Some(Field::LastUpdated),
- "description" => Some(Field::Description),
- "all_body" => Some(Field::AllBody),
- "num_trackers" => Some(Field::NumTrackers),
- "region" => Some(Field::Region),
- "site_hash" => Some(Field::SiteHash),
- "url_without_query_hash" => Some(Field::UrlWithoutQueryHash),
- "domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),
- "pre_computed_score" => Some(Field::PreComputedScore),
- "title_if_homepage" => Some(Field::TitleIfHomepage),
- "url_hash" => Some(Field::UrlHash),
- "domain_hash" => Some(Field::DomainHash),
- "title_hash" => Some(Field::TitleHash),
+ "title" => Some(Field::Text(TextField::Title)),
+ "body" => Some(Field::Text(TextField::CleanBody)),
+ "url" => Some(Field::Text(TextField::Url)),
+ "site" => Some(Field::Text(TextField::Site)),
+ "backlink_text" => Some(Field::Text(TextField::BacklinkText)),
+ "stemmed_title" => Some(Field::Text(TextField::StemmedTitle)),
+ "stemmed_body" => Some(Field::Text(TextField::StemmedCleanBody)),
+ "domain" => Some(Field::Text(TextField::Domain)),
+ "domain_if_homepage" => Some(Field::Text(TextField::DomainIfHomepage)),
+ "primary_image_uuid" => Some(Field::Text(TextField::PrimaryImage)),
+ "domain_name_if_homepage_no_tokenizer" => {
+ Some(Field::Text(TextField::DomainNameIfHomepageNoTokenizer))
+ }
+ "description" => Some(Field::Text(TextField::Description)),
+ "all_body" => Some(Field::Text(TextField::AllBody)),
+ "title_if_homepage" => Some(Field::Text(TextField::TitleIfHomepage)),
+ "host_centrality" => Some(Field::Fast(FastField::HostCentrality)),
+ "page_centrality" => Some(Field::Fast(FastField::PageCentrality)),
+ "is_homepage" => Some(Field::Fast(FastField::IsHomepage)),
+ "fetch_time_ms" => Some(Field::Fast(FastField::FetchTimeMs)),
+ "last_updated" => Some(Field::Fast(FastField::LastUpdated)),
+ "num_trackers" => Some(Field::Fast(FastField::NumTrackers)),
+ "region" => Some(Field::Fast(FastField::Region)),
+ "site_hash" => Some(Field::Fast(FastField::SiteHash)),
+ "url_without_query_hash" => Some(Field::Fast(FastField::UrlWithoutQueryHash)),
+ "pre_computed_score" => Some(Field::Fast(FastField::PreComputedScore)),
+ "url_hash" => Some(Field::Fast(FastField::UrlHash)),
+ "domain_hash" => Some(Field::Fast(FastField::DomainHash)),
+ "title_hash" => Some(Field::Fast(FastField::TitleHash)),
_ => None,
}
}
+
+ pub fn as_text(&self) -> Option<TextField> {
+ match self {
+ Field::Fast(_) => None,
+ Field::Text(field) => Some(*field),
+ }
+ }
+
+ pub fn as_fast(&self) -> Option<FastField> {
+ match self {
+ Field::Fast(field) => Some(*field),
+ Field::Text(_) => None,
+ }
+ }
}
pub fn create_schema() -> tantivy::schema::Schema {
@@ -364,10 +380,10 @@ pub fn create_schema() -> tantivy::schema::Schema {
for field in &ALL_FIELDS {
match field.options() {
- IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),
- IndexingOption::Integer(options) => builder.add_u64_field(field.as_str(), options),
- IndexingOption::Float(options) => builder.add_f64_field(field.as_str(), options),
- IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),
+ IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
+ IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
+ IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),
+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),
};
}
@@ -380,3 +396,33 @@ pub enum IndexingOption {
Float(tantivy::schema::NumericOptions),
Bytes(tantivy::schema::BytesOptions),
}
+
+pub enum DataType {
+ U64,
+ U64s,
+ F64,
+}
+
+impl FastField {
+ pub fn data_type(&self) -> DataType {
+ match self {
+ FastField::IsHomepage => DataType::U64,
+ FastField::HostCentrality => DataType::U64,
+ FastField::PageCentrality => DataType::U64,
+ FastField::FetchTimeMs => DataType::U64,
+ FastField::LastUpdated => DataType::U64,
+ FastField::NumTrackers => DataType::U64,
+ FastField::Region => DataType::U64,
+ FastField::NumUrlTokens => DataType::U64,
+ FastField::NumTitleTokens => DataType::U64,
+ FastField::NumCleanBodyTokens => DataType::U64,
+ FastField::NumDescriptionTokens => DataType::U64,
+ FastField::SiteHash => DataType::U64s,
+ FastField::UrlWithoutQueryHash => DataType::U64s,
+ FastField::TitleHash => DataType::U64s,
+ FastField::UrlHash => DataType::U64s,
+ FastField::DomainHash => DataType::U64s,
+ FastField::PreComputedScore => DataType::F64,
+ }
+ }
+}
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index 609ab31f..00133172 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -107,6 +107,7 @@ impl LocalSearcher {
let mut ranker = Ranker::new(
self.index.region_count.clone(),
goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),
+ self.index.inverted_index.fastfield_cache(),
);
if let Some(skip_pages) = query.skip_pages {
diff --git a/src/snippet.rs b/src/snippet.rs
index 28027490..b197a821 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::schema::TextField;
use crate::search_prettifier::html_escape;
use crate::tokenizer::Stemmed;
use crate::webpage::region::Region;
@@ -203,12 +204,12 @@ impl SnippetGenerator {
) -> crate::Result<SnippetGenerator> {
let clean_field = searcher
.schema()
- .get_field(Field::CleanBody.as_str())
+ .get_field(Field::Text(TextField::CleanBody).name())
.unwrap();
let stemmed_field = searcher
.schema()
- .get_field(Field::StemmedCleanBody.as_str())
+ .get_field(Field::Text(TextField::StemmedCleanBody).name())
.unwrap();
let mut terms: BTreeSet<&tantivy::Term> = BTreeSet::new();
diff --git a/src/subdomain_count.rs b/src/subdomain_count.rs
index add016f1..591575fb 100644
--- a/src/subdomain_count.rs
+++ b/src/subdomain_count.rs
@@ -46,4 +46,8 @@ impl SubdomainCounter {
pub fn commit(&self) {
self.inner.flush();
}
+
+ pub fn merge(&mut self, other: Self) {
+ todo!();
+ }
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index aed42b6c..59e75976 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
prehashed::{hash, split_u128},
+ schema::{FastField, TextField},
schema_org::SchemaOrg,
tokenizer, Error, Result,
};
@@ -153,14 +154,14 @@ impl Webpage {
if let Ok(region) = region {
doc.add_u64(
schema
- .get_field(Field::Region.as_str())
+ .get_field(Field::Fast(FastField::Region).name())
.expect("Failed to get region field"),
region.id() as u64,
);
} else {
doc.add_u64(
schema
- .get_field(Field::Region.as_str())
+ .get_field(Field::Fast(FastField::Region).name())
.expect("Failed to get region field"),
Region::All.id() as u64,
);
@@ -174,35 +175,35 @@ impl Webpage {
doc.add_text(
schema
- .get_field(Field::BacklinkText.as_str())
+ .get_field(Field::Text(TextField::BacklinkText).name())
.expect("Failed to get backlink-text field"),
backlink_text,
);
doc.add_u64(
schema
- .get_field(Field::HostCentrality.as_str())
+ .get_field(Field::Fast(FastField::HostCentrality).name())
.expect("Failed to get host_centrality field"),
(self.host_centrality * CENTRALITY_SCALING as f64) as u64,
);
doc.add_u64(
schema
- .get_field(Field::PageCentrality.as_str())
+ .get_field(Field::Fast(FastField::PageCentrality).name())
.expect("Failed to get page_centrality field"),
(self.page_centrality * CENTRALITY_SCALING as f64) as u64,
);
doc.add_u64(
schema
- .get_field(Field::FetchTimeMs.as_str())
+ .get_field(Field::Fast(FastField::FetchTimeMs).name())
.expect("Failed to get fetch_time_ms field"),
self.fetch_time_ms,
);
doc.add_f64(
schema
- .get_field(Field::PreComputedScore.as_str())
+ .get_field(Field::Fast(FastField::PreComputedScore).name())
.expect("failed to get pre_computed_score field"),
self.pre_computed_score,
);
@@ -210,7 +211,7 @@ impl Webpage {
let image = bincode::serialize(&self.primary_image).unwrap();
doc.add_bytes(
schema
- .get_field(Field::PrimaryImage.as_str())
+ .get_field(Field::Text(TextField::PrimaryImage).name())
.expect("Failed to get primary_image field"),
image,
);
@@ -531,12 +532,14 @@ impl Html {
for field in &ALL_FIELDS {
let tantivy_field = schema
- .get_field(field.as_str())
- .unwrap_or_else(|| panic!("Unknown field: {}", field.as_str()));
+ .get_field(field.name())
+ .unwrap_or_else(|| panic!("Unknown field: {}", field.name()));
match field {
- Field::Title => doc.add_pre_tokenized_text(tantivy_field, title.clone()),
- Field::StemmedTitle => {
+ Field::Text(TextField::Title) => {
+ doc.add_pre_tokenized_text(tantivy_field, title.clone())
+ }
+ Field::Text(TextField::StemmedTitle) => {
let mut tokens = title.tokens.clone();
stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
@@ -548,8 +551,10 @@ impl Html {
},
);
}
- Field::CleanBody => doc.add_pre_tokenized_text(tantivy_field, clean_text.clone()),
- Field::StemmedCleanBody => {
+ Field::Text(TextField::CleanBody) => {
+ doc.add_pre_tokenized_text(tantivy_field, clean_text.clone())
+ }
+ Field::Text(TextField::StemmedCleanBody) => {
let mut tokens = clean_text.tokens.clone();
stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
@@ -561,13 +566,15 @@ impl Html {
},
);
}
- Field::Description => {
+ Field::Text(TextField::Description) => {
doc.add_pre_tokenized_text(tantivy_field, description.clone());
}
- Field::Url => doc.add_pre_tokenized_text(tantivy_field, url.clone()),
- Field::Site => doc.add_text(tantivy_field, self.url().site()),
- Field::Domain => doc.add_text(tantivy_field, self.url().domain()),
- Field::SiteNoTokenizer => doc.add_pre_tokenized_text(
+ Field::Text(TextField::Url) => {
+ doc.add_pre_tokenized_text(tantivy_field, url.clone())
+ }
+ Field::Text(TextField::Site) => doc.add_text(tantivy_field, self.url().site()),
+ Field::Text(TextField::Domain) => doc.add_text(tantivy_field, self.url().domain()),
+ Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {
text: self.url().site().to_string(),
@@ -580,7 +587,7 @@ impl Html {
}],
},
),
- Field::DomainNoTokenizer => doc.add_pre_tokenized_text(
+ Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {
text: self.url().domain().to_string(),
@@ -593,21 +600,21 @@ impl Html {
}],
},
),
- Field::TitleIfHomepage => {
+ Field::Text(TextField::TitleIfHomepage) => {
if self.url().is_homepage() {
doc.add_pre_tokenized_text(tantivy_field, title.clone());
} else {
doc.add_text(tantivy_field, "");
}
}
- Field::DomainIfHomepage => {
+ Field::Text(TextField::DomainIfHomepage) => {
if self.url().is_homepage() {
doc.add_text(tantivy_field, self.url().domain());
} else {
doc.add_text(tantivy_field, "");
}
}
- Field::DomainNameIfHomepageNoTokenizer => {
+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
if self.url().is_homepage() {
doc.add_pre_tokenized_text(
tantivy_field,
@@ -626,61 +633,69 @@ impl Html {
doc.add_text(tantivy_field, "");
}
}
- Field::IsHomepage => {
+ Field::Text(TextField::AllBody) => {
+ doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
+ }
+ Field::Fast(FastField::IsHomepage) => {
doc.add_u64(tantivy_field, self.url().is_homepage().into());
}
- Field::LastUpdated => doc.add_u64(
+ Field::Fast(FastField::LastUpdated) => doc.add_u64(
tantivy_field,
self.updated_time()
.map_or(0, |time| time.timestamp().max(0) as u64),
),
- Field::AllBody => doc.add_pre_tokenized_text(tantivy_field, all_text.clone()),
- Field::NumTrackers => doc.add_u64(tantivy_field, self.trackers().len() as u64),
- Field::NumUrlTokens => doc.add_u64(tantivy_field, url.tokens.len() as u64),
- Field::NumTitleTokens => doc.add_u64(tantivy_field, title.tokens.len() as u64),
- Field::NumCleanBodyTokens => {
+ Field::Fast(FastField::NumTrackers) => {
+ doc.add_u64(tantivy_field, self.trackers().len() as u64)
+ }
+ Field::Fast(FastField::NumUrlTokens) => {
+ doc.add_u64(tantivy_field, url.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumTitleTokens) => {
+ doc.add_u64(tantivy_field, title.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumCleanBodyTokens) => {
doc.add_u64(tantivy_field, clean_text.tokens.len() as u64)
}
- Field::NumDescriptionTokens => {
+ Field::Fast(FastField::NumDescriptionTokens) => {
doc.add_u64(tantivy_field, description.tokens.len() as u64)
}
- Field::SiteHash => {
+ Field::Fast(FastField::SiteHash) => {
let hash = hash(self.url().site()).0;
let u64s = split_u128(hash);
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
- Field::UrlWithoutQueryHash => {
+ Field::Fast(FastField::UrlWithoutQueryHash) => {
let hash = hash(self.url().without_query()).0;
let u64s = split_u128(hash);
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
- Field::UrlHash => {
+ Field::Fast(FastField::UrlHash) => {
let hash = hash(self.url().full()).0;
let u64s = split_u128(hash);
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
- Field::DomainHash => {
+ Field::Fast(FastField::DomainHash) => {
let hash = hash(self.url().domain()).0;
let u64s = split_u128(hash);
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
- Field::TitleHash => {
+ Field::Fast(FastField::TitleHash) => {
let hash = hash(self.title().unwrap_or_default()).0;
let u64s = split_u128(hash);
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
- Field::BacklinkText
- | Field::HostCentrality
- | Field::PageCentrality
- | Field::FetchTimeMs
- | Field::PreComputedScore
- | Field::Region
- | Field::PrimaryImage => {}
+ Field::Text(TextField::BacklinkText)
+ | Field::Fast(FastField::HostCentrality)
+ | Field::Fast(FastField::PageCentrality)
+ | Field::Fast(FastField::FetchTimeMs)
+ | Field::Fast(FastField::PreComputedScore)
+ | Field::Fast(FastField::Region)
+ | Field::Text(TextField::PrimaryImage) => {}
}
}