web search done right
fastfield cache
Mikkel Denker7a65ebbe00f0
diff --git a/src/collector.rs b/src/collector.rsindex 833c4969..741ff4c3 100644--- a/src/collector.rs+++ b/src/collector.rs@@ -24,7 +24,7 @@ use tantivy::{use crate::{inverted_index::{DocAddress, WebsitePointer},prehashed::{combine_u64s, PrehashMap, Prehashed},- schema::Field,+ schema::{FastField, Field},};fn adjust_score(num_taken: usize, original_score: f64) -> f64 {@@ -109,7 +109,7 @@ impl Collector for TopDocs {let key_reader = segment.fast_fields().u64s(segment.schema()- .get_field(Field::SiteHash.as_str())+ .get_field(Field::Fast(FastField::SiteHash).name()).unwrap(),)?;diff --git a/src/fastfield_cache.rs b/src/fastfield_cache.rsnew file mode 100644index 00000000..57b39917--- /dev/null+++ b/src/fastfield_cache.rs@@ -0,0 +1,143 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ collections::HashMap,+ sync::{Arc, RwLock},+};++use tantivy::{fastfield::Column, DocId, SegmentId};++use crate::schema::{DataType, FastField, Field, ALL_FIELDS};++#[derive(Default)]+pub struct FastFieldCache {+ segments: RwLock<HashMap<SegmentId, Arc<SegmentCache>>>,+}++impl FastFieldCache {+ pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentCache> {+ Arc::clone(self.segments.read().unwrap().get(segment).unwrap())+ }+}++impl tantivy::Warmer for FastFieldCache {+ fn warm(&self, searcher: &tantivy::Searcher) -> tantivy::Result<()> {+ let mut segments_wrt = self.segments.write().unwrap();++ let mut fast_fields: Vec<_> = ALL_FIELDS+ .into_iter()+ .filter_map(|field| field.as_fast())+ .collect();+ fast_fields.sort_by_key(|field| *field as usize);++ let schema = searcher.schema();++ for reader in searcher.segment_readers() {+ let fastfield_readers = reader.fast_fields();+ let max_doc = reader.max_doc();++ let mut doc_caches = Vec::new();++ for field in &fast_fields {+ let tv_field = schema.get_field(Field::Fast(*field).name()).unwrap();++ let doc_cache = match field.data_type() {+ DataType::U64 => {+ let reader = fastfield_readers.u64(tv_field).unwrap();+ let mut output = vec![0; max_doc as usize];+ reader.get_range(0, &mut output);++ let docs = output.into_iter().map(FieldValue::U64).collect();++ DocCache { docs }+ }+ DataType::F64 => {+ let reader = fastfield_readers.f64(tv_field).unwrap();+ let mut output = vec![0.0; max_doc as usize];+ reader.get_range(0, &mut output);++ let docs = output.into_iter().map(FieldValue::F64).collect();+ DocCache { docs }+ }+ DataType::U64s => {+ let reader = fastfield_readers.u64s(tv_field).unwrap();+ let mut docs = Vec::new();++ for doc_id in 0..max_doc {+ let mut output = Vec::new();+ reader.get_vals(doc_id, &mut output);+ docs.push(output);+ }++ let docs = docs.into_iter().map(FieldValue::U64s).collect();++ DocCache { docs }+ }+ };++ doc_caches.push((field, doc_cache));+ }++ doc_caches.sort_by_key(|(field, _)| **field as usize);++ segments_wrt.insert(+ reader.segment_id(),+ Arc::new(SegmentCache {+ doc_caches: doc_caches.into_iter().map(|(_, cache)| cache).collect(),+ }),+ );+ }++ Ok(())+ }++ fn garbage_collect(&self, _live_generations: &[&tantivy::SearcherGeneration]) {}+}++pub enum FieldValue {+ U64(u64),+ U64s(Vec<u64>),+ F64(f64),+}++#[derive(Default)]+pub struct DocCache {+ docs: Vec<FieldValue>,+}++impl DocCache {+ pub fn get(&self, doc: &DocId) -> &FieldValue {+ self.docs.get(*doc as usize).unwrap()+ }++ pub fn get_u64(&self, doc: &DocId) -> Option<u64> {+ match self.get(doc) {+ FieldValue::U64(res) => Some(*res),+ _ => None,+ }+ }+}++pub struct SegmentCache {+ doc_caches: Vec<DocCache>, // fast_field -> doc_cache+}++impl SegmentCache {+ pub fn get_doc_cache(&self, field: &FastField) -> &DocCache {+ self.doc_caches.get(*field as usize).unwrap()+ }+}diff --git a/src/index.rs b/src/index.rsindex 71c35664..22b9a2ad 100644--- a/src/index.rs+++ b/src/index.rs@@ -154,6 +154,9 @@ impl Index {self.region_count.merge(other.region_count);+ self.subdomain_counter.merge(other.subdomain_counter);+ drop(self.subdomain_counter);+Self::open(&self.path).expect("failed to open index")}@@ -363,7 +366,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.inverted_index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector())diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 5b8846aa..9d35a2b3 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -23,10 +23,11 @@ use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};+use crate::fastfield_cache::FastFieldCache;use crate::image_store::Image;use crate::prehashed::Prehashed;use crate::query::Query;-use crate::schema::{Field, ALL_FIELDS};+use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::snippet;use crate::tokenizer::Identity;use crate::webpage::region::Region;@@ -35,7 +36,7 @@ use crate::Result;use crate::{schema::create_schema, tokenizer::Tokenizer};use std::fs;use std::path::Path;-use std::sync::Arc;+use std::sync::{Arc, Weak};#[derive(Debug, Serialize, Deserialize)]pub struct InitialSearchResult {@@ -84,6 +85,7 @@ pub struct InvertedIndex {tantivy_index: tantivy::Index,writer: IndexWriter,reader: IndexReader,+ fastfield_cache: Arc<FastFieldCache>,schema: Arc<Schema>,}@@ -97,7 +99,7 @@ impl InvertedIndex {} else {let index_settings = tantivy::IndexSettings {sort_by_field: Some(tantivy::IndexSortByField {- field: Field::PreComputedScore.as_str().to_string(),+ field: Field::Fast(FastField::PreComputedScore).name().to_string(),order: tantivy::Order::Desc,}),..Default::default()@@ -130,15 +132,27 @@ impl InvertedIndex {let merge_policy = NoMergePolicy::default();writer.set_merge_policy(Box::new(merge_policy));+ let fastfield_cache = Arc::new(FastFieldCache::default());++ let warmers: Vec<Weak<dyn tantivy::Warmer>> = vec![Arc::downgrade(+ &(Arc::clone(&fastfield_cache) as Arc<dyn tantivy::Warmer>),+ )];+ let reader = tantivy_index.reader_builder().warmers(warmers).try_into()?;+Ok(InvertedIndex {writer,- reader: tantivy_index.reader()?,+ reader,schema: Arc::new(schema),path: path.as_ref().to_str().unwrap().to_string(),+ fastfield_cache,tantivy_index,})}+ pub fn fastfield_cache(&self) -> Arc<FastFieldCache> {+ Arc::clone(&self.fastfield_cache)+ }+pub fn tokenizers(&self) -> &TokenizerManager {self.tantivy_index.tokenizers()}@@ -379,21 +393,21 @@ impl From<Document> for RetrievedWebpage {for value in doc.field_values() {match ALL_FIELDS[value.field.field_id() as usize] {- Field::Title => {+ Field::Text(TextField::Title) => {webpage.title = value.value.as_text().expect("Title field should be text").to_string()}- Field::StemmedCleanBody => {+ Field::Text(TextField::StemmedCleanBody) => {webpage.body = value.value.as_text().expect("Body field should be text").to_string()}- Field::Description => {+ Field::Text(TextField::Description) => {let desc = value.value.as_text()@@ -402,14 +416,14 @@ impl From<Document> for RetrievedWebpage {webpage.description = if desc.is_empty() { None } else { Some(desc) }}- Field::Url => {+ Field::Text(TextField::Url) => {webpage.url = value.value.as_text().expect("Url field should be text").to_string()}- Field::PrimaryImage => {+ Field::Text(TextField::PrimaryImage) => {webpage.primary_image = {let bytes = value.value@@ -419,7 +433,7 @@ impl From<Document> for RetrievedWebpage {bincode::deserialize(bytes).unwrap()}}- Field::LastUpdated => {+ Field::Fast(FastField::LastUpdated) => {webpage.updated_time = {let timestamp = value.value.as_u64().unwrap() as i64;if timestamp == 0 {@@ -429,44 +443,20 @@ impl From<Document> for RetrievedWebpage {}}}- Field::AllBody => {+ Field::Text(TextField::AllBody) => {webpage.dirty_body = value.value.as_text().expect("All body field should be text").to_string()}- Field::Region => {+ Field::Fast(FastField::Region) => {webpage.region = {let id = value.value.as_u64().unwrap();Region::from_id(id)}}- Field::BacklinkText- | Field::HostCentrality- | Field::PageCentrality- | Field::Site- | Field::Domain- | Field::SiteNoTokenizer- | Field::DomainNoTokenizer- | Field::DomainNameIfHomepageNoTokenizer- | Field::StemmedTitle- | Field::CleanBody- | Field::DomainIfHomepage- | Field::TitleIfHomepage- | Field::IsHomepage- | Field::SiteHash- | Field::UrlWithoutQueryHash- | Field::TitleHash- | Field::UrlHash- | Field::DomainHash- | Field::NumTrackers- | Field::PreComputedScore- | Field::NumCleanBodyTokens- | Field::NumDescriptionTokens- | Field::NumTitleTokens- | Field::NumUrlTokens- | Field::FetchTimeMs => {}+ _ => {}}}@@ -497,7 +487,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector())@@ -542,7 +536,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );index.insert(Webpage::new(@@ -580,7 +578,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );index.insert(Webpage::new(@@ -618,7 +620,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );index.insert(Webpage::new(@@ -656,7 +662,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );index.insert(Webpage::new(@@ -728,7 +738,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );for _ in 0..100 {index@@ -768,7 +782,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );index.insert(Webpage::new(@@ -848,7 +866,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector())@@ -869,7 +891,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector())@@ -907,7 +933,11 @@ mod tests {#[test]fn only_show_primary_images_when_relevant() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let mut webpage = Webpage::new(&format!(@@ -985,7 +1015,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector())diff --git a/src/lib.rs b/src/lib.rsindex 70b572ff..42868b36 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -38,6 +38,7 @@ mod collector;mod directory;mod entity_index;mod exponential_backoff;+mod fastfield_cache;mod frontend;mod image_downloader;mod image_store;diff --git a/src/query/mod.rs b/src/query/mod.rsindex ea9520d9..7ae521b5 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -14,7 +14,12 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{ranking::goggles::Goggle, ranking::SignalAggregator, schema::Field, Result};+use crate::{+ ranking::goggles::Goggle,+ ranking::SignalAggregator,+ schema::{Field, TextField},+ Result,+};use std::{collections::HashMap, sync::Arc};use tantivy::{query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},@@ -44,10 +49,13 @@ fn proximity_queries() -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();- let proxmity_fields = [Field::Title, Field::CleanBody];+ let proxmity_fields = [+ Field::Text(TextField::Title),+ Field::Text(TextField::CleanBody),+ ];for field in &proxmity_fields {- let tantivy_field = schema.get_field(field.as_str()).unwrap();+ let tantivy_field = schema.get_field(field.name()).unwrap();let tantivy_entry = schema.get_field_entry(tantivy_field);for (boost, slop) in [(6, 0), (5, 1), (4, 2), (3, 4), (2, 16), (1, 32)] {@@ -296,7 +304,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );index.insert(Webpage::new(@@ -381,7 +393,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -396,7 +412,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -411,7 +431,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -463,7 +487,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -515,7 +543,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -597,7 +629,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -611,7 +647,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -626,7 +666,11 @@ mod tests {&SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );let result = index.search(&query, ranker.collector()).expect("Search failed");diff --git a/src/query/parser.rs b/src/query/parser.rsindex 56714e1f..f2d7c4a3 100644--- a/src/query/parser.rs+++ b/src/query/parser.rs@@ -23,7 +23,7 @@ use tantivy::{use crate::{bangs::BANG_PREFIX,ranking::FieldBoost,- schema::{Field, ALL_FIELDS},+ schema::{Field, TextField, ALL_FIELDS},};#[derive(Debug, Clone, PartialEq, Eq, Hash)]@@ -59,7 +59,12 @@ fn simple_into_tantivy() -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {let (backlink_field, backlink_field_entry) = fields.iter()- .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText))+ .find(|(field, _)| {+ matches!(+ ALL_FIELDS[field.field_id() as usize],+ Field::Text(TextField::BacklinkText)+ )+ }).unwrap();vec![@@ -115,7 +120,10 @@ impl Term {let (field, entry) = fields.iter().find(|(field, _)| {- matches!(ALL_FIELDS[field.field_id() as usize], Field::Title)+ matches!(+ ALL_FIELDS[field.field_id() as usize],+ Field::Text(TextField::Title)+ )}).unwrap();vec![(@@ -127,7 +135,10 @@ impl Term {let (field, entry) = fields.iter().find(|(field, _)| {- matches!(ALL_FIELDS[field.field_id() as usize], Field::AllBody)+ matches!(+ ALL_FIELDS[field.field_id() as usize],+ Field::Text(TextField::AllBody)+ )}).unwrap();vec![(@@ -138,7 +149,12 @@ impl Term {Term::Url(url) => {let (field, entry) = fields.iter()- .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::Url))+ .find(|(field, _)| {+ matches!(+ ALL_FIELDS[field.field_id() as usize],+ Field::Text(TextField::Url)+ )+ }).unwrap();vec![(Occur::Must,@@ -185,7 +201,8 @@ impl Term {.filter(|(field, _)| {matches!(ALL_FIELDS[field.field_id() as usize],- Field::DomainNoTokenizer | Field::SiteNoTokenizer+ Field::Text(TextField::DomainNoTokenizer)+ | Field::Text(TextField::SiteNoTokenizer))}).into_iter()@@ -218,7 +235,8 @@ impl Term {))};- let boost = field_boost.get(&ALL_FIELDS[field.field_id() as usize]) as f32;+ let boost =+ field_boost.get(&ALL_FIELDS[field.field_id() as usize].as_text().unwrap()) as f32;Box::new(BoostQuery::new(processed_query, boost))}diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex d10cc10f..b3662a67 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -22,7 +22,11 @@ mod pattern_query;use std::convert::TryFrom;-use crate::{query::union::UnionQuery, schema::Field, Result};+use crate::{+ query::union::UnionQuery,+ schema::{Field, TextField},+ Result,+};use itertools::Itertools;use tantivy::{query::{BooleanQuery, BoostQuery, Occur, QueryClone, TermQuery},@@ -203,10 +207,18 @@ impl Instruction {for option in &self.options {match option {PatternOption::Site(site) if field.is_none() => {- field = Some(schema.get_field(Field::Site.as_str()).unwrap());-- let domain_field = schema.get_field(Field::DomainNoTokenizer.as_str()).unwrap();- let site_field = schema.get_field(Field::SiteNoTokenizer.as_str()).unwrap();+ field = Some(+ schema+ .get_field(Field::Text(TextField::Site).name())+ .unwrap(),+ );++ let domain_field = schema+ .get_field(Field::Text(TextField::DomainNoTokenizer).name())+ .unwrap();+ let site_field = schema+ .get_field(Field::Text(TextField::SiteNoTokenizer).name())+ .unwrap();subqueries.push((Occur::Must,@@ -218,16 +230,32 @@ impl Instruction {));}PatternOption::InUrl if field.is_none() => {- field = Some(schema.get_field(Field::Url.as_str()).unwrap())+ field = Some(+ schema+ .get_field(Field::Text(TextField::Url).name())+ .unwrap(),+ )}PatternOption::InTitle if field.is_none() => {- field = Some(schema.get_field(Field::Title.as_str()).unwrap())+ field = Some(+ schema+ .get_field(Field::Text(TextField::Title).name())+ .unwrap(),+ )}PatternOption::InDescription if field.is_none() => {- field = Some(schema.get_field(Field::Description.as_str()).unwrap())+ field = Some(+ schema+ .get_field(Field::Text(TextField::Description).name())+ .unwrap(),+ )}PatternOption::InContent if field.is_none() => {- field = Some(schema.get_field(Field::CleanBody.as_str()).unwrap())+ field = Some(+ schema+ .get_field(Field::Text(TextField::CleanBody).name())+ .unwrap(),+ )}PatternOption::Action(pattern_action) if action.is_none() => {action = Some(*pattern_action)@@ -237,7 +265,11 @@ impl Instruction {}let action = action.unwrap_or(Action::Boost(1));- let field = field.unwrap_or_else(|| schema.get_field(Field::Url.as_str()).unwrap());+ let field = field.unwrap_or_else(|| {+ schema+ .get_field(Field::Text(TextField::Url).name())+ .unwrap()+ });if !self.patterns.is_empty() {let query = self.pattern_query(field);diff --git a/src/ranking/goggles/pattern_query.rs b/src/ranking/goggles/pattern_query.rsindex c35d450e..2aeec39c 100644--- a/src/ranking/goggles/pattern_query.rs+++ b/src/ranking/goggles/pattern_query.rs@@ -27,7 +27,7 @@ use tantivy::{use crate::{query::intersection::Intersection,ranking::bm25::Bm25Weight,- schema::{Field, ALL_FIELDS},+ schema::{FastField, Field, TextField, ALL_FIELDS},tokenizer,};@@ -149,33 +149,33 @@ impl PatternWeight {.collect();let num_tokens_reader = match &ALL_FIELDS[self.field.field_id() as usize] {- Field::Title => reader.fast_fields().u64(+ Field::Text(TextField::Title) => reader.fast_fields().u64(reader.schema()- .get_field(Field::NumTitleTokens.as_str())+ .get_field(Field::Fast(FastField::NumTitleTokens).name()).unwrap(),),- Field::CleanBody => reader.fast_fields().u64(+ Field::Text(TextField::CleanBody) => reader.fast_fields().u64(reader.schema()- .get_field(Field::NumCleanBodyTokens.as_str())+ .get_field(Field::Fast(FastField::NumCleanBodyTokens).name()).unwrap(),),- Field::Url => reader.fast_fields().u64(+ Field::Text(TextField::Url) => reader.fast_fields().u64(reader.schema()- .get_field(Field::NumUrlTokens.as_str())+ .get_field(Field::Fast(FastField::NumUrlTokens).name()).unwrap(),),- Field::Description => reader.fast_fields().u64(+ Field::Text(TextField::Description) => reader.fast_fields().u64(reader.schema()- .get_field(Field::NumDescriptionTokens.as_str())+ .get_field(Field::Fast(FastField::NumDescriptionTokens).name()).unwrap(),),field => Err(TantivyError::InvalidArgument(format!("{} is not supported in pattern query",- field.as_str()+ field.name()))),}?;diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 2ec08a5a..838e3065 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -16,6 +16,7 @@use std::sync::Arc;+use crate::fastfield_cache::FastFieldCache;use crate::webpage::region::{Region, RegionCount};use chrono::Utc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};@@ -26,6 +27,7 @@ use super::SignalAggregator;pub(crate) struct InitialScoreTweaker {region_count: Arc<RegionCount>,selected_region: Option<Region>,+ fastfield_cache: Arc<FastFieldCache>,aggregator: SignalAggregator,}@@ -34,11 +36,13 @@ impl InitialScoreTweaker {region_count: Arc<RegionCount>,selected_region: Option<Region>,aggregator: SignalAggregator,+ fastfield_cache: Arc<FastFieldCache>,) -> Self {Self {region_count,selected_region,aggregator,+ fastfield_cache,}}}@@ -54,8 +58,13 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {type Child = InitialSegmentScoreTweaker;fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {- let mut aggregator = SignalAggregator::new_like(&self.aggregator);- aggregator.register_readers(segment_reader);+ let mut aggregator = self.aggregator.clone();++ let fastfield_cache_segment = self+ .fastfield_cache+ .get_segment(&segment_reader.segment_id());++ aggregator.register_segment(fastfield_cache_segment);let current_timestamp = Utc::now().timestamp() as usize;diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex d34a5776..eda19d72 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -28,6 +28,7 @@ use tantivy::collector::Collector;use crate::{collector::{MaxDocsConsidered, TopDocs},+ fastfield_cache::FastFieldCache,inverted_index,searcher::NUM_RESULTS_PER_PAGE,webpage::region::{Region, RegionCount},@@ -41,11 +42,16 @@ pub struct Ranker {max_docs: Option<MaxDocsConsidered>,offset: Option<usize>,aggregator: SignalAggregator,+ fastfield_cache: Arc<FastFieldCache>,de_rank_similar: bool,}impl Ranker {- pub fn new(region_count: RegionCount, aggregator: SignalAggregator) -> Self {+ pub fn new(+ region_count: RegionCount,+ aggregator: SignalAggregator,+ fastfield_cache: Arc<FastFieldCache>,+ ) -> Self {Ranker {region_count: Arc::new(region_count),selected_region: None,@@ -53,6 +59,7 @@ impl Ranker {aggregator,max_docs: None,de_rank_similar: true,+ fastfield_cache,}}@@ -82,7 +89,8 @@ impl Ranker {let score_tweaker = InitialScoreTweaker::new(Arc::clone(&self.region_count),self.selected_region,- SignalAggregator::new_like(&self.aggregator),+ self.aggregator.clone(),+ Arc::clone(&self.fastfield_cache),);let mut collector = TopDocs::with_limit(NUM_RESULTS_PER_PAGE);diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex 6e0f2ddb..f22ae4ee 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -14,17 +14,19 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{webpage::Webpage, Result};+use crate::{+ fastfield_cache,+ schema::{FastField, TextField},+ webpage::Webpage,+ Result,+};use std::{array, convert::TryFrom, ops::Deref, sync::Arc};use chrono::Utc;-use tantivy::{- fastfield::{Column, DynamicFastFieldReader},- DocId, Score, SegmentReader,-};+use tantivy::{DocId, Score};use crate::{- schema::{Field, ALL_FIELDS, CENTRALITY_SCALING},+ schema::{Field, CENTRALITY_SCALING},webpage::region::{Region, RegionCount},};@@ -42,7 +44,7 @@ pub enum Signal {Region,}-const ALL_SIGNALS: [Signal; 8] = [+pub const ALL_SIGNALS: [Signal; 8] = [Signal::Bm25,Signal::HostCentrality,Signal::PageCentrality,@@ -54,23 +56,6 @@ const ALL_SIGNALS: [Signal; 8] = [];impl Signal {- fn from_field(field: &Field) -> Option<Self> {- match field {- Field::IsHomepage => Some(Signal::IsHomepage),- Field::HostCentrality => Some(Signal::HostCentrality),- Field::PageCentrality => Some(Signal::PageCentrality),- Field::FetchTimeMs => Some(Signal::FetchTimeMs),- Field::LastUpdated => Some(Signal::UpdateTimestamp),- Field::NumTrackers => Some(Signal::NumTrackers),- Field::Region => Some(Signal::Region),- _ => None,- }- }-- fn has_fast_reader(&self) -> bool {- !matches!(self, Signal::Bm25)- }-fn is_computable_before_search(&self) -> bool {!matches!(self, Signal::Bm25)}@@ -153,18 +138,19 @@ impl Signal {_ => None,}}-}--fn fastfield_reader(segment_reader: &SegmentReader, field: &Field) -> DynamicFastFieldReader<u64> {- let tv_field = segment_reader- .schema()- .get_field(field.as_str())- .unwrap_or_else(|| panic!("Faild to load {} field", field.as_str()));- segment_reader- .fast_fields()- .u64(tv_field)- .unwrap_or_else(|_| panic!("Failed to get {} fast-field reader", field.as_str()))+ fn as_fastfield(&self) -> Option<FastField> {+ match self {+ Signal::Bm25 => None,+ Signal::HostCentrality => Some(FastField::HostCentrality),+ Signal::PageCentrality => Some(FastField::PageCentrality),+ Signal::IsHomepage => Some(FastField::IsHomepage),+ Signal::FetchTimeMs => Some(FastField::FetchTimeMs),+ Signal::UpdateTimestamp => Some(FastField::LastUpdated),+ Signal::NumTrackers => Some(FastField::NumTrackers),+ Signal::Region => Some(FastField::Region),+ }+ }}#[derive(Debug, Clone)]@@ -182,34 +168,34 @@ impl SignalCoefficient {.unwrap_or_else(|| signal.default_coefficient())}- pub fn new(scores: impl Iterator<Item = (Signal, f64)>) -> Self {- let mut fast_scores = Vec::new();+ pub fn new(coefficients: impl Iterator<Item = (Signal, f64)>) -> Self {+ let mut fast_coefficients = Vec::new();- for (signal, score) in scores {+ for (signal, coefficient) in coefficients {let idx = signal as usize;- while idx >= fast_scores.len() {- fast_scores.push(None);+ while idx >= fast_coefficients.len() {+ fast_coefficients.push(None);}- fast_scores[idx] = Some(score);+ fast_coefficients[idx] = Some(coefficient);}- Self(fast_scores)+ Self(fast_coefficients)}}impl FieldBoost {- pub fn get(&self, field: &Field) -> f64 {+ pub fn get(&self, field: &TextField) -> f64 {self.0.get((*field) as usize).copied().flatten()- .or_else(|| field.boost().map(|s| s as f64))+ .or_else(|| Field::Text(*field).boost().map(|s| s as f64)).unwrap_or(1.0)}- pub fn new(scores: impl Iterator<Item = (Field, f64)>) -> Self {+ pub fn new(scores: impl Iterator<Item = (TextField, f64)>) -> Self {let mut fast_scores = Vec::new();for (field, score) in scores {@@ -228,7 +214,7 @@ impl FieldBoost {#[derive(Clone)]pub struct SignalAggregator {- readers: Vec<Option<DynamicFastFieldReader<u64>>>,+ fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,signal_coefficients: SignalCoefficient,field_boost: FieldBoost,fetch_time_ms_cache: [f64; 1000],@@ -253,7 +239,7 @@ impl Default for SignalAggregator {impl SignalAggregator {pub fn new(coefficients: impl Iterator<Item = (Signal, f64)>,- boosts: impl Iterator<Item = (Field, f64)>,+ boosts: impl Iterator<Item = (TextField, f64)>,) -> Self {let signal_coefficients = SignalCoefficient::new(coefficients);let field_boost = FieldBoost::new(boosts);@@ -265,7 +251,7 @@ impl SignalAggregator {.collect();Self {- readers: Vec::new(),+ fastfield_cache: None,signal_coefficients,field_boost,fetch_time_ms_cache,@@ -273,32 +259,8 @@ impl SignalAggregator {}}- pub fn new_like(other: &SignalAggregator) -> Self {- Self {- readers: Vec::new(),- signal_coefficients: other.signal_coefficients.clone(),- field_boost: other.field_boost.clone(),- fetch_time_ms_cache: other.fetch_time_ms_cache,- update_time_cache: other.update_time_cache.clone(),- }- }-- pub fn register_readers(&mut self, segment_reader: &SegmentReader) {- self.readers.clear();-- for field in &ALL_FIELDS {- if let Some(signal) = Signal::from_field(field) {- if signal.has_fast_reader() {- let idx = signal as usize;-- while idx >= self.readers.len() {- self.readers.push(None);- }-- self.readers[idx] = Some(fastfield_reader(segment_reader, field))- }- }- }+ pub fn register_segment(&mut self, cache: Arc<fastfield_cache::SegmentCache>) {+ self.fastfield_cache = Some(cache);}pub fn score(@@ -312,10 +274,11 @@ impl SignalAggregator {ALL_SIGNALS.into_iter().map(|signal| {- let fastfield_value = match self.readers.get(signal as usize) {- Some(Some(reader)) => Some(reader.get_val(doc as u64)),- _ => None,- };+ let fastfield_value = signal.as_fastfield().and_then(|field| {+ self.fastfield_cache+ .as_ref()+ .and_then(|cache| cache.get_doc_cache(&field).get_u64(&doc))+ });self.coefficients().get(&signal)* signal.value(@@ -410,8 +373,10 @@ impl TryFrom<Vec<RawAlteration>> for SignalAggregator {}}Target::Field(name) => {- if let Some(field) = Field::from_string(name) {- boosts.push((field, alteration.score));+ if let Some(field) = Field::from_name(name) {+ if let Some(text_field) = field.as_text() {+ boosts.push((text_field, alteration.score));+ }}}}diff --git a/src/schema.rs b/src/schema.rsindex 47c2a535..e2b9e544 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -22,8 +22,8 @@ use crate::tokenizer::{Identity, Normal, Stemmed};pub const CENTRALITY_SCALING: u64 = 1_000_000_000;-#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub enum Field {+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum TextField {Title,CleanBody,StemmedTitle,@@ -40,14 +40,18 @@ pub enum Field {DomainNameIfHomepageNoTokenizer,/// this field is only set if the webpage is the homepage for the site. Allows us to boostTitleIfHomepage,- IsHomepage,BacklinkText,+ PrimaryImage,+ Description,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum FastField {+ IsHomepage,HostCentrality,PageCentrality,FetchTimeMs,- PrimaryImage,LastUpdated,- Description,NumTrackers,Region,NumUrlTokens,@@ -62,40 +66,47 @@ pub enum Field {PreComputedScore,}+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum Field {+ Fast(FastField),+ Text(TextField),+}+pub static ALL_FIELDS: [Field; 33] = [- Field::Title,- Field::CleanBody,- Field::StemmedTitle,- Field::StemmedCleanBody,- Field::AllBody,- Field::Url,- Field::Site,- Field::Domain,- Field::SiteNoTokenizer,- Field::DomainNoTokenizer,- Field::DomainIfHomepage,- Field::DomainNameIfHomepageNoTokenizer,- Field::TitleIfHomepage,- Field::IsHomepage,- Field::BacklinkText,- Field::HostCentrality,- Field::PageCentrality,- Field::FetchTimeMs,- Field::PrimaryImage,- Field::LastUpdated,- Field::Description,- Field::NumTrackers,- Field::Region,- Field::NumUrlTokens,- Field::NumTitleTokens,- Field::NumCleanBodyTokens,- Field::NumDescriptionTokens,- Field::SiteHash,- Field::UrlWithoutQueryHash,- Field::TitleHash,- Field::UrlHash,- Field::DomainHash,- Field::PreComputedScore,+ Field::Text(TextField::Title),+ Field::Text(TextField::CleanBody),+ Field::Text(TextField::StemmedTitle),+ Field::Text(TextField::StemmedCleanBody),+ Field::Text(TextField::AllBody),+ Field::Text(TextField::Url),+ Field::Text(TextField::Site),+ Field::Text(TextField::Domain),+ Field::Text(TextField::SiteNoTokenizer),+ Field::Text(TextField::DomainNoTokenizer),+ Field::Text(TextField::DomainIfHomepage),+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer),+ Field::Text(TextField::TitleIfHomepage),+ Field::Text(TextField::BacklinkText),+ Field::Text(TextField::PrimaryImage),+ Field::Text(TextField::Description),+ // FAST FIELDS+ Field::Fast(FastField::IsHomepage),+ Field::Fast(FastField::HostCentrality),+ Field::Fast(FastField::PageCentrality),+ Field::Fast(FastField::FetchTimeMs),+ Field::Fast(FastField::LastUpdated),+ Field::Fast(FastField::NumTrackers),+ Field::Fast(FastField::Region),+ Field::Fast(FastField::NumUrlTokens),+ Field::Fast(FastField::NumTitleTokens),+ Field::Fast(FastField::NumCleanBodyTokens),+ Field::Fast(FastField::NumDescriptionTokens),+ Field::Fast(FastField::SiteHash),+ Field::Fast(FastField::UrlWithoutQueryHash),+ Field::Fast(FastField::TitleHash),+ Field::Fast(FastField::UrlHash),+ Field::Fast(FastField::DomainHash),+ Field::Fast(FastField::PreComputedScore),];impl Field {@@ -116,106 +127,122 @@ impl Field {pub fn options(&self) -> IndexingOption {match self {- Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),- Field::CleanBody => IndexingOption::Text(self.default_text_options()),- Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),- Field::Site => IndexingOption::Text(self.default_text_options()),- Field::Domain => IndexingOption::Text(self.default_text_options()),- Field::SiteNoTokenizer => {+ Field::Text(TextField::Title) => {+ IndexingOption::Text(self.default_text_options().set_stored())+ }+ Field::Text(TextField::CleanBody) => IndexingOption::Text(self.default_text_options()),+ Field::Text(TextField::Url) => {+ IndexingOption::Text(self.default_text_options().set_stored())+ }+ Field::Text(TextField::Site) => IndexingOption::Text(self.default_text_options()),+ Field::Text(TextField::Domain) => IndexingOption::Text(self.default_text_options()),+ Field::Text(TextField::SiteNoTokenizer) => {IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))}- Field::DomainNoTokenizer => {+ Field::Text(TextField::DomainNoTokenizer) => {IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))}- Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),- Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),- Field::TitleIfHomepage => IndexingOption::Text(self.default_text_options()),- Field::DomainNameIfHomepageNoTokenizer => {+ Field::Text(TextField::AllBody) => {+ IndexingOption::Text(self.default_text_options().set_stored())+ }+ Field::Text(TextField::DomainIfHomepage) => {+ IndexingOption::Text(self.default_text_options())+ }+ Field::Text(TextField::TitleIfHomepage) => {+ IndexingOption::Text(self.default_text_options())+ }+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))}- Field::IsHomepage => IndexingOption::Integer(+ Field::Text(TextField::BacklinkText) => {+ IndexingOption::Text(self.default_text_options())+ }+ Field::Text(TextField::StemmedTitle) => {+ IndexingOption::Text(self.default_text_options_with_tokenizer(Stemmed::as_str()))+ }+ Field::Text(TextField::StemmedCleanBody) => IndexingOption::Text(+ self.default_text_options_with_tokenizer(Stemmed::as_str())+ .set_stored(),+ ),+ Field::Text(TextField::PrimaryImage) => {+ IndexingOption::Bytes(BytesOptions::default().set_stored())+ }+ Field::Text(TextField::Description) => {+ IndexingOption::Text(self.default_text_options().set_stored())+ }+ Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::BacklinkText => IndexingOption::Text(self.default_text_options()),- Field::HostCentrality => IndexingOption::Integer(+ Field::Fast(FastField::HostCentrality) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::PageCentrality => IndexingOption::Integer(+ Field::Fast(FastField::PageCentrality) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::StemmedTitle => {- IndexingOption::Text(self.default_text_options_with_tokenizer(Stemmed::as_str()))- }- Field::StemmedCleanBody => IndexingOption::Text(- self.default_text_options_with_tokenizer(Stemmed::as_str())- .set_stored(),- ),- Field::FetchTimeMs => IndexingOption::Integer(+ Field::Fast(FastField::FetchTimeMs) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumTrackers => IndexingOption::Integer(+ Field::Fast(FastField::NumTrackers) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::PrimaryImage => IndexingOption::Bytes(BytesOptions::default().set_stored()),- Field::LastUpdated => IndexingOption::Integer(+ Field::Fast(FastField::LastUpdated) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_stored().set_indexed(),),- Field::Description => IndexingOption::Text(self.default_text_options().set_stored()),- Field::Region => IndexingOption::Integer(+ Field::Fast(FastField::Region) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_stored().set_indexed(),),- Field::NumCleanBodyTokens => IndexingOption::Integer(+ Field::Fast(FastField::NumCleanBodyTokens) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumDescriptionTokens => IndexingOption::Integer(+ Field::Fast(FastField::NumDescriptionTokens) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumTitleTokens => IndexingOption::Integer(+ Field::Fast(FastField::NumTitleTokens) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumUrlTokens => IndexingOption::Integer(+ Field::Fast(FastField::NumUrlTokens) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::SiteHash => IndexingOption::Integer(+ Field::Fast(FastField::SiteHash) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),- Field::UrlWithoutQueryHash => IndexingOption::Integer(+ Field::Fast(FastField::UrlWithoutQueryHash) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),- Field::UrlHash => IndexingOption::Integer(+ Field::Fast(FastField::UrlHash) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),- Field::DomainHash => IndexingOption::Integer(+ Field::Fast(FastField::DomainHash) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),- Field::TitleHash => IndexingOption::Integer(+ Field::Fast(FastField::TitleHash) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),- Field::PreComputedScore => IndexingOption::Float(+ Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed()@@ -224,139 +251,128 @@ impl Field {}}- pub fn as_str(&self) -> &str {+ pub fn name(&self) -> &str {match self {- Field::Title => "title",- Field::CleanBody => "body",- Field::Url => "url",- Field::Site => "site",- Field::Domain => "domain",- Field::SiteNoTokenizer => "site_no_tokenizer",- Field::DomainNoTokenizer => "domain_no_tokenizer",- Field::BacklinkText => "backlink_text",- Field::HostCentrality => "host_centrality",- Field::PageCentrality => "page_centrality",- Field::StemmedTitle => "stemmed_title",- Field::StemmedCleanBody => "stemmed_body",- Field::DomainIfHomepage => "domain_if_homepage",- Field::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",- Field::IsHomepage => "is_homepage",- Field::FetchTimeMs => "fetch_time_ms",- Field::PrimaryImage => "primary_image_uuid",- Field::LastUpdated => "last_updated",- Field::Description => "description",- Field::AllBody => "all_body",- Field::NumTrackers => "num_trackers",- Field::Region => "region",- Field::NumUrlTokens => "num_url_tokens",- Field::NumTitleTokens => "num_title_tokens",- Field::NumCleanBodyTokens => "num_clean_body_tokens",- Field::NumDescriptionTokens => "num_description_tokens",- Field::SiteHash => "site_hash",- Field::UrlWithoutQueryHash => "url_without_query_hash",- Field::PreComputedScore => "pre_computed_score",- Field::TitleIfHomepage => "title_if_homepage",- Field::TitleHash => "title_hash",- Field::UrlHash => "url_hash",- Field::DomainHash => "domain_hash",+ Field::Text(TextField::Title) => "title",+ Field::Text(TextField::CleanBody) => "body",+ Field::Text(TextField::Url) => "url",+ Field::Text(TextField::Site) => "site",+ Field::Text(TextField::Domain) => "domain",+ Field::Text(TextField::SiteNoTokenizer) => "site_no_tokenizer",+ Field::Text(TextField::DomainNoTokenizer) => "domain_no_tokenizer",+ Field::Text(TextField::BacklinkText) => "backlink_text",+ Field::Text(TextField::StemmedTitle) => "stemmed_title",+ Field::Text(TextField::StemmedCleanBody) => "stemmed_body",+ Field::Text(TextField::DomainIfHomepage) => "domain_if_homepage",+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {+ "domain_name_if_homepage_no_tokenizer"+ }+ Field::Text(TextField::Description) => "description",+ Field::Text(TextField::PrimaryImage) => "primary_image_uuid",+ Field::Text(TextField::TitleIfHomepage) => "title_if_homepage",+ Field::Text(TextField::AllBody) => "all_body",+ Field::Fast(FastField::HostCentrality) => "host_centrality",+ Field::Fast(FastField::PageCentrality) => "page_centrality",+ Field::Fast(FastField::IsHomepage) => "is_homepage",+ Field::Fast(FastField::FetchTimeMs) => "fetch_time_ms",+ Field::Fast(FastField::LastUpdated) => "last_updated",+ Field::Fast(FastField::NumTrackers) => "num_trackers",+ Field::Fast(FastField::Region) => "region",+ Field::Fast(FastField::NumUrlTokens) => "num_url_tokens",+ Field::Fast(FastField::NumTitleTokens) => "num_title_tokens",+ Field::Fast(FastField::NumCleanBodyTokens) => "num_clean_body_tokens",+ Field::Fast(FastField::NumDescriptionTokens) => "num_description_tokens",+ Field::Fast(FastField::SiteHash) => "site_hash",+ Field::Fast(FastField::UrlWithoutQueryHash) => "url_without_query_hash",+ Field::Fast(FastField::PreComputedScore) => "pre_computed_score",+ Field::Fast(FastField::TitleHash) => "title_hash",+ Field::Fast(FastField::UrlHash) => "url_hash",+ Field::Fast(FastField::DomainHash) => "domain_hash",}}pub fn boost(&self) -> Option<f32> {match self {- Field::Site => Some(3.0),- Field::TitleIfHomepage => Some(2.0),- Field::DomainIfHomepage => Some(10.0),- Field::DomainNameIfHomepageNoTokenizer => Some(30.0),- Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),- Field::CleanBody => Some(4.0),- Field::Title => Some(10.0),- Field::Url => Some(1.0),- Field::Domain => Some(1.0),- Field::AllBody => Some(0.01),- Field::BacklinkText => Some(4.0),- Field::HostCentrality- | Field::PageCentrality- | Field::SiteHash- | Field::TitleHash- | Field::UrlHash- | Field::UrlWithoutQueryHash- | Field::SiteNoTokenizer- | Field::DomainNoTokenizer- | Field::IsHomepage- | Field::PrimaryImage- | Field::FetchTimeMs- | Field::Description- | Field::NumTrackers- | Field::NumUrlTokens- | Field::DomainHash- | Field::NumTitleTokens- | Field::NumCleanBodyTokens- | Field::NumDescriptionTokens- | Field::Region- | Field::PreComputedScore- | Field::LastUpdated => None,+ Field::Text(TextField::Site) => Some(3.0),+ Field::Text(TextField::TitleIfHomepage) => Some(2.0),+ Field::Text(TextField::DomainIfHomepage) => Some(10.0),+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => Some(30.0),+ Field::Text(TextField::StemmedCleanBody) | Field::Text(TextField::StemmedTitle) => {+ Some(0.1)+ }+ Field::Text(TextField::CleanBody) => Some(4.0),+ Field::Text(TextField::Title) => Some(10.0),+ Field::Text(TextField::Url) => Some(1.0),+ Field::Text(TextField::Domain) => Some(1.0),+ Field::Text(TextField::AllBody) => Some(0.01),+ Field::Text(TextField::BacklinkText) => Some(4.0),+ Field::Text(TextField::SiteNoTokenizer)+ | Field::Text(TextField::DomainNoTokenizer)+ | Field::Text(TextField::Description)+ | Field::Text(TextField::PrimaryImage) => None,+ Field::Fast(_) => None,}}pub fn is_searchable(&self) -> bool {- !matches!(self, Field::PrimaryImage | Field::BacklinkText) && !self.is_fast()+ !matches!(+ self,+ Field::Text(TextField::PrimaryImage) | Field::Text(TextField::BacklinkText)+ ) && !self.is_fast()}pub fn is_fast(&self) -> bool {- matches!(- self,- Field::IsHomepage- | Field::HostCentrality- | Field::FetchTimeMs- | Field::LastUpdated- | Field::NumTrackers- | Field::Region- | Field::NumUrlTokens- | Field::NumTitleTokens- | Field::NumCleanBodyTokens- | Field::NumDescriptionTokens- | Field::SiteHash- | Field::UrlWithoutQueryHash- | Field::TitleHash- | Field::UrlHash- | Field::DomainHash- | Field::PreComputedScore- )+ matches!(self, Field::Fast(_))}- pub fn from_string(name: String) -> Option<Field> {+ pub fn from_name(name: String) -> Option<Field> {match name.as_str() {- "title" => Some(Field::Title),- "body" => Some(Field::CleanBody),- "url" => Some(Field::Url),- "site" => Some(Field::Site),- "backlink_text" => Some(Field::BacklinkText),- "host_centrality" => Some(Field::HostCentrality),- "page_centrality" => Some(Field::PageCentrality),- "stemmed_title" => Some(Field::StemmedTitle),- "stemmed_body" => Some(Field::StemmedCleanBody),- "domain" => Some(Field::Domain),- "domain_if_homepage" => Some(Field::DomainIfHomepage),- "is_homepage" => Some(Field::IsHomepage),- "fetch_time_ms" => Some(Field::FetchTimeMs),- "primary_image_uuid" => Some(Field::PrimaryImage),- "last_updated" => Some(Field::LastUpdated),- "description" => Some(Field::Description),- "all_body" => Some(Field::AllBody),- "num_trackers" => Some(Field::NumTrackers),- "region" => Some(Field::Region),- "site_hash" => Some(Field::SiteHash),- "url_without_query_hash" => Some(Field::UrlWithoutQueryHash),- "domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),- "pre_computed_score" => Some(Field::PreComputedScore),- "title_if_homepage" => Some(Field::TitleIfHomepage),- "url_hash" => Some(Field::UrlHash),- "domain_hash" => Some(Field::DomainHash),- "title_hash" => Some(Field::TitleHash),+ "title" => Some(Field::Text(TextField::Title)),+ "body" => Some(Field::Text(TextField::CleanBody)),+ "url" => Some(Field::Text(TextField::Url)),+ "site" => Some(Field::Text(TextField::Site)),+ "backlink_text" => Some(Field::Text(TextField::BacklinkText)),+ "stemmed_title" => Some(Field::Text(TextField::StemmedTitle)),+ "stemmed_body" => Some(Field::Text(TextField::StemmedCleanBody)),+ "domain" => Some(Field::Text(TextField::Domain)),+ "domain_if_homepage" => Some(Field::Text(TextField::DomainIfHomepage)),+ "primary_image_uuid" => Some(Field::Text(TextField::PrimaryImage)),+ "domain_name_if_homepage_no_tokenizer" => {+ Some(Field::Text(TextField::DomainNameIfHomepageNoTokenizer))+ }+ "description" => Some(Field::Text(TextField::Description)),+ "all_body" => Some(Field::Text(TextField::AllBody)),+ "title_if_homepage" => Some(Field::Text(TextField::TitleIfHomepage)),+ "host_centrality" => Some(Field::Fast(FastField::HostCentrality)),+ "page_centrality" => Some(Field::Fast(FastField::PageCentrality)),+ "is_homepage" => Some(Field::Fast(FastField::IsHomepage)),+ "fetch_time_ms" => Some(Field::Fast(FastField::FetchTimeMs)),+ "last_updated" => Some(Field::Fast(FastField::LastUpdated)),+ "num_trackers" => Some(Field::Fast(FastField::NumTrackers)),+ "region" => Some(Field::Fast(FastField::Region)),+ "site_hash" => Some(Field::Fast(FastField::SiteHash)),+ "url_without_query_hash" => Some(Field::Fast(FastField::UrlWithoutQueryHash)),+ "pre_computed_score" => Some(Field::Fast(FastField::PreComputedScore)),+ "url_hash" => Some(Field::Fast(FastField::UrlHash)),+ "domain_hash" => Some(Field::Fast(FastField::DomainHash)),+ "title_hash" => Some(Field::Fast(FastField::TitleHash)),_ => None,}}++ pub fn as_text(&self) -> Option<TextField> {+ match self {+ Field::Fast(_) => None,+ Field::Text(field) => Some(*field),+ }+ }++ pub fn as_fast(&self) -> Option<FastField> {+ match self {+ Field::Fast(field) => Some(*field),+ Field::Text(_) => None,+ }+ }}pub fn create_schema() -> tantivy::schema::Schema {@@ -364,10 +380,10 @@ pub fn create_schema() -> tantivy::schema::Schema {for field in &ALL_FIELDS {match field.options() {- IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),- IndexingOption::Integer(options) => builder.add_u64_field(field.as_str(), options),- IndexingOption::Float(options) => builder.add_f64_field(field.as_str(), options),- IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),+ IndexingOption::Text(options) => builder.add_text_field(field.name(), options),+ IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),+ IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),};}@@ -380,3 +396,33 @@ pub enum IndexingOption {Float(tantivy::schema::NumericOptions),Bytes(tantivy::schema::BytesOptions),}++pub enum DataType {+ U64,+ U64s,+ F64,+}++impl FastField {+ pub fn data_type(&self) -> DataType {+ match self {+ FastField::IsHomepage => DataType::U64,+ FastField::HostCentrality => DataType::U64,+ FastField::PageCentrality => DataType::U64,+ FastField::FetchTimeMs => DataType::U64,+ FastField::LastUpdated => DataType::U64,+ FastField::NumTrackers => DataType::U64,+ FastField::Region => DataType::U64,+ FastField::NumUrlTokens => DataType::U64,+ FastField::NumTitleTokens => DataType::U64,+ FastField::NumCleanBodyTokens => DataType::U64,+ FastField::NumDescriptionTokens => DataType::U64,+ FastField::SiteHash => DataType::U64s,+ FastField::UrlWithoutQueryHash => DataType::U64s,+ FastField::TitleHash => DataType::U64s,+ FastField::UrlHash => DataType::U64s,+ FastField::DomainHash => DataType::U64s,+ FastField::PreComputedScore => DataType::F64,+ }+ }+}diff --git a/src/searcher/local.rs b/src/searcher/local.rsindex 609ab31f..00133172 100644--- a/src/searcher/local.rs+++ b/src/searcher/local.rs@@ -107,6 +107,7 @@ impl LocalSearcher {let mut ranker = Ranker::new(self.index.region_count.clone(),goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),+ self.index.inverted_index.fastfield_cache(),);if let Some(skip_pages) = query.skip_pages {diff --git a/src/snippet.rs b/src/snippet.rsindex 28027490..b197a821 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::schema::TextField;use crate::search_prettifier::html_escape;use crate::tokenizer::Stemmed;use crate::webpage::region::Region;@@ -203,12 +204,12 @@ impl SnippetGenerator {) -> crate::Result<SnippetGenerator> {let clean_field = searcher.schema()- .get_field(Field::CleanBody.as_str())+ .get_field(Field::Text(TextField::CleanBody).name()).unwrap();let stemmed_field = searcher.schema()- .get_field(Field::StemmedCleanBody.as_str())+ .get_field(Field::Text(TextField::StemmedCleanBody).name()).unwrap();let mut terms: BTreeSet<&tantivy::Term> = BTreeSet::new();diff --git a/src/subdomain_count.rs b/src/subdomain_count.rsindex add016f1..591575fb 100644--- a/src/subdomain_count.rs+++ b/src/subdomain_count.rs@@ -46,4 +46,8 @@ impl SubdomainCounter {pub fn commit(&self) {self.inner.flush();}++ pub fn merge(&mut self, other: Self) {+ todo!();+ }}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex aed42b6c..59e75976 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{prehashed::{hash, split_u128},+ schema::{FastField, TextField},schema_org::SchemaOrg,tokenizer, Error, Result,};@@ -153,14 +154,14 @@ impl Webpage {if let Ok(region) = region {doc.add_u64(schema- .get_field(Field::Region.as_str())+ .get_field(Field::Fast(FastField::Region).name()).expect("Failed to get region field"),region.id() as u64,);} else {doc.add_u64(schema- .get_field(Field::Region.as_str())+ .get_field(Field::Fast(FastField::Region).name()).expect("Failed to get region field"),Region::All.id() as u64,);@@ -174,35 +175,35 @@ impl Webpage {doc.add_text(schema- .get_field(Field::BacklinkText.as_str())+ .get_field(Field::Text(TextField::BacklinkText).name()).expect("Failed to get backlink-text field"),backlink_text,);doc.add_u64(schema- .get_field(Field::HostCentrality.as_str())+ .get_field(Field::Fast(FastField::HostCentrality).name()).expect("Failed to get host_centrality field"),(self.host_centrality * CENTRALITY_SCALING as f64) as u64,);doc.add_u64(schema- .get_field(Field::PageCentrality.as_str())+ .get_field(Field::Fast(FastField::PageCentrality).name()).expect("Failed to get page_centrality field"),(self.page_centrality * CENTRALITY_SCALING as f64) as u64,);doc.add_u64(schema- .get_field(Field::FetchTimeMs.as_str())+ .get_field(Field::Fast(FastField::FetchTimeMs).name()).expect("Failed to get fetch_time_ms field"),self.fetch_time_ms,);doc.add_f64(schema- .get_field(Field::PreComputedScore.as_str())+ .get_field(Field::Fast(FastField::PreComputedScore).name()).expect("failed to get pre_computed_score field"),self.pre_computed_score,);@@ -210,7 +211,7 @@ impl Webpage {let image = bincode::serialize(&self.primary_image).unwrap();doc.add_bytes(schema- .get_field(Field::PrimaryImage.as_str())+ .get_field(Field::Text(TextField::PrimaryImage).name()).expect("Failed to get primary_image field"),image,);@@ -531,12 +532,14 @@ impl Html {for field in &ALL_FIELDS {let tantivy_field = schema- .get_field(field.as_str())- .unwrap_or_else(|| panic!("Unknown field: {}", field.as_str()));+ .get_field(field.name())+ .unwrap_or_else(|| panic!("Unknown field: {}", field.name()));match field {- Field::Title => doc.add_pre_tokenized_text(tantivy_field, title.clone()),- Field::StemmedTitle => {+ Field::Text(TextField::Title) => {+ doc.add_pre_tokenized_text(tantivy_field, title.clone())+ }+ Field::Text(TextField::StemmedTitle) => {let mut tokens = title.tokens.clone();stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));@@ -548,8 +551,10 @@ impl Html {},);}- Field::CleanBody => doc.add_pre_tokenized_text(tantivy_field, clean_text.clone()),- Field::StemmedCleanBody => {+ Field::Text(TextField::CleanBody) => {+ doc.add_pre_tokenized_text(tantivy_field, clean_text.clone())+ }+ Field::Text(TextField::StemmedCleanBody) => {let mut tokens = clean_text.tokens.clone();stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));@@ -561,13 +566,15 @@ impl Html {},);}- Field::Description => {+ Field::Text(TextField::Description) => {doc.add_pre_tokenized_text(tantivy_field, description.clone());}- Field::Url => doc.add_pre_tokenized_text(tantivy_field, url.clone()),- Field::Site => doc.add_text(tantivy_field, self.url().site()),- Field::Domain => doc.add_text(tantivy_field, self.url().domain()),- Field::SiteNoTokenizer => doc.add_pre_tokenized_text(+ Field::Text(TextField::Url) => {+ doc.add_pre_tokenized_text(tantivy_field, url.clone())+ }+ Field::Text(TextField::Site) => doc.add_text(tantivy_field, self.url().site()),+ Field::Text(TextField::Domain) => doc.add_text(tantivy_field, self.url().domain()),+ Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(tantivy_field,PreTokenizedString {text: self.url().site().to_string(),@@ -580,7 +587,7 @@ impl Html {}],},),- Field::DomainNoTokenizer => doc.add_pre_tokenized_text(+ Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(tantivy_field,PreTokenizedString {text: self.url().domain().to_string(),@@ -593,21 +600,21 @@ impl Html {}],},),- Field::TitleIfHomepage => {+ Field::Text(TextField::TitleIfHomepage) => {if self.url().is_homepage() {doc.add_pre_tokenized_text(tantivy_field, title.clone());} else {doc.add_text(tantivy_field, "");}}- Field::DomainIfHomepage => {+ Field::Text(TextField::DomainIfHomepage) => {if self.url().is_homepage() {doc.add_text(tantivy_field, self.url().domain());} else {doc.add_text(tantivy_field, "");}}- Field::DomainNameIfHomepageNoTokenizer => {+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {if self.url().is_homepage() {doc.add_pre_tokenized_text(tantivy_field,@@ -626,61 +633,69 @@ impl Html {doc.add_text(tantivy_field, "");}}- Field::IsHomepage => {+ Field::Text(TextField::AllBody) => {+ doc.add_pre_tokenized_text(tantivy_field, all_text.clone())+ }+ Field::Fast(FastField::IsHomepage) => {doc.add_u64(tantivy_field, self.url().is_homepage().into());}- Field::LastUpdated => doc.add_u64(+ Field::Fast(FastField::LastUpdated) => doc.add_u64(tantivy_field,self.updated_time().map_or(0, |time| time.timestamp().max(0) as u64),),- Field::AllBody => doc.add_pre_tokenized_text(tantivy_field, all_text.clone()),- Field::NumTrackers => doc.add_u64(tantivy_field, self.trackers().len() as u64),- Field::NumUrlTokens => doc.add_u64(tantivy_field, url.tokens.len() as u64),- Field::NumTitleTokens => doc.add_u64(tantivy_field, title.tokens.len() as u64),- Field::NumCleanBodyTokens => {+ Field::Fast(FastField::NumTrackers) => {+ doc.add_u64(tantivy_field, self.trackers().len() as u64)+ }+ Field::Fast(FastField::NumUrlTokens) => {+ doc.add_u64(tantivy_field, url.tokens.len() as u64)+ }+ Field::Fast(FastField::NumTitleTokens) => {+ doc.add_u64(tantivy_field, title.tokens.len() as u64)+ }+ Field::Fast(FastField::NumCleanBodyTokens) => {doc.add_u64(tantivy_field, clean_text.tokens.len() as u64)}- Field::NumDescriptionTokens => {+ Field::Fast(FastField::NumDescriptionTokens) => {doc.add_u64(tantivy_field, description.tokens.len() as u64)}- Field::SiteHash => {+ Field::Fast(FastField::SiteHash) => {let hash = hash(self.url().site()).0;let u64s = split_u128(hash);doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}- Field::UrlWithoutQueryHash => {+ Field::Fast(FastField::UrlWithoutQueryHash) => {let hash = hash(self.url().without_query()).0;let u64s = split_u128(hash);doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}- Field::UrlHash => {+ Field::Fast(FastField::UrlHash) => {let hash = hash(self.url().full()).0;let u64s = split_u128(hash);doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}- Field::DomainHash => {+ Field::Fast(FastField::DomainHash) => {let hash = hash(self.url().domain()).0;let u64s = split_u128(hash);doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}- Field::TitleHash => {+ Field::Fast(FastField::TitleHash) => {let hash = hash(self.title().unwrap_or_default()).0;let u64s = split_u128(hash);doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}- Field::BacklinkText- | Field::HostCentrality- | Field::PageCentrality- | Field::FetchTimeMs- | Field::PreComputedScore- | Field::Region- | Field::PrimaryImage => {}+ Field::Text(TextField::BacklinkText)+ | Field::Fast(FastField::HostCentrality)+ | Field::Fast(FastField::PageCentrality)+ | Field::Fast(FastField::FetchTimeMs)+ | Field::Fast(FastField::PreComputedScore)+ | Field::Fast(FastField::Region)+ | Field::Text(TextField::PrimaryImage) => {}}}