web search done right
Ftr/sort postings by score (#57)
Mikkel Denkerb902a3b7aa0f
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex 961b05e8..adc77d5a 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -152,9 +152,9 @@ import Header from "../components/Header.astro";<Hero class="change-page-active" icon="chevron-right" /></a></div>- <div class="flex flex-col w-full mt-5 pl-4 pr-24">+ <div class="flex flex-col w-full mt-7 pl-4 pr-24">Didn't find what you were looking for? Try on:- <div class="flex w-full justify-between mt-2">+ <div class="flex w-full justify-between mt-4"><div class="bg-neutral-100 rounded-full py-2 px-5"><a href="https://www.google.com/search?q={{ query|urlencode }}">Google</adiff --git a/src/collector.rs b/src/collector.rsindex c826d74f..6df5752c 100644--- a/src/collector.rs+++ b/src/collector.rs@@ -31,14 +31,25 @@ fn adjust_score(num_taken: usize, original_score: f64) -> f64 {original_score * (SCALE / (num_taken as f64 + SCALE))}+#[derive(Clone)]+pub struct MaxDocsConsidered {+ pub total_docs: usize,+ pub segments: usize,+}+pub struct TopDocs {top_n: usize,offset: usize,+ max_docs: Option<MaxDocsConsidered>,}impl TopDocs {pub fn with_limit(top_n: usize) -> Self {- Self { top_n, offset: 0 }+ Self {+ top_n,+ offset: 0,+ max_docs: None,+ }}pub fn and_offset(mut self, offset: usize) -> Self {@@ -46,6 +57,12 @@ impl TopDocs {self}+ pub fn and_max_docs(mut self, max_docs: MaxDocsConsidered) -> Self {+ self.max_docs = Some(max_docs);++ self+ }+pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(self,score_tweaker: TScoreTweaker,@@ -75,8 +92,15 @@ impl Collector for TopDocs {.unwrap(),)?;+ let max_docs = self+ .max_docs+ .as_ref()+ .map(|max_docs| max_docs.total_docs / max_docs.segments);+Ok(TopSegmentCollector {key_reader,+ max_docs,+ num_docs_taken: 0,segment_ord: segment_local_id,bucket_collector: BucketCollector::new(self.top_n + self.offset),})@@ -117,6 +141,8 @@ impl Collector for TopDocs {pub struct TopSegmentCollector {key_reader: MultiValuedFastFieldReader<u64>,+ max_docs: Option<usize>,+ num_docs_taken: usize,segment_ord: SegmentOrdinal,bucket_collector: BucketCollector,}@@ -125,6 +151,13 @@ impl SegmentCollector for TopSegmentCollector {type Fruit = Vec<Doc>;fn collect(&mut self, doc: DocId, score: Score) {+ if let Some(max_docs) = &self.max_docs {+ if self.num_docs_taken >= *max_docs {+ return;+ }++ self.num_docs_taken += 1;+ }let mut keys = Vec::new();self.key_reader.get_vals(doc, &mut keys);debug_assert_eq!(keys.len(), 2);@@ -137,7 +170,7 @@ impl SegmentCollector for TopSegmentCollector {id: doc,segment: self.segment_ord,score: score as f64,- })+ });}fn harvest(self) -> Self::Fruit {diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex c0070087..8306c2a1 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -25,6 +25,7 @@ use crate::entrypoint::download_all_warc_files;use crate::index::{FrozenIndex, Index};use crate::mapreduce::{Map, MapReduce, Reduce, Worker};use crate::ranking::centrality_store::CentralityStore;+use crate::ranking::SignalAggregator;use crate::warc::WarcFile;use crate::webgraph::{Node, Webgraph, WebgraphBuilder};use crate::webpage::{Html, Link, Webpage};@@ -86,6 +87,7 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {};let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);+ let signal_aggregator = SignalAggregator::default();for file in warc_files {if let Ok(file) = WarcFile::open(&file) {@@ -131,14 +133,19 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {trace!("title = {:?}", html.title());trace!("text = {:?}", html.clean_text());- let webpage = Webpage {+ let mut webpage = Webpage {html,backlinks,page_centrality,host_centrality,fetch_time_ms,primary_image: None,+ pre_computed_score: 0.0,};++ webpage.pre_computed_score =+ signal_aggregator.precompute_score(&webpage, &index.region_count);+if let Err(err) = index.insert(webpage) {debug!("{:?}", err);}diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 88305fb6..a257a46d 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -62,7 +62,7 @@ pub struct DisplayedWebpage {}const MAX_PRETTY_URL_LEN: usize = 50;-const MAX_TITLE_LEN: usize = 50;+const MAX_TITLE_LEN: usize = 70;fn prettify_url(url: &Url) -> String {let mut pretty_url = url.strip_query().to_string();diff --git a/src/index.rs b/src/index.rsindex 06484075..81fa492d 100644--- a/src/index.rs+++ b/src/index.rs@@ -250,6 +250,10 @@ impl Index {pub fn spell_correction(&self, terms: &[String]) -> Option<String> {self.spell_check(terms).or_else(|| self.split_words(terms))}++ pub fn num_segments(&self) -> usize {+ self.inverted_index.num_segments()+ }}#[derive(Debug, Serialize, Deserialize)]diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 25163e5d..e3c991b9 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -17,6 +17,7 @@use chrono::NaiveDateTime;use serde::Serialize;use tantivy::collector::{Collector, Count};+use tantivy::directory::MmapDirectory;use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;@@ -52,10 +53,20 @@ impl InvertedIndex {let schema = create_schema();let mut tantivy_index = if path.as_ref().exists() {- tantivy::Index::open_in_dir(&path)?+ let mmap_directory = MmapDirectory::open(&path)?;+ tantivy::Index::open(mmap_directory)?} else {+ let index_settings = tantivy::IndexSettings {+ sort_by_field: Some(tantivy::IndexSortByField {+ field: Field::PreComputedScore.as_str().to_string(),+ order: tantivy::Order::Desc,+ }),+ ..Default::default()+ };+fs::create_dir_all(&path)?;- tantivy::Index::create_in_dir(&path, schema.clone())?+ let mmap_directory = MmapDirectory::open(&path)?;+ tantivy::Index::create(mmap_directory, schema.clone(), index_settings)?};tantivy_index.set_default_multithread_executor()?;@@ -271,6 +282,10 @@ impl InvertedIndex {pub fn schema(&self) -> Arc<Schema> {Arc::clone(&self.schema)}++ pub fn num_segments(&self) -> usize {+ self.tantivy_index.searchable_segments().unwrap().len()+ }}#[derive(Debug, Serialize)]@@ -377,6 +392,7 @@ impl From<Document> for RetrievedWebpage {| Field::SiteHash| Field::UrlWithoutQueryHash| Field::NumTrackers+ | Field::PreComputedScore| Field::NumCleanBodyTokens| Field::NumDescriptionTokens| Field::NumTitleTokens@@ -614,6 +630,7 @@ mod tests {host_centrality: 1.0,page_centrality: 0.0,fetch_time_ms: 500,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");diff --git a/src/lib.rs b/src/lib.rsindex be985de5..7e5311be 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -195,6 +195,9 @@ pub enum Error {#[error("String is not float")]ParseFloat(#[from] std::num::ParseFloatError),++ #[error("Could not open inverted-index directory")]+ Directory(#[from] tantivy::directory::error::OpenDirectoryError),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex d7a7bf69..5b23a169 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -327,6 +327,7 @@ mod tests {host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -351,6 +352,7 @@ mod tests {host_centrality: 0.0001,page_centrality: 0.0,primary_image: None,+ pre_computed_score: 0.0,fetch_time_ms: 500,}).expect("failed to insert webpage");@@ -447,6 +449,7 @@ mod tests {host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -471,6 +474,7 @@ mod tests {host_centrality: 0.0001,page_centrality: 0.0,primary_image: None,+ pre_computed_score: 0.0,fetch_time_ms: 500,}).expect("failed to insert webpage");@@ -546,6 +550,7 @@ mod tests {host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -570,6 +575,7 @@ mod tests {host_centrality: 0.0001,page_centrality: 0.0,primary_image: None,+ pre_computed_score: 0.0,fetch_time_ms: 500,}).expect("failed to insert webpage");@@ -594,6 +600,7 @@ mod tests {host_centrality: 0.0001,page_centrality: 0.0,primary_image: None,+ pre_computed_score: 0.0,fetch_time_ms: 500,}).expect("failed to insert webpage");diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 5c7dc382..bf2976c2 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -26,7 +26,7 @@ use initial::InitialScoreTweaker;use tantivy::collector::Collector;use crate::{- collector::TopDocs,+ collector::{MaxDocsConsidered, TopDocs},searcher::NUM_RESULTS_PER_PAGE,webpage::region::{Region, RegionCount},};@@ -36,6 +36,7 @@ pub use self::signal::*;pub struct Ranker {region_count: Arc<RegionCount>,selected_region: Option<Region>,+ max_docs: Option<MaxDocsConsidered>,offset: Option<usize>,aggregator: SignalAggregator,}@@ -47,6 +48,7 @@ impl Ranker {selected_region: None,offset: None,aggregator,+ max_docs: None,}}@@ -60,6 +62,14 @@ impl Ranker {self}+ pub fn with_max_docs(mut self, total_docs: usize, segments: usize) -> Self {+ self.max_docs = Some(MaxDocsConsidered {+ total_docs,+ segments,+ });+ self+ }+pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {let score_tweaker = InitialScoreTweaker::new(Arc::clone(&self.region_count),@@ -73,6 +83,10 @@ impl Ranker {collector = collector.and_offset(offset);}+ if let Some(max_docs) = &self.max_docs {+ collector = collector.and_max_docs(max_docs.clone());+ }+collector.tweak_score(score_tweaker)}}@@ -113,6 +127,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -137,6 +152,7 @@ mod tests {host_centrality: 5.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -179,6 +195,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -203,6 +220,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 5.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -244,6 +262,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -268,6 +287,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -290,6 +310,7 @@ mod tests {host_centrality: 0.003,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -332,6 +353,7 @@ mod tests {host_centrality: 0.092,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -356,6 +378,7 @@ mod tests {backlinks: vec![],host_centrality: 0.09,fetch_time_ms: 500,+ pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,})@@ -395,6 +418,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -429,6 +453,7 @@ mod tests {host_centrality: 0.003,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -472,6 +497,7 @@ mod tests {host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -493,6 +519,7 @@ mod tests {backlinks: vec![],host_centrality: 0.003,fetch_time_ms: 500,+ pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,})@@ -532,6 +559,7 @@ mod tests {backlinks: vec![],host_centrality: 1.0,fetch_time_ms: 20,+ pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,})@@ -556,6 +584,7 @@ mod tests {host_centrality: 1.0,fetch_time_ms: 20,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -580,6 +609,7 @@ mod tests {host_centrality: 1.02,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");@@ -665,6 +695,7 @@ mod tests {),backlinks: vec![],host_centrality: 1.0,+ pre_computed_score: 0.0,fetch_time_ms: 500,page_centrality: 0.0,primary_image: None,@@ -690,6 +721,7 @@ mod tests {backlinks: vec![],host_centrality: 1.0,fetch_time_ms: 500,+ pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,})@@ -714,6 +746,7 @@ mod tests {backlinks: vec![],host_centrality: 1.0,fetch_time_ms: 500,+ pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,})@@ -759,6 +792,7 @@ mod tests {backlinks: vec![],host_centrality: 1.0,fetch_time_ms: 0,+ pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,})@@ -784,6 +818,7 @@ mod tests {host_centrality: 1.0,fetch_time_ms: 5000,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex 566229c2..f81bb59c 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -14,9 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::Result;-use std::{array, convert::TryFrom, sync::Arc};+use crate::{webpage::Webpage, Result};+use std::{array, convert::TryFrom, ops::Deref, sync::Arc};+use chrono::Utc;use tantivy::{fastfield::{Column, DynamicFastFieldReader},DocId, Score, SegmentReader,@@ -70,13 +71,15 @@ impl Signal {!matches!(self, Signal::Bm25)}- #[allow(clippy::too_many_arguments)]+ fn is_computable_before_search(&self) -> bool {+ todo!();+ }+fn value(&self,- doc: DocId,bm25: Score,- fastfield: Option<&DynamicFastFieldReader<u64>>,- region_count: &Arc<RegionCount>,+ fastfield_value: Option<u64>,+ region_count: &impl Deref<Target = RegionCount>,current_timestamp: usize,selected_region: Option<Region>,aggregator: &SignalAggregator,@@ -84,11 +87,11 @@ impl Signal {match self {Signal::Bm25 => bm25 as f64,Signal::HostCentrality | Signal::PageCentrality => {- fastfield.unwrap().get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64+ fastfield_value.unwrap() as f64 / CENTRALITY_SCALING as f64}- Signal::IsHomepage => fastfield.unwrap().get_val(doc as u64) as f64,+ Signal::IsHomepage => fastfield_value.unwrap() as f64,Signal::FetchTimeMs => {- let fetch_time_ms = fastfield.unwrap().get_val(doc as u64) as usize;+ let fetch_time_ms = fastfield_value.unwrap() as usize;if fetch_time_ms >= aggregator.fetch_time_ms_cache.len() {0.0@@ -97,7 +100,7 @@ impl Signal {}}Signal::UpdateTimestamp => {- let update_timestamp = fastfield.unwrap().get_val(doc as u64) as i64;+ let update_timestamp = fastfield_value.unwrap() as i64;if current_timestamp as i64 - update_timestamp <= 0 {return 0.0;@@ -113,11 +116,11 @@ impl Signal {}}Signal::NumTrackers => {- let num_trackers = fastfield.unwrap().get_val(doc as u64) as f64;+ let num_trackers = fastfield_value.unwrap() as f64;1.0 / (num_trackers + 1.0)}Signal::Region => {- let webpage_region = Region::from_id(fastfield.unwrap().get_val(doc as u64));+ let webpage_region = Region::from_id(fastfield_value.unwrap());let boost =selected_region.map_or(@@ -308,16 +311,15 @@ impl SignalAggregator {ALL_SIGNALS.into_iter().map(|signal| {- let reader = match self.readers.get(signal as usize) {- Some(Some(reader)) => Some(reader),+ let fastfield_value = match self.readers.get(signal as usize) {+ Some(Some(reader)) => Some(reader.get_val(doc as u64)),_ => None,};self.coefficients().get(&signal)* signal.value(- doc,bm25,- reader,+ fastfield_value,region_count,current_timestamp,selected_region,@@ -327,6 +329,51 @@ impl SignalAggregator {.sum()}+ pub fn precompute_score(&self, webpage: &Webpage, region_count: &RegionCount) -> f64 {+ ALL_SIGNALS+ .into_iter()+ .filter(|signal| signal.is_computable_before_search())+ .map(|signal| {+ let fastfield_value = match &signal {+ Signal::HostCentrality => {+ (webpage.host_centrality * (CENTRALITY_SCALING as f64)) as u64+ }+ Signal::PageCentrality => {+ (webpage.page_centrality * (CENTRALITY_SCALING as f64)) as u64+ }+ Signal::IsHomepage => {+ if webpage.html.url().is_homepage() {+ 1+ } else {+ 0+ }+ }+ Signal::FetchTimeMs => webpage.fetch_time_ms,+ Signal::UpdateTimestamp => webpage+ .html+ .updated_time()+ .map(|date| date.timestamp().max(0) as u64)+ .unwrap_or(0),+ Signal::NumTrackers => webpage.html.trackers().len() as u64,+ Signal::Region => Region::guess_from(webpage).unwrap_or(Region::All).id(),+ _ => panic!("signal cannot be determined from webpage"),+ };++ let current_timestamp = Utc::now().timestamp() as usize;++ self.coefficients().get(&signal)+ * signal.value(+ 0.0,+ Some(fastfield_value),+ ®ion_count,+ current_timestamp,+ None,+ self,+ )+ })+ .sum()+ }+pub fn coefficients(&self) -> &SignalCoefficient {&self.signal_coefficients}diff --git a/src/schema.rs b/src/schema.rsindex 3c0ce05e..8ef7d10f 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -54,9 +54,10 @@ pub enum Field {NumDescriptionTokens,SiteHash,UrlWithoutQueryHash,+ PreComputedScore,}-pub static ALL_FIELDS: [Field; 28] = [+pub static ALL_FIELDS: [Field; 29] = [Field::Title,Field::CleanBody,Field::StemmedTitle,@@ -85,6 +86,7 @@ pub static ALL_FIELDS: [Field; 28] = [Field::NumDescriptionTokens,Field::SiteHash,Field::UrlWithoutQueryHash,+ Field::PreComputedScore,];impl Field {@@ -130,18 +132,18 @@ impl Field {.set_index_option(IndexRecordOption::WithFreqsAndPositions),),),- Field::IsHomepage => IndexingOption::Numeric(+ Field::IsHomepage => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),Field::BacklinkText => IndexingOption::Text(self.default_text_options()),- Field::HostCentrality => IndexingOption::Numeric(+ Field::HostCentrality => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::PageCentrality => IndexingOption::Numeric(+ Field::PageCentrality => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),@@ -153,56 +155,62 @@ impl Field {self.default_text_options_with_tokenizer(Stemmed::as_str()).set_stored(),),- Field::FetchTimeMs => IndexingOption::Numeric(+ Field::FetchTimeMs => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumTrackers => IndexingOption::Numeric(+ Field::NumTrackers => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),Field::PrimaryImage => IndexingOption::Bytes(BytesOptions::default().set_stored()),- Field::LastUpdated => IndexingOption::Numeric(+ Field::LastUpdated => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_stored().set_indexed(),),Field::Description => IndexingOption::Text(self.default_text_options().set_stored()),- Field::Region => IndexingOption::Numeric(+ Field::Region => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_stored().set_indexed(),),- Field::NumCleanBodyTokens => IndexingOption::Numeric(+ Field::NumCleanBodyTokens => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumDescriptionTokens => IndexingOption::Numeric(+ Field::NumDescriptionTokens => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumTitleTokens => IndexingOption::Numeric(+ Field::NumTitleTokens => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::NumUrlTokens => IndexingOption::Numeric(+ Field::NumUrlTokens => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),),- Field::SiteHash => IndexingOption::Numeric(+ Field::SiteHash => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),- Field::UrlWithoutQueryHash => IndexingOption::Numeric(+ Field::UrlWithoutQueryHash => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),+ Field::PreComputedScore => IndexingOption::Float(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed()+ .set_stored(),+ ),}}@@ -236,6 +244,7 @@ impl Field {Field::NumDescriptionTokens => "num_description_tokens",Field::SiteHash => "site_hash",Field::UrlWithoutQueryHash => "url_without_query_hash",+ Field::PreComputedScore => "pre_computed_score",}}@@ -267,6 +276,7 @@ impl Field {| Field::NumCleanBodyTokens| Field::NumDescriptionTokens| Field::Region+ | Field::PreComputedScore| Field::LastUpdated => None,}}@@ -290,6 +300,7 @@ impl Field {| Field::NumDescriptionTokens| Field::SiteHash| Field::UrlWithoutQueryHash+ | Field::PreComputedScore)}@@ -317,6 +328,7 @@ impl Field {"site_hash" => Some(Field::SiteHash),"url_without_query_hash" => Some(Field::UrlWithoutQueryHash),"domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),+ "pre_computed_score" => Some(Field::PreComputedScore),_ => None,}}@@ -328,7 +340,8 @@ pub fn create_schema() -> tantivy::schema::Schema {for field in &ALL_FIELDS {match field.options() {IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),- IndexingOption::Numeric(options) => builder.add_u64_field(field.as_str(), options),+ IndexingOption::Integer(options) => builder.add_u64_field(field.as_str(), options),+ IndexingOption::Float(options) => builder.add_f64_field(field.as_str(), options),IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),};}@@ -338,6 +351,7 @@ pub fn create_schema() -> tantivy::schema::Schema {pub enum IndexingOption {Text(tantivy::schema::TextOptions),- Numeric(tantivy::schema::NumericOptions),+ Integer(tantivy::schema::NumericOptions),+ Float(tantivy::schema::NumericOptions),Bytes(tantivy::schema::BytesOptions),}diff --git a/src/searcher.rs b/src/searcher.rsindex 2df572f4..58b9dd20 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -119,6 +119,8 @@ impl Searcher {}}+ ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());+let webpages = self.index.search(&query, ranker.collector())?;let correction = self.index.spell_correction(&query.simple_terms());@@ -204,6 +206,7 @@ mod tests {host_centrality: (NUM_WEBSITES - i) as f64,fetch_time_ms: 500,page_centrality: 0.0,+ pre_computed_score: 0.0,primary_image: None,}).expect("failed to insert webpage");diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 1da074dc..62bfe623 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -125,6 +125,7 @@ pub struct Webpage {pub host_centrality: f64,pub page_centrality: f64,pub fetch_time_ms: u64,+ pub pre_computed_score: f64,pub primary_image: Option<StoredPrimaryImage>,}@@ -139,6 +140,7 @@ impl Webpage {host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 0,+ pre_computed_score: 0.0,primary_image: None,}}@@ -198,6 +200,13 @@ impl Webpage {self.fetch_time_ms,);+ doc.add_f64(+ schema+ .get_field(Field::PreComputedScore.as_str())+ .expect("failed to get pre_computed_score field"),+ self.pre_computed_score,+ );+let image = bincode::serialize(&self.primary_image).unwrap();doc.add_bytes(schema@@ -644,6 +653,7 @@ impl Html {| Field::HostCentrality| Field::PageCentrality| Field::FetchTimeMs+ | Field::PreComputedScore| Field::Region| Field::PrimaryImage => {}}