Skip to content

web search done right

Ftr/sort postings by score (#57)

Mikkel Denkerb902a3b7aa0f
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 961b05e8..adc77d5a 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -152,9 +152,9 @@ import Header from "../components/Header.astro";
<Hero class="change-page-active" icon="chevron-right" />
</a>
</div>
- <div class="flex flex-col w-full mt-5 pl-4 pr-24">
+ <div class="flex flex-col w-full mt-7 pl-4 pr-24">
Didn't find what you were looking for? Try on:
- <div class="flex w-full justify-between mt-2">
+ <div class="flex w-full justify-between mt-4">
<div class="bg-neutral-100 rounded-full py-2 px-5">
<a href="https://www.google.com/search?q={{ query|urlencode }}"
>Google</a
diff --git a/src/collector.rs b/src/collector.rs
index c826d74f..6df5752c 100644
--- a/src/collector.rs
+++ b/src/collector.rs
@@ -31,14 +31,25 @@ fn adjust_score(num_taken: usize, original_score: f64) -> f64 {
original_score * (SCALE / (num_taken as f64 + SCALE))
}
+#[derive(Clone)]
+pub struct MaxDocsConsidered {
+ pub total_docs: usize,
+ pub segments: usize,
+}
+
pub struct TopDocs {
top_n: usize,
offset: usize,
+ max_docs: Option<MaxDocsConsidered>,
}
impl TopDocs {
pub fn with_limit(top_n: usize) -> Self {
- Self { top_n, offset: 0 }
+ Self {
+ top_n,
+ offset: 0,
+ max_docs: None,
+ }
}
pub fn and_offset(mut self, offset: usize) -> Self {
@@ -46,6 +57,12 @@ impl TopDocs {
self
}
+ pub fn and_max_docs(mut self, max_docs: MaxDocsConsidered) -> Self {
+ self.max_docs = Some(max_docs);
+
+ self
+ }
+
pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(
self,
score_tweaker: TScoreTweaker,
@@ -75,8 +92,15 @@ impl Collector for TopDocs {
.unwrap(),
)?;
+ let max_docs = self
+ .max_docs
+ .as_ref()
+ .map(|max_docs| max_docs.total_docs / max_docs.segments);
+
Ok(TopSegmentCollector {
key_reader,
+ max_docs,
+ num_docs_taken: 0,
segment_ord: segment_local_id,
bucket_collector: BucketCollector::new(self.top_n + self.offset),
})
@@ -117,6 +141,8 @@ impl Collector for TopDocs {
pub struct TopSegmentCollector {
key_reader: MultiValuedFastFieldReader<u64>,
+ max_docs: Option<usize>,
+ num_docs_taken: usize,
segment_ord: SegmentOrdinal,
bucket_collector: BucketCollector,
}
@@ -125,6 +151,13 @@ impl SegmentCollector for TopSegmentCollector {
type Fruit = Vec<Doc>;
fn collect(&mut self, doc: DocId, score: Score) {
+ if let Some(max_docs) = &self.max_docs {
+ if self.num_docs_taken >= *max_docs {
+ return;
+ }
+
+ self.num_docs_taken += 1;
+ }
let mut keys = Vec::new();
self.key_reader.get_vals(doc, &mut keys);
debug_assert_eq!(keys.len(), 2);
@@ -137,7 +170,7 @@ impl SegmentCollector for TopSegmentCollector {
id: doc,
segment: self.segment_ord,
score: score as f64,
- })
+ });
}
fn harvest(self) -> Self::Fruit {
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index c0070087..8306c2a1 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -25,6 +25,7 @@ use crate::entrypoint::download_all_warc_files;
use crate::index::{FrozenIndex, Index};
use crate::mapreduce::{Map, MapReduce, Reduce, Worker};
use crate::ranking::centrality_store::CentralityStore;
+use crate::ranking::SignalAggregator;
use crate::warc::WarcFile;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
use crate::webpage::{Html, Link, Webpage};
@@ -86,6 +87,7 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
};
let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);
+ let signal_aggregator = SignalAggregator::default();
for file in warc_files {
if let Ok(file) = WarcFile::open(&file) {
@@ -131,14 +133,19 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
trace!("title = {:?}", html.title());
trace!("text = {:?}", html.clean_text());
- let webpage = Webpage {
+ let mut webpage = Webpage {
html,
backlinks,
page_centrality,
host_centrality,
fetch_time_ms,
primary_image: None,
+ pre_computed_score: 0.0,
};
+
+ webpage.pre_computed_score =
+ signal_aggregator.precompute_score(&webpage, &index.region_count);
+
if let Err(err) = index.insert(webpage) {
debug!("{:?}", err);
}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 88305fb6..a257a46d 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -62,7 +62,7 @@ pub struct DisplayedWebpage {
}
const MAX_PRETTY_URL_LEN: usize = 50;
-const MAX_TITLE_LEN: usize = 50;
+const MAX_TITLE_LEN: usize = 70;
fn prettify_url(url: &Url) -> String {
let mut pretty_url = url.strip_query().to_string();
diff --git a/src/index.rs b/src/index.rs
index 06484075..81fa492d 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -250,6 +250,10 @@ impl Index {
pub fn spell_correction(&self, terms: &[String]) -> Option<String> {
self.spell_check(terms).or_else(|| self.split_words(terms))
}
+
+ pub fn num_segments(&self) -> usize {
+ self.inverted_index.num_segments()
+ }
}
#[derive(Debug, Serialize, Deserialize)]
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 25163e5d..e3c991b9 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -17,6 +17,7 @@
use chrono::NaiveDateTime;
use serde::Serialize;
use tantivy::collector::{Collector, Count};
+use tantivy::directory::MmapDirectory;
use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
@@ -52,10 +53,20 @@ impl InvertedIndex {
let schema = create_schema();
let mut tantivy_index = if path.as_ref().exists() {
- tantivy::Index::open_in_dir(&path)?
+ let mmap_directory = MmapDirectory::open(&path)?;
+ tantivy::Index::open(mmap_directory)?
} else {
+ let index_settings = tantivy::IndexSettings {
+ sort_by_field: Some(tantivy::IndexSortByField {
+ field: Field::PreComputedScore.as_str().to_string(),
+ order: tantivy::Order::Desc,
+ }),
+ ..Default::default()
+ };
+
fs::create_dir_all(&path)?;
- tantivy::Index::create_in_dir(&path, schema.clone())?
+ let mmap_directory = MmapDirectory::open(&path)?;
+ tantivy::Index::create(mmap_directory, schema.clone(), index_settings)?
};
tantivy_index.set_default_multithread_executor()?;
@@ -271,6 +282,10 @@ impl InvertedIndex {
pub fn schema(&self) -> Arc<Schema> {
Arc::clone(&self.schema)
}
+
+ pub fn num_segments(&self) -> usize {
+ self.tantivy_index.searchable_segments().unwrap().len()
+ }
}
#[derive(Debug, Serialize)]
@@ -377,6 +392,7 @@ impl From<Document> for RetrievedWebpage {
| Field::SiteHash
| Field::UrlWithoutQueryHash
| Field::NumTrackers
+ | Field::PreComputedScore
| Field::NumCleanBodyTokens
| Field::NumDescriptionTokens
| Field::NumTitleTokens
@@ -614,6 +630,7 @@ mod tests {
host_centrality: 1.0,
page_centrality: 0.0,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
diff --git a/src/lib.rs b/src/lib.rs
index be985de5..7e5311be 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -195,6 +195,9 @@ pub enum Error {
#[error("String is not float")]
ParseFloat(#[from] std::num::ParseFloatError),
+
+ #[error("Could not open inverted-index directory")]
+ Directory(#[from] tantivy::directory::error::OpenDirectoryError),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index d7a7bf69..5b23a169 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -327,6 +327,7 @@ mod tests {
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -351,6 +352,7 @@ mod tests {
host_centrality: 0.0001,
page_centrality: 0.0,
primary_image: None,
+ pre_computed_score: 0.0,
fetch_time_ms: 500,
})
.expect("failed to insert webpage");
@@ -447,6 +449,7 @@ mod tests {
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -471,6 +474,7 @@ mod tests {
host_centrality: 0.0001,
page_centrality: 0.0,
primary_image: None,
+ pre_computed_score: 0.0,
fetch_time_ms: 500,
})
.expect("failed to insert webpage");
@@ -546,6 +550,7 @@ mod tests {
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -570,6 +575,7 @@ mod tests {
host_centrality: 0.0001,
page_centrality: 0.0,
primary_image: None,
+ pre_computed_score: 0.0,
fetch_time_ms: 500,
})
.expect("failed to insert webpage");
@@ -594,6 +600,7 @@ mod tests {
host_centrality: 0.0001,
page_centrality: 0.0,
primary_image: None,
+ pre_computed_score: 0.0,
fetch_time_ms: 500,
})
.expect("failed to insert webpage");
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 5c7dc382..bf2976c2 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -26,7 +26,7 @@ use initial::InitialScoreTweaker;
use tantivy::collector::Collector;
use crate::{
- collector::TopDocs,
+ collector::{MaxDocsConsidered, TopDocs},
searcher::NUM_RESULTS_PER_PAGE,
webpage::region::{Region, RegionCount},
};
@@ -36,6 +36,7 @@ pub use self::signal::*;
pub struct Ranker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
+ max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
aggregator: SignalAggregator,
}
@@ -47,6 +48,7 @@ impl Ranker {
selected_region: None,
offset: None,
aggregator,
+ max_docs: None,
}
}
@@ -60,6 +62,14 @@ impl Ranker {
self
}
+ pub fn with_max_docs(mut self, total_docs: usize, segments: usize) -> Self {
+ self.max_docs = Some(MaxDocsConsidered {
+ total_docs,
+ segments,
+ });
+ self
+ }
+
pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
let score_tweaker = InitialScoreTweaker::new(
Arc::clone(&self.region_count),
@@ -73,6 +83,10 @@ impl Ranker {
collector = collector.and_offset(offset);
}
+ if let Some(max_docs) = &self.max_docs {
+ collector = collector.and_max_docs(max_docs.clone());
+ }
+
collector.tweak_score(score_tweaker)
}
}
@@ -113,6 +127,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -137,6 +152,7 @@ mod tests {
host_centrality: 5.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -179,6 +195,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -203,6 +220,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 5.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -244,6 +262,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -268,6 +287,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -290,6 +310,7 @@ mod tests {
host_centrality: 0.003,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -332,6 +353,7 @@ mod tests {
host_centrality: 0.092,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -356,6 +378,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.09,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
})
@@ -395,6 +418,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -429,6 +453,7 @@ mod tests {
host_centrality: 0.003,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -472,6 +497,7 @@ mod tests {
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -493,6 +519,7 @@ mod tests {
backlinks: vec![],
host_centrality: 0.003,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
})
@@ -532,6 +559,7 @@ mod tests {
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
+ pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
})
@@ -556,6 +584,7 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 20,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -580,6 +609,7 @@ mod tests {
host_centrality: 1.02,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
@@ -665,6 +695,7 @@ mod tests {
),
backlinks: vec![],
host_centrality: 1.0,
+ pre_computed_score: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
primary_image: None,
@@ -690,6 +721,7 @@ mod tests {
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
})
@@ -714,6 +746,7 @@ mod tests {
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
+ pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
})
@@ -759,6 +792,7 @@ mod tests {
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
+ pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
})
@@ -784,6 +818,7 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 5000,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index 566229c2..f81bb59c 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -14,9 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::Result;
-use std::{array, convert::TryFrom, sync::Arc};
+use crate::{webpage::Webpage, Result};
+use std::{array, convert::TryFrom, ops::Deref, sync::Arc};
+use chrono::Utc;
use tantivy::{
fastfield::{Column, DynamicFastFieldReader},
DocId, Score, SegmentReader,
@@ -70,13 +71,15 @@ impl Signal {
!matches!(self, Signal::Bm25)
}
- #[allow(clippy::too_many_arguments)]
+ fn is_computable_before_search(&self) -> bool {
+ todo!();
+ }
+
fn value(
&self,
- doc: DocId,
bm25: Score,
- fastfield: Option<&DynamicFastFieldReader<u64>>,
- region_count: &Arc<RegionCount>,
+ fastfield_value: Option<u64>,
+ region_count: &impl Deref<Target = RegionCount>,
current_timestamp: usize,
selected_region: Option<Region>,
aggregator: &SignalAggregator,
@@ -84,11 +87,11 @@ impl Signal {
match self {
Signal::Bm25 => bm25 as f64,
Signal::HostCentrality | Signal::PageCentrality => {
- fastfield.unwrap().get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64
+ fastfield_value.unwrap() as f64 / CENTRALITY_SCALING as f64
}
- Signal::IsHomepage => fastfield.unwrap().get_val(doc as u64) as f64,
+ Signal::IsHomepage => fastfield_value.unwrap() as f64,
Signal::FetchTimeMs => {
- let fetch_time_ms = fastfield.unwrap().get_val(doc as u64) as usize;
+ let fetch_time_ms = fastfield_value.unwrap() as usize;
if fetch_time_ms >= aggregator.fetch_time_ms_cache.len() {
0.0
@@ -97,7 +100,7 @@ impl Signal {
}
}
Signal::UpdateTimestamp => {
- let update_timestamp = fastfield.unwrap().get_val(doc as u64) as i64;
+ let update_timestamp = fastfield_value.unwrap() as i64;
if current_timestamp as i64 - update_timestamp <= 0 {
return 0.0;
@@ -113,11 +116,11 @@ impl Signal {
}
}
Signal::NumTrackers => {
- let num_trackers = fastfield.unwrap().get_val(doc as u64) as f64;
+ let num_trackers = fastfield_value.unwrap() as f64;
1.0 / (num_trackers + 1.0)
}
Signal::Region => {
- let webpage_region = Region::from_id(fastfield.unwrap().get_val(doc as u64));
+ let webpage_region = Region::from_id(fastfield_value.unwrap());
let boost =
selected_region.map_or(
@@ -308,16 +311,15 @@ impl SignalAggregator {
ALL_SIGNALS
.into_iter()
.map(|signal| {
- let reader = match self.readers.get(signal as usize) {
- Some(Some(reader)) => Some(reader),
+ let fastfield_value = match self.readers.get(signal as usize) {
+ Some(Some(reader)) => Some(reader.get_val(doc as u64)),
_ => None,
};
self.coefficients().get(&signal)
* signal.value(
- doc,
bm25,
- reader,
+ fastfield_value,
region_count,
current_timestamp,
selected_region,
@@ -327,6 +329,51 @@ impl SignalAggregator {
.sum()
}
+ pub fn precompute_score(&self, webpage: &Webpage, region_count: &RegionCount) -> f64 {
+ ALL_SIGNALS
+ .into_iter()
+ .filter(|signal| signal.is_computable_before_search())
+ .map(|signal| {
+ let fastfield_value = match &signal {
+ Signal::HostCentrality => {
+ (webpage.host_centrality * (CENTRALITY_SCALING as f64)) as u64
+ }
+ Signal::PageCentrality => {
+ (webpage.page_centrality * (CENTRALITY_SCALING as f64)) as u64
+ }
+ Signal::IsHomepage => {
+ if webpage.html.url().is_homepage() {
+ 1
+ } else {
+ 0
+ }
+ }
+ Signal::FetchTimeMs => webpage.fetch_time_ms,
+ Signal::UpdateTimestamp => webpage
+ .html
+ .updated_time()
+ .map(|date| date.timestamp().max(0) as u64)
+ .unwrap_or(0),
+ Signal::NumTrackers => webpage.html.trackers().len() as u64,
+ Signal::Region => Region::guess_from(webpage).unwrap_or(Region::All).id(),
+ _ => panic!("signal cannot be determined from webpage"),
+ };
+
+ let current_timestamp = Utc::now().timestamp() as usize;
+
+ self.coefficients().get(&signal)
+ * signal.value(
+ 0.0,
+ Some(fastfield_value),
+ &region_count,
+ current_timestamp,
+ None,
+ self,
+ )
+ })
+ .sum()
+ }
+
pub fn coefficients(&self) -> &SignalCoefficient {
&self.signal_coefficients
}
diff --git a/src/schema.rs b/src/schema.rs
index 3c0ce05e..8ef7d10f 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -54,9 +54,10 @@ pub enum Field {
NumDescriptionTokens,
SiteHash,
UrlWithoutQueryHash,
+ PreComputedScore,
}
-pub static ALL_FIELDS: [Field; 28] = [
+pub static ALL_FIELDS: [Field; 29] = [
Field::Title,
Field::CleanBody,
Field::StemmedTitle,
@@ -85,6 +86,7 @@ pub static ALL_FIELDS: [Field; 28] = [
Field::NumDescriptionTokens,
Field::SiteHash,
Field::UrlWithoutQueryHash,
+ Field::PreComputedScore,
];
impl Field {
@@ -130,18 +132,18 @@ impl Field {
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
),
),
- Field::IsHomepage => IndexingOption::Numeric(
+ Field::IsHomepage => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
- Field::HostCentrality => IndexingOption::Numeric(
+ Field::HostCentrality => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::PageCentrality => IndexingOption::Numeric(
+ Field::PageCentrality => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
@@ -153,56 +155,62 @@ impl Field {
self.default_text_options_with_tokenizer(Stemmed::as_str())
.set_stored(),
),
- Field::FetchTimeMs => IndexingOption::Numeric(
+ Field::FetchTimeMs => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumTrackers => IndexingOption::Numeric(
+ Field::NumTrackers => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
Field::PrimaryImage => IndexingOption::Bytes(BytesOptions::default().set_stored()),
- Field::LastUpdated => IndexingOption::Numeric(
+ Field::LastUpdated => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_stored()
.set_indexed(),
),
Field::Description => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::Region => IndexingOption::Numeric(
+ Field::Region => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_stored()
.set_indexed(),
),
- Field::NumCleanBodyTokens => IndexingOption::Numeric(
+ Field::NumCleanBodyTokens => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumDescriptionTokens => IndexingOption::Numeric(
+ Field::NumDescriptionTokens => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumTitleTokens => IndexingOption::Numeric(
+ Field::NumTitleTokens => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::NumUrlTokens => IndexingOption::Numeric(
+ Field::NumUrlTokens => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::SiteHash => IndexingOption::Numeric(
+ Field::SiteHash => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
- Field::UrlWithoutQueryHash => IndexingOption::Numeric(
+ Field::UrlWithoutQueryHash => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
+ Field::PreComputedScore => IndexingOption::Float(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed()
+ .set_stored(),
+ ),
}
}
@@ -236,6 +244,7 @@ impl Field {
Field::NumDescriptionTokens => "num_description_tokens",
Field::SiteHash => "site_hash",
Field::UrlWithoutQueryHash => "url_without_query_hash",
+ Field::PreComputedScore => "pre_computed_score",
}
}
@@ -267,6 +276,7 @@ impl Field {
| Field::NumCleanBodyTokens
| Field::NumDescriptionTokens
| Field::Region
+ | Field::PreComputedScore
| Field::LastUpdated => None,
}
}
@@ -290,6 +300,7 @@ impl Field {
| Field::NumDescriptionTokens
| Field::SiteHash
| Field::UrlWithoutQueryHash
+ | Field::PreComputedScore
)
}
@@ -317,6 +328,7 @@ impl Field {
"site_hash" => Some(Field::SiteHash),
"url_without_query_hash" => Some(Field::UrlWithoutQueryHash),
"domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),
+ "pre_computed_score" => Some(Field::PreComputedScore),
_ => None,
}
}
@@ -328,7 +340,8 @@ pub fn create_schema() -> tantivy::schema::Schema {
for field in &ALL_FIELDS {
match field.options() {
IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),
- IndexingOption::Numeric(options) => builder.add_u64_field(field.as_str(), options),
+ IndexingOption::Integer(options) => builder.add_u64_field(field.as_str(), options),
+ IndexingOption::Float(options) => builder.add_f64_field(field.as_str(), options),
IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),
};
}
@@ -338,6 +351,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
pub enum IndexingOption {
Text(tantivy::schema::TextOptions),
- Numeric(tantivy::schema::NumericOptions),
+ Integer(tantivy::schema::NumericOptions),
+ Float(tantivy::schema::NumericOptions),
Bytes(tantivy::schema::BytesOptions),
}
diff --git a/src/searcher.rs b/src/searcher.rs
index 2df572f4..58b9dd20 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -119,6 +119,8 @@ impl Searcher {
}
}
+ ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());
+
let webpages = self.index.search(&query, ranker.collector())?;
let correction = self.index.spell_correction(&query.simple_terms());
@@ -204,6 +206,7 @@ mod tests {
host_centrality: (NUM_WEBSITES - i) as f64,
fetch_time_ms: 500,
page_centrality: 0.0,
+ pre_computed_score: 0.0,
primary_image: None,
})
.expect("failed to insert webpage");
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 1da074dc..62bfe623 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -125,6 +125,7 @@ pub struct Webpage {
pub host_centrality: f64,
pub page_centrality: f64,
pub fetch_time_ms: u64,
+ pub pre_computed_score: f64,
pub primary_image: Option<StoredPrimaryImage>,
}
@@ -139,6 +140,7 @@ impl Webpage {
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 0,
+ pre_computed_score: 0.0,
primary_image: None,
}
}
@@ -198,6 +200,13 @@ impl Webpage {
self.fetch_time_ms,
);
+ doc.add_f64(
+ schema
+ .get_field(Field::PreComputedScore.as_str())
+ .expect("failed to get pre_computed_score field"),
+ self.pre_computed_score,
+ );
+
let image = bincode::serialize(&self.primary_image).unwrap();
doc.add_bytes(
schema
@@ -644,6 +653,7 @@ impl Html {
| Field::HostCentrality
| Field::PageCentrality
| Field::FetchTimeMs
+ | Field::PreComputedScore
| Field::Region
| Field::PrimaryImage => {}
}