Skip to content

web search done right

prepare for ltr models

Mikkel Denker44f61b98ce9d
diff --git a/.gitignore b/.gitignore
index 4c3b39d8..c85885ca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,4 @@ notes
*.log
/data/**/*
*.tar.gz
-
+*.csv
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index ced98610..d92b4969 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -103,7 +103,6 @@ pub async fn route(
.get("p")
.and_then(|p| p.parse().ok())
.unwrap_or_default();
- let offset = skip_pages * NUM_RESULTS_PER_PAGE;
let mut optic = None;
// should only be set if optic was successfully downloaded
@@ -136,7 +135,7 @@ pub async fn route(
query: query.clone(),
selected_region,
optic_program: optic,
- offset,
+ page: skip_pages,
site_rankings,
num_results: NUM_RESULTS_PER_PAGE,
})
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 9dbe950d..2cbf5999 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -1,3 +1,4 @@
+use chrono::Utc;
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
@@ -95,7 +96,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
let warc_files = async_download_all_warc_files(&job.warc_paths, &source, &job.base_path).await;
pin!(warc_files);
- let signal_aggregator = SignalAggregator::default();
+ let current_timestamp = Utc::now().timestamp().max(0) as usize;
while let Some(file) = warc_files.next().await {
let name = file.split('/').last().unwrap();
@@ -210,8 +211,10 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
dmoz_description,
};
- webpage.pre_computed_score =
- signal_aggregator.precompute_score(&webpage, &index.region_count);
+ let mut signal_aggregator = SignalAggregator::new(None);
+ signal_aggregator.set_current_timestamp(current_timestamp);
+
+ webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);
if let Err(err) = index.insert(webpage) {
debug!("{:?}", err);
diff --git a/core/src/enum_map.rs b/core/src/enum_map.rs
new file mode 100644
index 00000000..a42e166c
--- /dev/null
+++ b/core/src/enum_map.rs
@@ -0,0 +1,118 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct EnumMap<K: Into<usize>, V> {
+ inner: Vec<Option<V>>,
+ _phantom: std::marker::PhantomData<K>,
+}
+
+impl<K: Into<usize>, V> Default for EnumMap<K, V> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<K, V> EnumMap<K, V>
+where
+ K: Into<usize>,
+{
+ pub fn new() -> Self {
+ Self {
+ inner: vec![],
+ _phantom: std::marker::PhantomData,
+ }
+ }
+
+ pub fn insert(&mut self, key: K, value: V) {
+ let key = key.into();
+
+ if key >= self.inner.len() {
+ self.inner.resize_with(key + 1, || None);
+ }
+
+ self.inner[key] = Some(value);
+ }
+
+ pub fn get(&self, key: K) -> Option<&V> {
+ let key = key.into();
+ if key >= self.inner.len() {
+ None
+ } else {
+ self.inner[key].as_ref()
+ }
+ }
+
+ pub fn values_mut(&mut self) -> impl Iterator<Item = &mut V> {
+ self.inner.iter_mut().filter_map(|value| value.as_mut())
+ }
+
+ pub fn get_mut(&mut self, key: K) -> Option<&mut V> {
+ let key = key.into();
+ if key >= self.inner.len() {
+ None
+ } else {
+ self.inner[key].as_mut()
+ }
+ }
+}
+
+impl<K, V> FromIterator<(K, V)> for EnumMap<K, V>
+where
+ K: Into<usize>,
+{
+ fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
+ let mut map = Self::new();
+
+ for (key, value) in iter {
+ map.insert(key, value);
+ }
+
+ map
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+ enum TestEnum {
+ A,
+ B,
+ C,
+ }
+
+ impl From<TestEnum> for usize {
+ fn from(val: TestEnum) -> Self {
+ val as usize
+ }
+ }
+
+ #[test]
+ fn test_enum_map() {
+ let mut map = EnumMap::new();
+ map.insert(TestEnum::A, TestEnum::B);
+ map.insert(TestEnum::B, TestEnum::C);
+ map.insert(TestEnum::C, TestEnum::A);
+
+ assert_eq!(map.get(TestEnum::A), Some(&TestEnum::B));
+ assert_eq!(map.get(TestEnum::B), Some(&TestEnum::C));
+ assert_eq!(map.get(TestEnum::C), Some(&TestEnum::A));
+ }
+}
diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rs
index e335d8f6..47e0457a 100644
--- a/core/src/fastfield_reader.rs
+++ b/core/src/fastfield_reader.rs
@@ -19,8 +19,8 @@ use std::{collections::HashMap, sync::Arc};
use tantivy::{fastfield::Column, DocId, SegmentId};
use crate::{
+ enum_map::EnumMap,
schema::{DataType, FastField, Field, ALL_FIELDS},
- search_ctx::Ctx,
};
#[derive(Default, Clone)]
@@ -35,12 +35,12 @@ impl FastFieldReader {
}
impl FastFieldReader {
- pub fn new(ctx: &Ctx) -> Self {
+ pub fn new(tv_searcher: &tantivy::Searcher) -> Self {
let mut segments = HashMap::new();
- let schema = ctx.tv_searcher.schema();
+ let schema = tv_searcher.schema();
- for reader in ctx.tv_searcher.segment_readers() {
+ for reader in tv_searcher.segment_readers() {
let fastfield_readers = reader.fast_fields();
let mut field_readers = Vec::new();
@@ -67,10 +67,7 @@ impl FastFieldReader {
segments.insert(
reader.segment_id(),
Arc::new(SegmentReader {
- field_readers: field_readers
- .into_iter()
- .map(|(_, reader)| reader)
- .collect(),
+ field_readers: field_readers.into_iter().collect(),
}),
);
}
@@ -118,11 +115,11 @@ impl FieldReader {
}
pub struct SegmentReader {
- field_readers: Vec<FieldReader>, // fast_field -> field_reader
+ field_readers: EnumMap<FastField, FieldReader>,
}
impl SegmentReader {
pub fn get_field_reader(&self, field: &FastField) -> &FieldReader {
- self.field_readers.get(*field as usize).unwrap()
+ self.field_readers.get(*field).unwrap()
}
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 18f4cd91..43a5e235 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -149,8 +149,8 @@ impl InvertedIndex {
})
}
- pub fn fastfield_reader(&self, ctx: &Ctx) -> FastFieldReader {
- FastFieldReader::new(ctx)
+ pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {
+ FastFieldReader::new(tv_searcher)
}
pub fn tokenizers(&self) -> &TokenizerManager {
@@ -194,25 +194,53 @@ impl InvertedIndex {
}
pub fn local_search_ctx(&self) -> Ctx {
+ let tv_searcher = self.tv_searcher();
Ctx {
- tv_searcher: self.reader.searcher(),
+ fastfield_reader: self.fastfield_reader(&tv_searcher),
+ tv_searcher,
}
}
+ pub fn tv_searcher(&self) -> tantivy::Searcher {
+ self.reader.searcher()
+ }
+
pub fn retrieve_ranking_websites(
&self,
ctx: &Ctx,
pointers: Vec<WebsitePointer>,
- aggregator: &SignalAggregator,
+ mut aggregator: SignalAggregator,
+ fastfield_reader: &FastFieldReader,
) -> Result<Vec<RankingWebsite>> {
let mut top_websites = Vec::new();
- for pointer in pointers {
- let doc = ctx.tv_searcher.doc(pointer.address.into())?;
- top_websites.push(RankingWebsite::new(doc, pointer, aggregator));
+ let mut pointers: Vec<_> = pointers.into_iter().enumerate().collect();
+ pointers.sort_by(|a, b| a.1.address.segment.cmp(&b.1.address.segment));
+ let mut prev_segment = None;
+ for (orig_index, pointer) in pointers {
+ let update_segment = match prev_segment {
+ Some(prev_segment) if prev_segment != pointer.address.segment => true,
+ None => true,
+ _ => false,
+ };
+
+ if update_segment {
+ let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);
+ let fastfield_reader = fastfield_reader.get_segment(&segment_reader.segment_id());
+ aggregator.register_segment(&ctx.tv_searcher, segment_reader, fastfield_reader)?;
+ }
+
+ prev_segment = Some(pointer.address.segment);
+
+ top_websites.push((orig_index, RankingWebsite::new(pointer, &mut aggregator)));
}
- Ok(top_websites)
+ top_websites.sort_by(|a, b| a.0.cmp(&b.0));
+
+ Ok(top_websites
+ .into_iter()
+ .map(|(_, website)| website)
+ .collect())
}
pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {
@@ -236,14 +264,14 @@ impl InvertedIndex {
websites: &[WebsitePointer],
query: &Query,
) -> Result<Vec<RetrievedWebpage>> {
- let ctx = self.local_search_ctx();
+ let tv_searcher = self.reader.searcher();
let mut webpages: Vec<RetrievedWebpage> = websites
.iter()
- .map(|website| self.retrieve_doc(website.address, &ctx))
+ .map(|website| self.retrieve_doc(website.address, &tv_searcher))
.filter_map(|res| res.ok())
.map(|mut doc| {
if let Some(image) = doc.primary_image.as_ref() {
- if !query.simple_terms().into_iter().all(|term| {
+ if !query.simple_terms().iter().all(|term| {
image
.title_terms
.contains(term.to_ascii_lowercase().as_str())
@@ -321,8 +349,12 @@ impl InvertedIndex {
Ok(())
}
- fn retrieve_doc(&self, doc_address: DocAddress, ctx: &Ctx) -> Result<RetrievedWebpage> {
- let doc = ctx.tv_searcher.doc(doc_address.into())?;
+ fn retrieve_doc(
+ &self,
+ doc_address: DocAddress,
+ searcher: &tantivy::Searcher,
+ ) -> Result<RetrievedWebpage> {
+ let doc = searcher.doc(doc_address.into())?;
Ok(RetrievedWebpage::from(doc))
}
@@ -398,9 +430,8 @@ impl InvertedIndex {
}
pub(crate) fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- let ctx = self.local_search_ctx();
- let field = ctx
- .tv_searcher
+ let tv_searcher = self.reader.searcher();
+ let field = tv_searcher
.schema()
.get_field(Field::Text(TextField::Url).name())
.unwrap();
@@ -414,13 +445,12 @@ impl InvertedIndex {
}
let query = tantivy::query::PhraseQuery::new(term_queries);
- let mut res = ctx
- .tv_searcher
+ let mut res = tv_searcher
.search(&query, &tantivy::collector::TopDocs::with_limit(1))
.unwrap();
res.pop()
- .map(|(_, doc)| self.retrieve_doc(doc.into(), &ctx).unwrap())
+ .map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())
}
}
@@ -555,7 +585,8 @@ mod tests {
use crate::{
ranking::{Ranker, SignalAggregator},
- webpage::{region::RegionCount, Html, Link},
+ searcher::SearchQuery,
+ webpage::{Html, Link},
};
use super::*;
@@ -586,22 +617,24 @@ mod tests {
#[test]
fn simple_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
- let ctx = index.local_search_ctx();
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -623,15 +656,15 @@ mod tests {
))
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
-
let ctx = index.local_search_ctx();
+
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -662,19 +695,22 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "this query should not match",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "this query should not match".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
+
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
}
@@ -704,19 +740,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "runner",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "runner".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
@@ -746,19 +784,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "runners",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "runners".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
@@ -821,19 +861,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "great site",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "great site".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let mut result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let mut result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
result
.documents
@@ -873,19 +915,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "runner",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "runner".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 20);
}
@@ -914,19 +958,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "dr",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "dr".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
@@ -977,22 +1023,24 @@ mod tests {
let mut index = index1.merge(index2);
index.commit().unwrap();
- let ctx = index.local_search_ctx();
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "website".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 2);
assert_eq!(result.documents.len(), 2);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -1005,19 +1053,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "example test",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "example test".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -1042,11 +1092,11 @@ mod tests {
let ctx = index.local_search_ctx();
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -1080,22 +1130,23 @@ mod tests {
index.insert(webpage).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "website".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
-
- let ctx = index.local_search_ctx();
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
@@ -1110,14 +1161,17 @@ mod tests {
);
let query = Query::parse(
- "best website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "best website".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
@@ -1150,19 +1204,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "website".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -1212,19 +1268,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "dr",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "dr".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
@@ -1260,19 +1318,21 @@ mod tests {
let ctx = index.local_search_ctx();
let query = Query::parse(
- "test",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_reader(&ctx),
+ SignalAggregator::new(Some(query.clone())),
+ ctx.fastfield_reader.clone(),
);
- let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
let schema = result.documents[0].schema_org.clone();
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 703ec8f3..e1de85c8 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -40,6 +40,7 @@ mod cluster;
mod collector;
mod directory;
mod entity_index;
+mod enum_map;
mod executor;
mod exponential_backoff;
mod fastfield_reader;
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 3b9dd147..960d0b76 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -15,16 +15,16 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- index::Index,
- ranking::SignalAggregator,
+ inverted_index::InvertedIndex,
+ ranking::{Signal, SignalCoefficient},
schema::{Field, TextField},
search_ctx::Ctx,
- searcher::NUM_RESULTS_PER_PAGE,
+ searcher::SearchQuery,
webpage::region::Region,
Result,
};
-use optics::Optic;
-use std::{collections::HashMap, sync::Arc};
+use optics::{ast::RankingTarget, Optic, SiteRankings};
+use std::{collections::HashMap, str::FromStr, sync::Arc};
use tantivy::{
query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},
schema::Schema,
@@ -50,9 +50,11 @@ pub struct Query {
terms: Vec<Box<Term>>,
simple_terms_text: Vec<String>,
tantivy_query: Box<BooleanQuery>,
+ site_rankings: SiteRankings,
offset: usize,
region: Option<Region>,
- num_results: usize,
+ optic: Option<Optic>,
+ top_n: usize,
}
fn proximity_queries(
@@ -106,13 +108,8 @@ fn proximity_queries(
}
impl Query {
- pub fn parse(
- query: &str,
- schema: Arc<Schema>,
- tokenizer_manager: &TokenizerManager,
- aggregator: &SignalAggregator,
- ) -> Result<Query> {
- let parsed_terms = parser::parse(query);
+ pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {
+ let parsed_terms = parser::parse(&query.query);
let mut term_count = HashMap::new();
let mut terms = Vec::new();
@@ -127,14 +124,15 @@ impl Query {
*count += 1;
}
+ let schema = index.schema();
+ let tokenizer_manager = index.tokenizers();
+
let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =
schema.fields().collect();
- let field_boost = aggregator.field_boosts();
-
let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
- .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager, field_boost))
+ .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
.collect();
let simple_terms_text: Vec<String> = terms
@@ -154,43 +152,47 @@ impl Query {
tokenizer_manager,
));
- let tantivy_query = Box::new(BooleanQuery::new(queries));
+ let mut tantivy_query = Box::new(BooleanQuery::new(queries));
+
+ let mut optic = query
+ .site_rankings
+ .clone()
+ .map(|site_rankings| site_rankings.into_optic());
+
+ if let Some(optic_program) = &query.optic_program {
+ match optic.as_mut() {
+ Some(inner) => {
+ optic = Some(inner.clone().try_merge(Optic::parse(optic_program)?)?);
+ }
+ None => {
+ optic = Some(Optic::parse(optic_program)?);
+ }
+ }
+ }
+
+ if let Some(optic) = &optic {
+ let mut subqueries = vec![(Occur::Must, tantivy_query.box_clone())];
+ subqueries.append(&mut optic.as_multiple_tantivy(&schema, &ctx.fastfield_reader));
+ tantivy_query = Box::new(BooleanQuery::new(subqueries));
+ }
Ok(Query {
terms,
+ site_rankings: optic
+ .as_ref()
+ .map(|optic| optic.site_rankings.clone())
+ .unwrap_or_default(),
simple_terms_text,
tantivy_query,
- offset: 0,
- region: None,
- num_results: NUM_RESULTS_PER_PAGE,
+ optic,
+ offset: query.num_results * query.page,
+ region: query.selected_region,
+ top_n: query.num_results,
})
}
- pub fn set_optic(&mut self, optic: &Optic, index: &Index, ctx: &Ctx) {
- let mut subqueries = vec![(Occur::Must, self.tantivy_query.box_clone())];
-
- subqueries.append(
- &mut optic
- .as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_reader(ctx)),
- );
-
- self.tantivy_query = Box::new(BooleanQuery::new(subqueries));
- }
-
- pub fn simple_terms(&self) -> Vec<String> {
- self.simple_terms_text.clone()
- }
-
- pub fn set_num_results(&mut self, num_results: usize) {
- self.num_results = num_results;
- }
-
- pub fn set_offset(&mut self, offset: usize) {
- self.offset = offset;
- }
-
- pub fn set_region(&mut self, region: Region) {
- self.region = Some(region);
+ pub fn simple_terms(&self) -> &[String] {
+ &self.simple_terms_text
}
pub fn terms(&self) -> &[Box<Term>] {
@@ -202,7 +204,7 @@ impl Query {
}
pub fn num_results(&self) -> usize {
- self.num_results
+ self.top_n
}
pub fn offset(&self) -> usize {
@@ -212,6 +214,25 @@ impl Query {
pub fn region(&self) -> Option<&Region> {
self.region.as_ref()
}
+
+ pub fn site_rankings(&self) -> &SiteRankings {
+ &self.site_rankings
+ }
+
+ pub fn signal_coefficients(&self) -> SignalCoefficient {
+ self.optic
+ .as_ref()
+ .map(|optic| {
+ SignalCoefficient::new(optic.rankings.iter().filter_map(|coeff| {
+ match &coeff.target {
+ RankingTarget::Signal(signal) => Signal::from_str(signal)
+ .ok()
+ .map(|signal| (signal, coeff.value)),
+ }
+ }))
+ })
+ .unwrap_or_default()
+ }
}
impl tantivy::query::Query for Query {
@@ -232,22 +253,29 @@ mod tests {
use crate::{
index::Index,
rand_words,
- schema::create_schema,
searcher::{LocalSearcher, SearchQuery},
webpage::Webpage,
};
use super::*;
+ fn empty_index() -> InvertedIndex {
+ InvertedIndex::temporary().unwrap()
+ }
+
#[test]
fn simple_parse() {
- let schema = Arc::new(create_schema());
+ let index = empty_index();
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "this is a simple query the the the the the the the the the the the the the",
- Arc::clone(&schema),
- &TokenizerManager::new(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "this is a simple query the the the the the the the the the the the the the"
+ .to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
@@ -275,13 +303,16 @@ mod tests {
#[test]
fn parse_trailing_leading_whitespace() {
- let schema = Arc::new(create_schema());
+ let index = empty_index();
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- " this is a simple query ",
- Arc::clone(&schema),
- &TokenizerManager::new(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: " this is a simple query ".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
@@ -299,36 +330,46 @@ mod tests {
#[test]
fn parse_weird_characters() {
- let schema = Arc::new(create_schema());
+ let index = empty_index();
+ let ctx = index.local_search_ctx();
let terms = Query::parse(
- "123",
- Arc::clone(&schema),
- &TokenizerManager::new(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "123".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query")
- .simple_terms();
+ .simple_terms()
+ .to_vec();
assert_eq!(terms, vec!["123".to_string()]);
let terms = Query::parse(
- "123 33",
- Arc::clone(&schema),
- &TokenizerManager::new(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "123 33".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query")
- .simple_terms();
+ .simple_terms()
+ .to_vec();
assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
let terms = Query::parse(
- "term! term# $",
- Arc::clone(&schema),
- &TokenizerManager::new(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "term! term# $".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query")
- .simple_terms();
+ .simple_terms()
+ .to_vec();
assert_eq!(
terms,
vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
@@ -543,13 +584,16 @@ mod tests {
#[test]
fn empty_query() {
- let schema = Arc::new(create_schema());
+ let index = empty_index();
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "",
- Arc::clone(&schema),
- &TokenizerManager::new(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("failed to parse query");
@@ -558,13 +602,16 @@ mod tests {
#[test]
fn query_term_only_special_char() {
- let index = Index::temporary().expect("Unable to open index");
+ let index = empty_index();
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "&",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
+ &ctx,
+ &SearchQuery {
+ query: "&".to_string(),
+ ..Default::default()
+ },
+ &index,
)
.expect("Failed to parse query");
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index c3466294..ec26ac34 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -29,7 +29,7 @@ pub trait AsTantivyQuery {
fn as_tantivy(
&self,
schema: &Schema,
- fastfield_reader: FastFieldReader,
+ fastfield_reader: &FastFieldReader,
) -> Box<dyn tantivy::query::Query>;
}
@@ -37,7 +37,7 @@ pub trait AsMultipleTantivyQuery {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- fastfield_reader: FastFieldReader,
+ fastfield_reader: &FastFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;
}
@@ -45,7 +45,7 @@ impl AsMultipleTantivyQuery for Optic {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- fastfields: FastFieldReader,
+ fastfields: &FastFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
if self.discard_non_matching {
vec![(
@@ -55,7 +55,7 @@ impl AsMultipleTantivyQuery for Optic {
.iter()
.chain(self.site_rankings.rules().iter())
.filter_map(|rule| {
- let queries = rule.as_multiple_tantivy(schema, fastfields.clone());
+ let queries = rule.as_multiple_tantivy(schema, fastfields);
if queries.is_empty() {
None
@@ -72,7 +72,7 @@ impl AsMultipleTantivyQuery for Optic {
self.rules
.iter()
.chain(self.site_rankings.rules().iter())
- .filter_map(|rule| rule.as_multiple_tantivy(schema, fastfields.clone()).pop())
+ .filter_map(|rule| rule.as_multiple_tantivy(schema, fastfields).pop())
.collect()
}
}
@@ -82,17 +82,12 @@ impl AsMultipleTantivyQuery for Rule {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- fastfield_reader: FastFieldReader,
+ fastfield_reader: &FastFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
let mut subqueries: Vec<_> = self
.matches
.iter()
- .map(|matching| {
- (
- Occur::Must,
- matching.as_tantivy(schema, fastfield_reader.clone()),
- )
- })
+ .map(|matching| (Occur::Must, matching.as_tantivy(schema, fastfield_reader)))
.collect();
if subqueries.is_empty() {
@@ -131,35 +126,35 @@ impl AsTantivyQuery for Matching {
fn as_tantivy(
&self,
schema: &Schema,
- fastfield_reader: FastFieldReader,
+ fastfield_reader: &FastFieldReader,
) -> Box<dyn tantivy::query::Query> {
match &self.location {
MatchLocation::Site => PatternQuery::new(
self.pattern.clone(),
TextField::Site,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
MatchLocation::Url => PatternQuery::new(
self.pattern.clone(),
TextField::Url,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
MatchLocation::Domain => PatternQuery::new(
self.pattern.clone(),
TextField::Domain,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
MatchLocation::Title => PatternQuery::new(
self.pattern.clone(),
TextField::Title,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
MatchLocation::Description => UnionQuery::from(vec![
@@ -174,7 +169,7 @@ impl AsTantivyQuery for Matching {
self.pattern.clone(),
TextField::DmozDescription,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
])
@@ -183,14 +178,14 @@ impl AsTantivyQuery for Matching {
self.pattern.clone(),
TextField::CleanBody,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
MatchLocation::Schema => PatternQuery::new(
self.pattern.clone(),
TextField::FlattenedSchemaOrgJson,
schema,
- fastfield_reader,
+ fastfield_reader.clone(),
)
.box_clone(),
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 5e714c9c..9b554d3e 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -15,13 +15,12 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::{
- query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, TermQuery},
+ query::{BooleanQuery, Occur, PhraseQuery, TermQuery},
tokenizer::{TextAnalyzer, TokenizerManager},
};
use crate::{
bangs::BANG_PREFIX,
- ranking::FieldBoost,
schema::{Field, TextField, ALL_FIELDS},
};
@@ -56,7 +55,6 @@ fn simple_into_tantivy(
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
- field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
let (backlink_field, backlink_field_entry) = fields
.iter()
@@ -75,7 +73,6 @@ fn simple_into_tantivy(
term,
fields,
tokenizer_manager,
- field_boost,
))),
),
(
@@ -84,7 +81,6 @@ fn simple_into_tantivy(
backlink_field,
backlink_field_entry,
tokenizer_manager,
- field_boost,
term,
)),
),
@@ -96,10 +92,9 @@ impl Term {
&self,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
- field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
match self {
- Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager, field_boost),
+ Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),
Term::Phrase(phrase) => {
let mut phrases = Vec::with_capacity(fields.len());
@@ -110,13 +105,7 @@ impl Term {
{
phrases.push((
Occur::Should,
- Term::tantivy_text_query(
- field,
- entry,
- tokenizer_manager,
- field_boost,
- phrase,
- ),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, phrase),
));
}
@@ -124,11 +113,9 @@ impl Term {
}
Term::Not(subterm) => vec![(
Occur::MustNot,
- Box::new(BooleanQuery::new(subterm.as_tantivy_query(
- fields,
- tokenizer_manager,
- field_boost,
- ))),
+ Box::new(BooleanQuery::new(
+ subterm.as_tantivy_query(fields, tokenizer_manager),
+ )),
)],
Term::Site(site) => vec![(
Occur::Must,
@@ -136,7 +123,6 @@ impl Term {
site,
fields,
tokenizer_manager,
- field_boost,
))),
)],
Term::Title(title) => {
@@ -152,7 +138,7 @@ impl Term {
vec![(
Occur::Must,
- Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, title),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, title),
)]
}
Term::Body(body) => {
@@ -168,7 +154,7 @@ impl Term {
vec![(
Occur::Must,
- Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, body),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, body),
)]
}
Term::Url(url) => {
@@ -184,7 +170,7 @@ impl Term {
vec![(
Occur::Must,
- Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, url),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, url),
)]
}
Term::PossibleBang(text) => {
@@ -193,7 +179,7 @@ impl Term {
term.push(BANG_PREFIX);
term.push_str(text);
- simple_into_tantivy(&term, fields, tokenizer_manager, field_boost)
+ simple_into_tantivy(&term, fields, tokenizer_manager)
}
}
}
@@ -202,7 +188,6 @@ impl Term {
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
- field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
@@ -210,7 +195,7 @@ impl Term {
.map(|(field, entry)| {
(
Occur::Should,
- Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, term),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, term),
)
})
.collect()
@@ -220,7 +205,6 @@ impl Term {
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
- field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
@@ -234,7 +218,7 @@ impl Term {
.map(|(field, entry)| {
(
Occur::Should,
- Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, term),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, term),
)
})
.collect()
@@ -244,7 +228,6 @@ impl Term {
field: &tantivy::schema::Field,
entry: &tantivy::schema::FieldEntry,
tokenizer_manager: &TokenizerManager,
- field_boost: &FieldBoost,
term: &str,
) -> Box<dyn tantivy::query::Query + 'static> {
let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);
@@ -263,10 +246,7 @@ impl Term {
Box::new(TermQuery::new(term, option))
};
- let boost =
- field_boost.get(&ALL_FIELDS[field.field_id() as usize].as_text().unwrap()) as f32;
-
- Box::new(BoostQuery::new(processed_query, boost))
+ Box::new(processed_query)
}
pub fn get_tantivy_analyzer(
diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rs
index cdc659dc..6117af2a 100644
--- a/core/src/ranking/initial.rs
+++ b/core/src/ranking/initial.rs
@@ -14,10 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::sync::Arc;
-
use crate::fastfield_reader::FastFieldReader;
-use crate::webpage::region::{Region, RegionCount};
use chrono::Utc;
use serde::{Deserialize, Serialize};
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
@@ -26,22 +23,19 @@ use tantivy::{DocId, SegmentReader};
use super::SignalAggregator;
pub(crate) struct InitialScoreTweaker {
- region_count: Arc<RegionCount>,
- selected_region: Option<Region>,
+ tv_searcher: tantivy::Searcher,
fastfield_reader: FastFieldReader,
aggregator: SignalAggregator,
}
impl InitialScoreTweaker {
pub fn new(
- region_count: Arc<RegionCount>,
- selected_region: Option<Region>,
+ tv_searcher: tantivy::Searcher,
aggregator: SignalAggregator,
fastfield_reader: FastFieldReader,
) -> Self {
Self {
- region_count,
- selected_region,
+ tv_searcher,
aggregator,
fastfield_reader,
}
@@ -50,9 +44,6 @@ impl InitialScoreTweaker {
pub(crate) struct InitialSegmentScoreTweaker {
aggregator: SignalAggregator,
- region_count: Arc<RegionCount>,
- current_timestamp: usize,
- selected_region: Option<Region>,
}
impl ScoreTweaker<Score> for InitialScoreTweaker {
@@ -65,16 +56,14 @@ impl ScoreTweaker<Score> for InitialScoreTweaker {
.fastfield_reader
.get_segment(&segment_reader.segment_id());
- aggregator.register_segment(fastfield_segment_reader);
+ aggregator
+ .register_segment(&self.tv_searcher, segment_reader, fastfield_segment_reader)
+ .unwrap();
let current_timestamp = Utc::now().timestamp() as usize;
+ aggregator.set_current_timestamp(current_timestamp);
- Ok(InitialSegmentScoreTweaker {
- aggregator,
- current_timestamp,
- selected_region: self.selected_region,
- region_count: Arc::clone(&self.region_count),
- })
+ Ok(InitialSegmentScoreTweaker { aggregator })
}
}
@@ -86,12 +75,13 @@ pub struct Score {
impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {
fn score(&mut self, doc: DocId, score: tantivy::Score) -> Score {
- self.aggregator.score(
- doc,
- score,
- &self.region_count,
- self.current_timestamp,
- self.selected_region,
- )
+ let total = self
+ .aggregator
+ .compute_signals(doc)
+ .flatten()
+ .map(|computed| computed.coefficient * computed.value)
+ .sum();
+
+ Score { bm25: score, total }
}
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 9c4bc390..e2d0b847 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -24,7 +24,7 @@ pub mod optics;
pub mod pipeline;
pub mod signal;
-use std::{collections::BTreeMap, sync::Arc};
+use std::collections::BTreeMap;
use ::optics::SiteRankings;
use initial::InitialScoreTweaker;
@@ -34,6 +34,7 @@ use crate::{
collector::{MaxDocsConsidered, TopDocs},
fastfield_reader::FastFieldReader,
inverted_index,
+ search_ctx::Ctx,
searcher::NUM_RESULTS_PER_PAGE,
webgraph::{
centrality::{
@@ -42,18 +43,13 @@ use crate::{
},
Node, NodeID,
},
- webpage::{
- region::{Region, RegionCount},
- Url,
- },
+ webpage::{region::Region, Url},
};
pub use self::signal::*;
#[derive(Clone)]
pub struct Ranker {
- region_count: Arc<RegionCount>,
- selected_region: Option<Region>,
max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
aggregator: SignalAggregator,
@@ -63,14 +59,8 @@ pub struct Ranker {
}
impl Ranker {
- pub fn new(
- region_count: RegionCount,
- aggregator: SignalAggregator,
- fastfield_reader: FastFieldReader,
- ) -> Self {
+ pub fn new(aggregator: SignalAggregator, fastfield_reader: FastFieldReader) -> Self {
Ranker {
- region_count: Arc::new(region_count),
- selected_region: None,
offset: None,
aggregator,
max_docs: None,
@@ -81,7 +71,7 @@ impl Ranker {
}
pub fn with_region(mut self, region: Region) -> Self {
- self.selected_region = Some(region);
+ self.aggregator.set_selected_region(region);
self
}
@@ -111,15 +101,14 @@ impl Ranker {
self.aggregator.clone()
}
- pub fn collector(&self) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {
+ pub fn collector(
+ &self,
+ ctx: Ctx,
+ ) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {
let aggregator = self.aggregator();
- let score_tweaker = InitialScoreTweaker::new(
- Arc::clone(&self.region_count),
- self.selected_region,
- aggregator,
- self.fastfield_reader.clone(),
- );
+ let score_tweaker =
+ InitialScoreTweaker::new(ctx.tv_searcher, aggregator, self.fastfield_reader.clone());
let mut collector = TopDocs::with_limit(
self.num_results.unwrap_or(NUM_RESULTS_PER_PAGE),
@@ -794,12 +783,8 @@ mod tests {
query: "example".to_string(),
optic_program: Some(
r#"
- RankingPipeline {
- Stage {
- Ranking{Field("title"), 20000000},
- Ranking{Signal("host_centrality"), 0}
- }
- }
+ Ranking(Signal("bm25_title"), 20000000);
+ Ranking(Signal("host_centrality"), 0);
"#
.to_string(),
),
@@ -815,11 +800,7 @@ mod tests {
query: "example".to_string(),
optic_program: Some(
r#"
- RankingPipeline {
- Stage {
- Ranking{Signal("host_centrality"), 2000000}
- }
- }
+ Ranking(Signal("host_centrality"), 2000000)
"#
.to_string(),
),
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 9fc485c5..27aaec58 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -14,43 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use optics::ast::RankingStage;
-
-use crate::schema::Field;
-
-use super::{Signal, SignalAggregator};
-
pub const SCALE: f32 = 10.0;
-pub trait CreateAggregator {
- fn aggregator(&self) -> SignalAggregator;
-}
-
-impl CreateAggregator for RankingStage {
- fn aggregator(&self) -> SignalAggregator {
- let aggregator = SignalAggregator::new(
- self.coefficients
- .iter()
- .filter_map(|coeff| match &coeff.target {
- optics::ast::RankingTarget::Signal(s) => {
- Signal::from_string(s.clone()).map(|s| (s, coeff.score))
- }
- optics::ast::RankingTarget::Field(_) => None,
- }),
- self.coefficients
- .iter()
- .filter_map(|coeff| match &coeff.target {
- optics::ast::RankingTarget::Signal(_) => None,
- optics::ast::RankingTarget::Field(f) => Field::from_name(f.clone())
- .and_then(|f| f.as_text())
- .map(|f| (f, coeff.score)),
- }),
- );
-
- aggregator
- }
-}
-
#[cfg(test)]
mod tests {
use optics::SiteRankings;
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index ff405207..92fb92ce 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -16,17 +16,11 @@
use std::{cmp::Ordering, sync::Arc};
-use optics::ast::{RankingCoeff, RankingTarget};
use serde::{Deserialize, Serialize};
-use crate::{
- inverted_index::WebsitePointer,
- schema::{FastField, Field, TextField, ALL_FIELDS, FLOAT_SCALING},
- searcher::SearchQuery,
- Result,
-};
+use crate::{enum_map::EnumMap, inverted_index::WebsitePointer, searcher::SearchQuery, Result};
-use super::{models::cross_encoder::CrossEncoder, SignalAggregator};
+use super::{models::cross_encoder::CrossEncoder, Signal, SignalAggregator};
pub trait AsRankingWebsite: Clone {
fn as_ranking(&self) -> &RankingWebsite;
@@ -46,62 +40,25 @@ impl AsRankingWebsite for RankingWebsite {
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RankingWebsite {
pub pointer: WebsitePointer,
- pub host_centrality: f64,
- pub page_centrality: f64,
- pub topic_centrality: f64,
- pub personal_centrality: f64,
- pub query_centrality: f64,
- pub inbound_similarity: f64,
- pub title: String,
- pub clean_body: String,
+ pub signals: EnumMap<Signal, f64>,
+ pub title: Option<String>,
+ pub clean_body: Option<String>,
pub score: f64,
}
impl RankingWebsite {
- pub fn new(
- doc: tantivy::Document,
- pointer: WebsitePointer,
- aggregator: &SignalAggregator,
- ) -> Self {
+ pub fn new(pointer: WebsitePointer, aggregator: &mut SignalAggregator) -> Self {
let mut res = RankingWebsite {
- host_centrality: 0.0,
- page_centrality: 0.0,
- topic_centrality: 0.0,
- personal_centrality: 0.0,
- query_centrality: 0.0,
- inbound_similarity: 0.0,
- title: String::new(),
- clean_body: String::new(),
+ signals: EnumMap::new(),
+ title: None,
+ clean_body: None,
score: pointer.score.total,
- pointer,
+ pointer: pointer.clone(),
};
- for value in doc.field_values() {
- match ALL_FIELDS[value.field().field_id() as usize] {
- Field::Fast(FastField::HostCentrality) => {
- res.host_centrality =
- value.value.as_u64().unwrap() as f64 / FLOAT_SCALING as f64
- }
- Field::Fast(FastField::PageCentrality) => {
- res.page_centrality =
- value.value.as_u64().unwrap() as f64 / FLOAT_SCALING as f64
- }
- Field::Fast(FastField::HostNodeID) => {
- let node = value.value.as_u64().unwrap().into();
-
- res.personal_centrality = aggregator.personal_centrality(node);
- res.inbound_similarity = aggregator.inbound_similarity(node);
- res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();
- res.query_centrality = aggregator.query_centrality(node).unwrap_or_default();
- }
- Field::Text(TextField::Title) => {
- res.title = value.value.as_text().unwrap().to_string()
- }
- Field::Text(TextField::StemmedCleanBody) => {
- res.clean_body = value.value.as_text().unwrap().to_string()
- }
- _ => {}
- }
+ for computed_signal in aggregator.compute_signals(pointer.address.doc_id).flatten() {
+ res.signals
+ .insert(computed_signal.signal, computed_signal.value);
}
res
@@ -110,23 +67,18 @@ impl RankingWebsite {
trait Scorer<T: AsRankingWebsite>: Send + Sync {
fn score(&self, websites: &mut [T]);
- fn set_coefficients(&mut self, coefficients: &[RankingCoeff]);
fn set_query_info(&mut self, _query: &SearchQuery) {}
}
struct ReRanker<M: CrossEncoder> {
- coefficient: f64,
model: Arc<M>,
- prev_score: f64,
query: String,
}
impl<M: CrossEncoder> ReRanker<M> {
fn new(model: Arc<M>) -> Self {
Self {
- coefficient: 800.0,
model,
- prev_score: 1.0,
query: String::new(),
}
}
@@ -138,7 +90,9 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
for website in websites.iter_mut() {
let website = website.as_mut_ranking();
- let text = website.title.clone() + ". " + website.clean_body.as_str();
+ let title = website.title.clone().unwrap_or_default();
+ let body = website.clean_body.clone().unwrap_or_default();
+ let text = title + ". " + &body;
bodies.push(text);
}
@@ -146,65 +100,30 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
for (website, score) in websites.iter_mut().zip(scores.into_iter()) {
let website = website.as_mut_ranking();
- website.score = self.prev_score * website.score + self.coefficient * score;
+ website.score += score;
}
}
fn set_query_info(&mut self, query: &SearchQuery) {
self.query = query.query.clone();
}
-
- fn set_coefficients(&mut self, coefficients: &[RankingCoeff]) {
- for coeff in coefficients {
- if let RankingTarget::Signal(signal) = &coeff.target {
- match signal.as_str() {
- "prev_score" => self.prev_score = coeff.score,
- "crossencoder" => self.coefficient = coeff.score,
- _ => {}
- }
- }
- }
- }
}
-struct PrioritizeText {
- bm25: f64,
- prev_score: f64,
-}
-
-impl Default for PrioritizeText {
- fn default() -> Self {
- Self {
- bm25: 10.0,
- prev_score: 1.0,
- }
- }
-}
+#[derive(Default)]
+struct PrioritizeText {}
impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {
fn score(&self, websites: &mut [T]) {
for website in websites {
let bm25 = website.as_ranking().pointer.score.bm25 as f64;
- let prev_score = website.as_ranking().score;
- website.as_mut_ranking().score = self.bm25 * bm25 + self.prev_score * prev_score;
- }
- }
-
- fn set_coefficients(&mut self, coefficients: &[RankingCoeff]) {
- for coeff in coefficients {
- if let RankingTarget::Signal(signal) = &coeff.target {
- match signal.as_str() {
- "prev_score" => self.prev_score = coeff.score,
- "bm25" => self.bm25 = coeff.score,
- _ => {}
- }
- }
+ website.as_mut_ranking().score += bm25;
}
}
}
enum Prev<T: AsRankingWebsite> {
Initial,
+ #[allow(dead_code)]
Node(Box<RankingStage<T>>),
}
@@ -212,7 +131,7 @@ struct RankingStage<T: AsRankingWebsite> {
scorer: Box<dyn Scorer<T>>,
prev: Prev<T>,
top_n: usize,
- memory: Option<(Vec<T>, Vec<T>)>,
+ memory: Option<Vec<T>>,
}
impl<T: AsRankingWebsite> RankingStage<T> {
@@ -226,50 +145,32 @@ impl<T: AsRankingWebsite> RankingStage<T> {
pub fn populate(&mut self, websites: Vec<T>) {
match &mut self.prev {
Prev::Initial => {
- let a: Vec<_> = websites.clone().into_iter().take(self.top_n).collect();
- let b: Vec<_> = websites
- .into_iter()
- .skip(self.top_n)
- .take(self.top_n)
- .collect();
-
- self.memory = Some((a, b));
+ self.memory = Some(websites);
}
Prev::Node(n) => n.populate(websites),
}
}
- fn apply(&self, top_n: usize, offset: usize) -> Vec<T> {
- let (mut a, mut b) = match &self.prev {
+ fn apply(&self, top_n: usize, page: usize) -> Vec<T> {
+ let next_page = (page * top_n) / self.top_n;
+ let mut websites = match &self.prev {
Prev::Initial => self.memory.clone().unwrap(),
- Prev::Node(n) => {
- let k = offset / self.top_n;
- (
- n.apply(self.top_n, k * self.top_n),
- n.apply(self.top_n, (k + 1) * self.top_n),
- )
- }
+ Prev::Node(n) => n.apply(self.top_n, next_page),
};
- self.scorer.score(&mut a);
- a.sort_by(|a, b| {
- b.as_ranking()
- .score
- .partial_cmp(&a.as_ranking().score)
- .unwrap_or(Ordering::Equal)
- });
+ let page = page - next_page;
- self.scorer.score(&mut b);
- b.sort_by(|a, b| {
+ self.scorer.score(&mut websites);
+ websites.sort_by(|a, b| {
b.as_ranking()
.score
.partial_cmp(&a.as_ranking().score)
.unwrap_or(Ordering::Equal)
});
- a.into_iter()
- .chain(b.into_iter())
- .skip(offset % self.top_n)
+ websites
+ .into_iter()
+ .skip(page * top_n)
.take(top_n)
.collect()
}
@@ -285,91 +186,87 @@ impl<T: AsRankingWebsite> RankingStage<T> {
pub struct RankingPipeline<T: AsRankingWebsite> {
last_stage: RankingStage<T>,
- offset: usize,
- top_n: usize,
+ page: usize,
+ pub top_n: usize,
}
impl<T: AsRankingWebsite> RankingPipeline<T> {
- fn create<M: CrossEncoder + 'static>(crossencoder: Arc<M>) -> Result<Self> {
+ fn create_reranking<M: CrossEncoder + 'static>(crossencoder: Arc<M>) -> Result<Self> {
let last_stage = RankingStage {
scorer: Box::new(ReRanker::new(crossencoder)),
- prev: Prev::Node(Box::new(RankingStage {
- scorer: Box::<PrioritizeText>::default(),
- prev: Prev::Initial,
- memory: None,
- top_n: 40,
- })),
+ prev: Prev::Initial,
memory: None,
top_n: 20,
};
Ok(Self {
last_stage,
- offset: 0,
+ page: 0,
top_n: 0,
})
}
- pub fn for_query<M: CrossEncoder + 'static>(
+ pub fn reranking_for_query<M: CrossEncoder + 'static>(
query: &mut SearchQuery,
crossencoder: Arc<M>,
) -> Result<Self> {
- let mut pipeline = Self::create(crossencoder)?;
- if let Some(optic) = &query.optic_program {
- let optic = optics::parse(optic)?;
- if let Some(optic_pipeline) = optic.pipeline {
- let mut stage = Some(&mut pipeline.last_stage);
-
- // skip(1) since first stage is collector
- for optic_stage in optic_pipeline.stages.into_iter().skip(1).rev() {
- match stage {
- Some(inner_stage) => {
- inner_stage
- .scorer
- .set_coefficients(&optic_stage.coefficients);
-
- match &mut inner_stage.prev {
- Prev::Initial => stage = None,
- Prev::Node(next) => stage = Some(next),
- }
- }
- None => break,
- }
- }
- }
+ let mut pipeline = Self::create_reranking(crossencoder)?;
+ pipeline.set_query_info(query);
+
+ Ok(pipeline)
+ }
+
+ fn create_ltr() -> Self {
+ let last_stage = RankingStage {
+ scorer: Box::<PrioritizeText>::default(),
+ prev: Prev::Initial,
+ memory: None,
+ top_n: 10_000,
+ };
+
+ Self {
+ last_stage,
+ page: 0,
+ top_n: 0,
}
+ }
- pipeline.last_stage.set_query_info(query);
+ pub fn ltr_for_query(query: &mut SearchQuery) -> Self {
+ let mut pipeline = Self::create_ltr();
+ pipeline.set_query_info(query);
- pipeline.offset = query.offset;
- pipeline.top_n = query.num_results;
+ pipeline
+ }
- query.num_results = pipeline.collector_top_n();
- query.offset = pipeline.collector_offset();
+ fn set_query_info(&mut self, query: &mut SearchQuery) {
+ self.last_stage.set_query_info(query);
+ self.page = query.page;
+ self.top_n = query.num_results;
- Ok(pipeline)
+ query.num_results = self.collector_top_n();
+ query.page = self.collector_page();
}
pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {
self.last_stage.populate(websites);
- self.last_stage.apply(self.top_n, self.pipeline_offset())
+ self.last_stage.apply(self.top_n, self.pipeline_page())
}
pub fn collector_top_n(&self) -> usize {
- 2 * self.initial_top_n()
+ self.initial_top_n()
}
pub fn initial_top_n(&self) -> usize {
self.last_stage.initial_top_n()
}
- fn pipeline_offset(&self) -> usize {
- self.offset - self.collector_offset()
+ fn pipeline_page(&self) -> usize {
+ self.page - (self.collector_page() * self.collector_top_n() / self.top_n)
}
- pub fn collector_offset(&self) -> usize {
- (self.offset / self.initial_top_n()) * self.initial_top_n()
+ pub fn collector_page(&self) -> usize {
+ (self.page * self.top_n) / self.initial_top_n()
}
}
@@ -405,14 +302,9 @@ mod tests {
doc_id: i as u32,
},
},
- host_centrality: 0.0,
- page_centrality: 0.0,
- topic_centrality: 0.0,
- personal_centrality: 0.0,
- query_centrality: 0.0,
- inbound_similarity: 0.0,
- title: String::new(),
- clean_body: String::new(),
+ signals: EnumMap::new(),
+ title: None,
+ clean_body: None,
score: 1.0 / i as f64,
}
})
@@ -421,14 +313,14 @@ mod tests {
#[test]
fn simple() {
- let pipeline = RankingPipeline::for_query(
+ let pipeline = RankingPipeline::reranking_for_query(
&mut SearchQuery {
..Default::default()
},
Arc::new(DummyCrossEncoder {}),
)
.unwrap();
- assert_eq!(pipeline.collector_top_n(), 80);
+ assert_eq!(pipeline.collector_top_n(), 20);
let sample = sample_websites(pipeline.collector_top_n());
let res: Vec<_> = pipeline
@@ -448,9 +340,11 @@ mod tests {
#[test]
fn offsets() {
- let pipeline = RankingPipeline::for_query(
+ let num_results = 20;
+ let pipeline = RankingPipeline::reranking_for_query(
&mut SearchQuery {
- offset: 0,
+ page: 0,
+ num_results,
..Default::default()
},
Arc::new(DummyCrossEncoder {}),
@@ -458,174 +352,41 @@ mod tests {
.unwrap();
let sample: Vec<_> =
- sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
+ sample_websites(pipeline.collector_top_n() + pipeline.collector_page() * num_results)
.into_iter()
- .skip(pipeline.collector_offset())
+ .skip(pipeline.collector_page() * num_results)
.collect();
let mut prev: Vec<_> = pipeline.apply(sample);
-
- for offset in 1..1_000 {
- let pipeline = RankingPipeline::for_query(
+ for p in 1..1_000 {
+ let pipeline = RankingPipeline::reranking_for_query(
&mut SearchQuery {
- offset,
+ page: p,
..Default::default()
},
Arc::new(DummyCrossEncoder {}),
)
.unwrap();
- let sample: Vec<_> =
- sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
- .into_iter()
- .skip(pipeline.collector_offset())
- .collect();
- let res: Vec<_> = pipeline.apply(sample);
-
- assert_eq!(res.len(), 20, "Every page should have 20 results");
-
- if let Some(first) = prev.first() {
- assert!(!res
- .iter()
- .any(|r| r.pointer.address.doc_id == first.pointer.address.doc_id));
- }
-
- prev = res;
- }
-
- let pipeline = RankingPipeline::for_query(
- &mut SearchQuery {
- offset: 0,
- ..Default::default()
- },
- Arc::new(DummyCrossEncoder {}),
- )
- .unwrap();
-
- let sample: Vec<_> =
- sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
- .into_iter()
- .skip(pipeline.collector_offset())
- .collect();
- let mut prev: Vec<_> = pipeline.apply(sample);
- for p in 1..100 {
- let pipeline = RankingPipeline::for_query(
- &mut SearchQuery {
- offset: p * 20,
- ..Default::default()
- },
- Arc::new(DummyCrossEncoder {}),
+ let sample: Vec<_> = sample_websites(
+ pipeline.collector_top_n() + pipeline.collector_page() * num_results,
)
- .unwrap();
-
- let sample: Vec<_> =
- sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
- .into_iter()
- .skip(pipeline.collector_offset())
- .collect();
+ .into_iter()
+ .skip(pipeline.collector_page() * num_results)
+ .collect();
let res: Vec<_> = pipeline.apply(sample).into_iter().collect();
- assert_eq!(res.len(), 20, "Every page should have 20 results");
+ assert_eq!(
+ res.len(),
+ num_results,
+ "Every page should have {num_results} results"
+ );
- assert!(prev
+ assert!(!prev
.iter()
.zip_eq(res.iter())
- .all(|(p, r)| p.pointer.address.doc_id + 20 == r.pointer.address.doc_id));
+ .any(|(p, r)| p.pointer.address.doc_id == r.pointer.address.doc_id));
prev = res;
}
}
-
- #[test]
- fn multistage_coefficients() {
- let pipeline = RankingPipeline::for_query(
- &mut SearchQuery {
- optic_program: Some(
- r#"
- RankingPipeline {
- Stage {},
- Stage {
- Ranking{Signal("bm25"), 3},
- Ranking{Signal("prev_score"), 2},
- },
- Stage {
- Ranking{Signal("crossencoder"), 4},
- Ranking{Signal("prev_score"), 3},
- },
- }
- "#
- .to_string(),
- ),
- ..Default::default()
- },
- Arc::new(DummyCrossEncoder {}),
- )
- .unwrap();
-
- let w = RankingWebsite {
- pointer: WebsitePointer {
- score: Score {
- bm25: 1.0,
- total: 1.0,
- },
- hashes: Hashes {
- site: Prehashed(0),
- title: Prehashed(0),
- url: Prehashed(0),
- simhash: 0,
- },
- address: DocAddress {
- segment: 0,
- doc_id: 0,
- },
- },
- host_centrality: 0.0,
- page_centrality: 0.0,
- topic_centrality: 0.0,
- personal_centrality: 0.0,
- query_centrality: 0.0,
- inbound_similarity: 0.0,
- title: String::new(),
- clean_body: String::new(),
- score: 1.0,
- };
-
- let res = pipeline.apply(vec![w.clone()]);
- assert_eq!(res[0].score, 19.0);
-
- let pipeline = RankingPipeline::for_query(
- &mut SearchQuery {
- optic_program: Some(
- r#"
- RankingPipeline {
- Stage {},
- Stage {
- Ranking{Signal("bm25"), 3},
- Ranking{Signal("prev_score"), 2},
- },
- Stage {
- Ranking{Signal("crossencoder"), 4},
- Ranking{Signal("prev_score"), 3},
- },
- }
- "#
- .to_string(),
- ),
- ..Default::default()
- },
- Arc::new(DummyCrossEncoder {}),
- )
- .unwrap();
-
- let mut test = [w.clone()];
-
- pipeline.last_stage.scorer.score(&mut test);
- assert_eq!(test[0].score, 7.0);
-
- let mut test = [w];
-
- if let Prev::Node(prev) = pipeline.last_stage.prev {
- prev.scorer.score(&mut test);
- assert_eq!(test[0].score, 5.0);
- }
- }
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 68fb313c..7e8dc1ef 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -14,7 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::fastfield_reader::FieldValue;
+use crate::query::Query;
+use crate::Result;
use crate::{
+ enum_map::EnumMap,
fastfield_reader,
schema::{FastField, TextField},
webgraph::{
@@ -23,21 +27,48 @@ use crate::{
},
webpage::Webpage,
};
-use std::{ops::Deref, sync::Arc};
+use std::str::FromStr;
+use std::sync::Arc;
+use tantivy::fieldnorm::FieldNormReader;
+use tantivy::postings::SegmentPostings;
+use thiserror::Error;
-use chrono::Utc;
-use tantivy::DocId;
+use tantivy::DocSet;
+use tantivy::{DocId, Postings};
use crate::{
- schema::{Field, FLOAT_SCALING},
+ schema::FLOAT_SCALING,
webpage::region::{Region, RegionCount},
};
-use super::{inbound_similarity, initial::Score};
+use super::bm25::Bm25Weight;
+use super::inbound_similarity;
+
+#[derive(Debug, Error)]
+pub enum Error {
+ #[error("unknown signal: {0}")]
+ UnknownSignal(String),
+}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Signal {
Bm25,
+ Bm25Title,
+ Bm25CleanBody,
+ Bm25StemmedTitle,
+ Bm25StemmedCleanBody,
+ Bm25AllBody,
+ Bm25Url,
+ Bm25Site,
+ Bm25Domain,
+ Bm25SiteNoTokenizer,
+ Bm25DomainNoTokenizer,
+ Bm25DomainIfHomepage,
+ Bm25DomainNameIfHomepageNoTokenizer,
+ Bm25TitleIfHomepage,
+ Bm25BacklinkText,
+ Bm25Description,
+ CrossEncoder,
HostCentrality,
PageCentrality,
IsHomepage,
@@ -50,10 +81,33 @@ pub enum Signal {
TopicCentrality,
QueryCentrality,
InboundSimilarity,
+ LambdaMART,
+}
+
+impl From<Signal> for usize {
+ fn from(signal: Signal) -> Self {
+ signal as usize
+ }
}
-pub const ALL_SIGNALS: [Signal; 13] = [
+pub const ALL_SIGNALS: [Signal; 30] = [
Signal::Bm25,
+ Signal::Bm25Title,
+ Signal::Bm25CleanBody,
+ Signal::Bm25StemmedTitle,
+ Signal::Bm25StemmedCleanBody,
+ Signal::Bm25AllBody,
+ Signal::Bm25Url,
+ Signal::Bm25Site,
+ Signal::Bm25Domain,
+ Signal::Bm25SiteNoTokenizer,
+ Signal::Bm25DomainNoTokenizer,
+ Signal::Bm25DomainIfHomepage,
+ Signal::Bm25DomainNameIfHomepageNoTokenizer,
+ Signal::Bm25TitleIfHomepage,
+ Signal::Bm25BacklinkText,
+ Signal::Bm25Description,
+ Signal::CrossEncoder,
Signal::HostCentrality,
Signal::PageCentrality,
Signal::IsHomepage,
@@ -66,89 +120,68 @@ pub const ALL_SIGNALS: [Signal; 13] = [
Signal::TopicCentrality,
Signal::QueryCentrality,
Signal::InboundSimilarity,
+ Signal::LambdaMART,
];
-struct SignalValue {
- bm25: tantivy::Score,
- fastfield_value: Option<u64>,
- current_timestamp: usize,
- selected_region: Option<Region>,
- personal_centrality: Option<f64>,
- topic_score: Option<f64>,
- query_centrality: Option<f64>,
- inbound_similarity: Option<f64>,
-}
-
-impl Signal {
- fn is_computable_before_search(&self) -> bool {
- self.as_fastfield().is_some()
+fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f64 {
+ if timestamp >= signal_aggregator.current_timestamp.unwrap_or(0) {
+ return 0.0;
}
- fn value(
- &self,
- region_count: &impl Deref<Target = RegionCount>,
- aggregator: &SignalAggregator,
- value: SignalValue,
- ) -> f64 {
- match self {
- Signal::Bm25 => value.bm25 as f64,
- Signal::HostCentrality | Signal::PageCentrality => {
- value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64
- }
- Signal::IsHomepage => value.fastfield_value.unwrap() as f64,
- Signal::FetchTimeMs => {
- let fetch_time_ms = value.fastfield_value.unwrap() as usize;
+ let hours_since_update =
+ (signal_aggregator.current_timestamp.unwrap() - timestamp).max(1) / 3600;
- if fetch_time_ms >= aggregator.fetch_time_ms_cache.len() {
- 0.0
- } else {
- aggregator.fetch_time_ms_cache[fetch_time_ms]
- }
- }
- Signal::UpdateTimestamp => {
- let update_timestamp = value.fastfield_value.unwrap() as i64;
+ if hours_since_update < signal_aggregator.update_time_cache.len() {
+ signal_aggregator.update_time_cache[hours_since_update]
+ } else {
+ 0.0
+ }
+}
- if value.current_timestamp as i64 - update_timestamp <= 0 {
- return 0.0;
- }
+fn score_trackers(num_trackers: f64) -> f64 {
+ 1.0 / (num_trackers + 1.0)
+}
- let hours_since_update =
- ((value.current_timestamp as i64 - update_timestamp).max(1) / 3600) as usize;
+fn bm25(field: &mut TextFieldData, doc: DocId) -> f64 {
+ let mut term_freq = 0;
+ for posting in &mut field.postings {
+ if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {
+ term_freq += posting.term_freq();
+ }
+ }
- if hours_since_update < aggregator.update_time_cache.len() {
- aggregator.update_time_cache[hours_since_update]
- } else {
- 0.0
- }
- }
- Signal::TrackerScore => {
- let tracker_score = value.fastfield_value.unwrap() as f64;
- 1.0 / (tracker_score + 1.0)
- }
- Signal::Region => {
- let webpage_region = Region::from_id(value.fastfield_value.unwrap());
+ if term_freq == 0 {
+ return 0.0;
+ }
- let boost = value.selected_region.map_or(0.0, |region| {
- if region == webpage_region {
- 50.0
- } else {
- 0.0
- }
- });
+ let fieldnorm_id = field.fieldnorm_reader.fieldnorm_id(doc);
+ field.weight.score(fieldnorm_id, term_freq) as f64
+}
- boost + region_count.score(&webpage_region)
- }
- Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,
- Signal::TopicCentrality => value.topic_score.unwrap_or_default(),
- Signal::QueryCentrality => value.query_centrality.unwrap_or_default(),
- Signal::PersonalCentrality => value.personal_centrality.unwrap_or_default(),
- Signal::InboundSimilarity => value.inbound_similarity.unwrap_or_default(),
- }
+impl Signal {
+ fn is_computable_before_search(&self) -> bool {
+ self.as_fastfield().is_some()
}
fn default_coefficient(&self) -> f64 {
match self {
Signal::Bm25 => 1.0,
+ Signal::Bm25Title => 15.0,
+ Signal::Bm25CleanBody => 4.0,
+ Signal::Bm25StemmedTitle => 0.5,
+ Signal::Bm25StemmedCleanBody => 0.5,
+ Signal::Bm25AllBody => 1.0,
+ Signal::Bm25Url => 1.0,
+ Signal::Bm25Site => 10.0,
+ Signal::Bm25Domain => 1.0,
+ Signal::Bm25SiteNoTokenizer => 1.0,
+ Signal::Bm25DomainNoTokenizer => 1.0,
+ Signal::Bm25DomainIfHomepage => 6.0,
+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 30.0,
+ Signal::Bm25TitleIfHomepage => 3.0,
+ Signal::Bm25BacklinkText => 4.0,
+ Signal::Bm25Description => 1.0,
+ Signal::CrossEncoder => 100.0,
Signal::HostCentrality => 2_500.0,
Signal::PageCentrality => 4_500.0,
Signal::TopicCentrality => 2_500.0,
@@ -161,31 +194,226 @@ impl Signal {
Signal::CrawlStability => 20.0,
Signal::PersonalCentrality => 5_000.0,
Signal::InboundSimilarity => 5_000.0,
+ Signal::LambdaMART => 10_000.0,
}
}
- pub fn from_string(name: String) -> Option<Signal> {
- match name.as_str() {
- "bm25" => Some(Signal::Bm25),
- "host_centrality" => Some(Signal::HostCentrality),
- "page_centrality" => Some(Signal::PageCentrality),
- "is_homepage" => Some(Signal::IsHomepage),
- "fetch_time_ms" => Some(Signal::FetchTimeMs),
- "update_timestamp" => Some(Signal::UpdateTimestamp),
- "tracker_score" => Some(Signal::TrackerScore),
- "region" => Some(Signal::Region),
- "personal_centrality" => Some(Signal::PersonalCentrality),
- "topic_centrality" => Some(Signal::TopicCentrality),
- "query_centrality" => Some(Signal::QueryCentrality),
- "inbound_similarity" => Some(Signal::InboundSimilarity),
- "crawl_stability" => Some(Signal::CrawlStability),
- _ => None,
+ fn host_id(&self, aggregator: &SignalAggregator, doc: DocId) -> Option<NodeID> {
+ aggregator.segment_reader.as_ref().and_then(|reader| {
+ let node_id: Option<u64> = reader
+ .fastfield_reader
+ .get_field_reader(&FastField::HostNodeID)
+ .get(&doc)
+ .into();
+ let node_id = node_id.unwrap();
+
+ if node_id == u64::MAX {
+ None
+ } else {
+ Some(NodeID::from(node_id))
+ }
+ })
+ }
+
+ fn fastfield_value(&self, aggregator: &SignalAggregator, doc: DocId) -> Option<FieldValue> {
+ aggregator.segment_reader.as_ref().and_then(|reader| {
+ self.as_fastfield().map(|fast_field| {
+ reader
+ .fastfield_reader
+ .get_field_reader(&fast_field)
+ .get(&doc)
+ })
+ })
+ }
+
+ pub fn compute(
+ self,
+ signal_aggregator: &mut SignalAggregator,
+ doc: DocId,
+ ) -> Option<ComputedSignal> {
+ let value: Option<f64> = match self {
+ Signal::HostCentrality | Signal::PageCentrality | Signal::CrawlStability => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|val| val as f64 / FLOAT_SCALING as f64)
+ }
+ Signal::IsHomepage => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|val| val as f64)
+ }
+ Signal::FetchTimeMs => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|v| v as usize).map(|fetch_time_ms| {
+ if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {
+ 0.0
+ } else {
+ signal_aggregator.fetch_time_ms_cache[fetch_time_ms]
+ }
+ })
+ }
+ Signal::UpdateTimestamp => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value
+ .map(|v| v as usize)
+ .map(|update_timestamp| score_timestamp(update_timestamp, signal_aggregator))
+ }
+ Signal::TrackerScore => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|num_trackers| score_trackers(num_trackers as f64))
+ }
+ Signal::Region => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value
+ .map(Region::from_id)
+ .map(|region| score_region(region, signal_aggregator))
+ }
+ Signal::PersonalCentrality => {
+ let host_id = self.host_id(signal_aggregator, doc);
+ host_id.map(|host_id| signal_aggregator.personal_centrality(host_id))
+ }
+ Signal::TopicCentrality => {
+ let host_id = self.host_id(signal_aggregator, doc);
+ host_id.and_then(|host_id| signal_aggregator.topic_centrality(host_id))
+ }
+ Signal::QueryCentrality => {
+ let host_id = self.host_id(signal_aggregator, doc);
+ host_id.and_then(|host_id| signal_aggregator.query_centrality(host_id))
+ }
+ Signal::InboundSimilarity => {
+ let host_id = self.host_id(signal_aggregator, doc);
+ host_id.map(|host_id| signal_aggregator.inbound_similarity(host_id))
+ }
+ Signal::Bm25 => signal_aggregator.segment_reader.as_mut().map(|reader| {
+ reader
+ .text_fields
+ .values_mut()
+ .map(|field| bm25(field, doc))
+ .sum()
+ }),
+ Signal::Bm25Title
+ | Signal::Bm25CleanBody
+ | Signal::Bm25StemmedTitle
+ | Signal::Bm25StemmedCleanBody
+ | Signal::Bm25AllBody
+ | Signal::Bm25Url
+ | Signal::Bm25Site
+ | Signal::Bm25Domain
+ | Signal::Bm25SiteNoTokenizer
+ | Signal::Bm25DomainNoTokenizer
+ | Signal::Bm25DomainIfHomepage
+ | Signal::Bm25DomainNameIfHomepageNoTokenizer
+ | Signal::Bm25TitleIfHomepage
+ | Signal::Bm25BacklinkText
+ | Signal::Bm25Description => signal_aggregator.segment_reader.as_mut().map(|reader| {
+ reader
+ .text_fields
+ .get_mut(self.as_textfield().unwrap())
+ .map(|field| bm25(field, doc))
+ .unwrap_or(0.0)
+ }),
+ Signal::CrossEncoder => None, // this is calculated in a later step
+ Signal::LambdaMART => None,
+ };
+
+ value.map(|value| ComputedSignal {
+ signal: self,
+ coefficient: signal_aggregator.coefficients().get(&self),
+ value,
+ })
+ }
+
+ pub fn precompute(
+ self,
+ signal_aggregator: &SignalAggregator,
+ webpage: &Webpage,
+ ) -> Option<ComputedSignal> {
+ if !self.is_computable_before_search() {
+ return None;
}
+
+ let value = match self {
+ Signal::HostCentrality => Some(webpage.host_centrality),
+ Signal::PageCentrality => Some(webpage.page_centrality),
+ Signal::IsHomepage => Some(webpage.html.url().is_homepage().into()),
+ Signal::CrawlStability => Some(webpage.crawl_stability),
+ Signal::FetchTimeMs => {
+ let fetch_time_ms = webpage.fetch_time_ms as usize;
+ if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {
+ Some(0.0)
+ } else {
+ Some(signal_aggregator.fetch_time_ms_cache[fetch_time_ms])
+ }
+ }
+ Signal::UpdateTimestamp => {
+ let update_timestamp = webpage
+ .html
+ .updated_time()
+ .map(|date| date.timestamp().max(0))
+ .unwrap_or(0) as usize;
+
+ Some(score_timestamp(update_timestamp, signal_aggregator))
+ }
+ Signal::TrackerScore => {
+ let num_trackers = webpage.html.trackers().len() as f64;
+ Some(score_trackers(num_trackers))
+ }
+ Signal::Region => {
+ let region = Region::guess_from(webpage).unwrap_or(Region::All);
+ Some(score_region(region, signal_aggregator))
+ }
+ Signal::Bm25
+ | Signal::Bm25Title
+ | Signal::Bm25CleanBody
+ | Signal::Bm25StemmedTitle
+ | Signal::Bm25StemmedCleanBody
+ | Signal::Bm25AllBody
+ | Signal::Bm25Url
+ | Signal::Bm25Site
+ | Signal::Bm25Domain
+ | Signal::Bm25SiteNoTokenizer
+ | Signal::Bm25DomainNoTokenizer
+ | Signal::Bm25DomainIfHomepage
+ | Signal::Bm25DomainNameIfHomepageNoTokenizer
+ | Signal::Bm25TitleIfHomepage
+ | Signal::Bm25BacklinkText
+ | Signal::Bm25Description
+ | Signal::CrossEncoder
+ | Signal::PersonalCentrality
+ | Signal::TopicCentrality
+ | Signal::InboundSimilarity
+ | Signal::LambdaMART
+ | Signal::QueryCentrality => {
+ tracing::error!("signal {self:?} cannot be precomputed");
+ None
+ }
+ };
+
+ value.map(|value| ComputedSignal {
+ signal: self,
+ coefficient: signal_aggregator.coefficients().get(&self),
+ value,
+ })
}
fn as_fastfield(&self) -> Option<FastField> {
match self {
- Signal::Bm25 => None,
Signal::HostCentrality => Some(FastField::HostCentrality),
Signal::PageCentrality => Some(FastField::PageCentrality),
Signal::IsHomepage => Some(FastField::IsHomepage),
@@ -194,18 +422,94 @@ impl Signal {
Signal::TrackerScore => Some(FastField::TrackerScore),
Signal::Region => Some(FastField::Region),
Signal::CrawlStability => Some(FastField::CrawlStability),
- Signal::PersonalCentrality => None,
- Signal::TopicCentrality => None,
- Signal::QueryCentrality => None,
- Signal::InboundSimilarity => None,
+ _ => None,
+ }
+ }
+
+ fn as_textfield(&self) -> Option<TextField> {
+ match self {
+ Signal::Bm25Title => Some(TextField::Title),
+ Signal::Bm25CleanBody => Some(TextField::CleanBody),
+ Signal::Bm25StemmedTitle => Some(TextField::StemmedTitle),
+ Signal::Bm25StemmedCleanBody => Some(TextField::StemmedCleanBody),
+ Signal::Bm25AllBody => Some(TextField::AllBody),
+ Signal::Bm25Url => Some(TextField::Url),
+ Signal::Bm25Site => Some(TextField::Site),
+ Signal::Bm25Domain => Some(TextField::Domain),
+ Signal::Bm25SiteNoTokenizer => Some(TextField::SiteNoTokenizer),
+ Signal::Bm25DomainNoTokenizer => Some(TextField::DomainNoTokenizer),
+ Signal::Bm25DomainIfHomepage => Some(TextField::DomainIfHomepage),
+ Signal::Bm25DomainNameIfHomepageNoTokenizer => {
+ Some(TextField::DomainNameIfHomepageNoTokenizer)
+ }
+ Signal::Bm25TitleIfHomepage => Some(TextField::TitleIfHomepage),
+ Signal::Bm25BacklinkText => Some(TextField::BacklinkText),
+ Signal::Bm25Description => Some(TextField::Description),
+ _ => None,
+ }
+ }
+}
+
+impl FromStr for Signal {
+ type Err = Error;
+
+ fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {
+ match name {
+ "bm25" => Ok(Signal::Bm25),
+ "bm25_title" => Ok(Signal::Bm25Title),
+ "bm25_clean_body" => Ok(Signal::Bm25CleanBody),
+ "bm25_stemmed_title" => Ok(Signal::Bm25StemmedTitle),
+ "bm25_stemmed_clean_body" => Ok(Signal::Bm25StemmedCleanBody),
+ "bm25_all_body" => Ok(Signal::Bm25AllBody),
+ "bm25_url" => Ok(Signal::Bm25Url),
+ "bm25_site" => Ok(Signal::Bm25Site),
+ "bm25_domain" => Ok(Signal::Bm25Domain),
+ "bm25_site_no_tokenizer" => Ok(Signal::Bm25SiteNoTokenizer),
+ "bm25_domain_no_tokenizer" => Ok(Signal::Bm25DomainNoTokenizer),
+ "bm25_domain_if_homepage" => Ok(Signal::Bm25DomainIfHomepage),
+ "bm25_domain_name_if_homepage_no_tokenizer" => {
+ Ok(Signal::Bm25DomainNameIfHomepageNoTokenizer)
+ }
+ "bm25_title_if_homepage" => Ok(Signal::Bm25TitleIfHomepage),
+ "bm25_backlink_text" => Ok(Signal::Bm25BacklinkText),
+ "bm25_description" => Ok(Signal::Bm25Description),
+ "cross_encoder" => Ok(Signal::CrossEncoder),
+ "host_centrality" => Ok(Signal::HostCentrality),
+ "page_centrality" => Ok(Signal::PageCentrality),
+ "is_homepage" => Ok(Signal::IsHomepage),
+ "fetch_time_ms" => Ok(Signal::FetchTimeMs),
+ "update_timestamp" => Ok(Signal::UpdateTimestamp),
+ "tracker_score" => Ok(Signal::TrackerScore),
+ "region" => Ok(Signal::Region),
+ "personal_centrality" => Ok(Signal::PersonalCentrality),
+ "topic_centrality" => Ok(Signal::TopicCentrality),
+ "query_centrality" => Ok(Signal::QueryCentrality),
+ "inbound_similarity" => Ok(Signal::InboundSimilarity),
+ "crawl_stability" => Ok(Signal::CrawlStability),
+ "lambda_mart" => Ok(Signal::LambdaMART),
+ _ => Err(Error::UnknownSignal(name.to_string())),
}
}
}
-#[derive(Debug, Clone)]
-pub struct FieldBoost(Vec<Option<f64>>);
+fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
+ match aggregator.region_count.as_ref() {
+ Some(region_count) => {
+ let boost = aggregator.selected_region.map_or(0.0, |region| {
+ if region == webpage_region {
+ 50.0
+ } else {
+ 0.0
+ }
+ });
-#[derive(Debug, Clone)]
+ boost + region_count.score(&webpage_region)
+ }
+ None => 0.0,
+ }
+}
+
+#[derive(Debug, Clone, Default)]
pub struct SignalCoefficient(Vec<Option<f64>>);
impl SignalCoefficient {
@@ -234,68 +538,49 @@ impl SignalCoefficient {
}
}
-impl FieldBoost {
- pub fn get(&self, field: &TextField) -> f64 {
- self.0
- .get((*field) as usize)
- .copied()
- .flatten()
- .or_else(|| Field::Text(*field).boost().map(|s| s as f64))
- .unwrap_or(1.0)
- }
-
- pub fn new(scores: impl Iterator<Item = (TextField, f64)>) -> Self {
- let mut fast_scores = Vec::new();
-
- for (field, score) in scores {
- let idx = field as usize;
-
- while idx >= fast_scores.len() {
- fast_scores.push(None);
- }
-
- fast_scores[idx] = Some(score);
- }
+#[derive(Clone)]
+struct TextFieldData {
+ postings: Vec<SegmentPostings>,
+ weight: Bm25Weight,
+ fieldnorm_reader: FieldNormReader,
+}
- Self(fast_scores)
- }
+#[derive(Clone)]
+struct SegmentReader {
+ text_fields: EnumMap<TextField, TextFieldData>,
+ fastfield_reader: Arc<fastfield_reader::SegmentReader>,
}
#[derive(Clone)]
pub struct SignalAggregator {
- fastfield_reader: Option<Arc<fastfield_reader::SegmentReader>>,
+ query: Option<Query>,
signal_coefficients: SignalCoefficient,
+ segment_reader: Option<SegmentReader>,
personal_centrality: Option<Arc<online_harmonic::Scorer>>,
inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,
- field_boost: FieldBoost,
fetch_time_ms_cache: Vec<f64>,
update_time_cache: Vec<f64>,
topic_scorer: Option<topic::Scorer>,
query_centrality: Option<Arc<online_harmonic::Scorer>>,
+ region_count: Option<Arc<RegionCount>>,
+ selected_region: Option<Region>,
+ current_timestamp: Option<usize>,
}
impl std::fmt::Debug for SignalAggregator {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SignalAggregator")
- .field("signal_coefficients", &self.signal_coefficients)
- .field("field_boost", &self.field_boost)
+ .field("query", &self.query)
.finish()
}
}
-impl Default for SignalAggregator {
- fn default() -> Self {
- Self::new(Vec::new().into_iter(), Vec::new().into_iter())
- }
-}
-
impl SignalAggregator {
- pub fn new(
- coefficients: impl Iterator<Item = (Signal, f64)>,
- boosts: impl Iterator<Item = (TextField, f64)>,
- ) -> Self {
- let signal_coefficients = SignalCoefficient::new(coefficients);
- let field_boost = FieldBoost::new(boosts);
+ pub fn new(query: Option<Query>) -> Self {
+ let signal_coefficients = query
+ .as_ref()
+ .map(|q| q.signal_coefficients())
+ .unwrap_or_default();
let fetch_time_ms_cache: Vec<_> = (0..1000)
.map(|fetch_time| 1.0 / (fetch_time as f64 + 1.0))
@@ -306,20 +591,74 @@ impl SignalAggregator {
.collect();
Self {
- fastfield_reader: None,
+ segment_reader: None,
personal_centrality: None,
inbound_similariy: None,
signal_coefficients,
- field_boost,
fetch_time_ms_cache,
update_time_cache,
topic_scorer: None,
query_centrality: None,
+ region_count: None,
+ selected_region: None,
+ current_timestamp: None,
+ query,
}
}
- pub fn register_segment(&mut self, reader: Arc<fastfield_reader::SegmentReader>) {
- self.fastfield_reader = Some(reader);
+ pub fn register_segment(
+ &mut self,
+ tv_searcher: &tantivy::Searcher,
+ segment_reader: &tantivy::SegmentReader,
+ fastfield_reader: Arc<fastfield_reader::SegmentReader>,
+ ) -> Result<()> {
+ let mut text_fields = EnumMap::new();
+ let schema = tv_searcher.schema();
+
+ if let Some(query) = &self.query {
+ if !query.simple_terms().is_empty() {
+ for signal in ALL_SIGNALS {
+ if let Some(text_field) = signal.as_textfield() {
+ let tv_field = schema.get_field(text_field.name()).unwrap();
+ let terms: Vec<_> = query
+ .simple_terms()
+ .iter()
+ .map(|text| tantivy::Term::from_field_text(tv_field, text))
+ .collect();
+
+ let weight = Bm25Weight::for_terms(tv_searcher, &terms)?;
+
+ let fieldnorm_reader = segment_reader.get_fieldnorms_reader(tv_field)?;
+ let inverted_index = segment_reader.inverted_index(tv_field)?;
+
+ let mut postings = Vec::with_capacity(terms.len());
+ for term in &terms {
+ if let Some(p) =
+ inverted_index.read_postings(term, text_field.index_option())?
+ {
+ postings.push(p);
+ }
+ }
+
+ text_fields.insert(
+ text_field,
+ TextFieldData {
+ postings,
+ weight,
+ fieldnorm_reader,
+ },
+ );
+ }
+ }
+ }
+ }
+
+ self.segment_reader = Some(SegmentReader {
+ text_fields,
+ fastfield_reader,
+ });
+
+ Ok(())
}
pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
@@ -338,6 +677,18 @@ impl SignalAggregator {
self.inbound_similariy = Some(Arc::new(scorer));
}
+ pub fn set_region_count(&mut self, region_count: RegionCount) {
+ self.region_count = Some(Arc::new(region_count));
+ }
+
+ pub fn set_selected_region(&mut self, region: Region) {
+ self.selected_region = Some(region);
+ }
+
+ pub fn set_current_timestamp(&mut self, current_timestamp: usize) {
+ self.current_timestamp = Some(current_timestamp);
+ }
+
pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {
self.topic_scorer
.as_ref()
@@ -364,124 +715,31 @@ impl SignalAggregator {
.unwrap_or_default()
}
- pub fn score(
- &self,
+ pub fn compute_signals(
+ &mut self,
doc: DocId,
- bm25: tantivy::Score,
- region_count: &Arc<RegionCount>,
- current_timestamp: usize,
- selected_region: Option<Region>,
- ) -> Score {
- let host_id = self.fastfield_reader.as_ref().and_then(|reader| {
- let node_id: Option<u64> = reader
- .get_field_reader(&FastField::HostNodeID)
- .get(&doc)
- .into();
- let node_id = node_id.unwrap();
-
- if node_id == u64::MAX {
- None
- } else {
- Some(NodeID::from(node_id))
- }
- });
-
- let topic_score = host_id.and_then(|host_id| self.topic_centrality(host_id));
-
- let query_centrality = host_id.and_then(|host_id| self.query_centrality(host_id));
-
- let personal_centrality = host_id.map(|host_id| self.personal_centrality(host_id));
- let inbound_similarity = host_id.map(|host_id| self.inbound_similarity(host_id));
-
- let score = ALL_SIGNALS
+ ) -> impl Iterator<Item = Option<ComputedSignal>> + '_ {
+ ALL_SIGNALS
.into_iter()
- .map(|signal| {
- let fastfield_value = signal.as_fastfield().and_then(|field| {
- self.fastfield_reader
- .as_ref()
- .and_then(|reader| reader.get_field_reader(&field).get(&doc).into())
- });
-
- self.coefficients().get(&signal)
- * signal.value(
- region_count,
- self,
- SignalValue {
- bm25,
- fastfield_value,
- current_timestamp,
- selected_region,
- personal_centrality,
- inbound_similarity,
- topic_score,
- query_centrality,
- },
- )
- })
- .sum();
-
- Score { bm25, total: score }
+ .map(move |signal| signal.compute(self, doc))
}
- pub fn precompute_score(&self, webpage: &Webpage, region_count: &RegionCount) -> f64 {
+ pub fn precompute_score(&self, webpage: &Webpage) -> f64 {
ALL_SIGNALS
.into_iter()
- .filter(|signal| signal.is_computable_before_search())
- .map(|signal| {
- let fastfield_value = match &signal {
- Signal::HostCentrality => {
- (webpage.host_centrality * (FLOAT_SCALING as f64)) as u64
- }
- Signal::PageCentrality => {
- (webpage.page_centrality * (FLOAT_SCALING as f64)) as u64
- }
- Signal::IsHomepage => webpage.html.url().is_homepage().into(),
- Signal::FetchTimeMs => webpage.fetch_time_ms,
- Signal::UpdateTimestamp => webpage
- .html
- .updated_time()
- .map(|date| date.timestamp().max(0) as u64)
- .unwrap_or(0),
- Signal::TrackerScore => webpage.html.trackers().len() as u64,
- Signal::Region => Region::guess_from(webpage).unwrap_or(Region::All).id(),
- Signal::CrawlStability => {
- (webpage.crawl_stability * (FLOAT_SCALING as f64)) as u64
- }
- Signal::Bm25
- | Signal::PersonalCentrality
- | Signal::TopicCentrality
- | Signal::InboundSimilarity
- | Signal::QueryCentrality => {
- panic!("signal cannot be determined from webpage")
- }
- };
-
- let current_timestamp = Utc::now().timestamp() as usize;
-
- self.coefficients().get(&signal)
- * signal.value(
- &region_count,
- self,
- SignalValue {
- bm25: 0.0,
- fastfield_value: Some(fastfield_value),
- current_timestamp,
- selected_region: None,
- personal_centrality: None,
- topic_score: None,
- query_centrality: None,
- inbound_similarity: None,
- },
- )
- })
+ .filter_map(|signal| signal.precompute(self, webpage))
+ .map(|computed| computed.coefficient * computed.value)
.sum()
}
- pub fn coefficients(&self) -> &SignalCoefficient {
+ fn coefficients(&self) -> &SignalCoefficient {
&self.signal_coefficients
}
+}
- pub fn field_boosts(&self) -> &FieldBoost {
- &self.field_boost
- }
+#[derive(Debug, Clone, Copy)]
+pub struct ComputedSignal {
+ pub signal: Signal,
+ pub coefficient: f64,
+ pub value: f64,
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 837f8ff5..f9eff207 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -41,14 +41,20 @@ pub enum TextField {
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
TitleIfHomepage,
BacklinkText,
- PrimaryImage,
Description,
+ PrimaryImage,
HostTopic,
DmozDescription,
SchemaOrgJson,
FlattenedSchemaOrgJson,
}
+impl From<TextField> for usize {
+ fn from(value: TextField) -> Self {
+ value as usize
+ }
+}
+
impl TextField {
pub fn tokenizer(&self) -> Tokenizer {
match self {
@@ -107,6 +113,31 @@ impl TextField {
TextField::FlattenedSchemaOrgJson => true,
}
}
+
+ pub fn name(&self) -> &str {
+ match self {
+ TextField::Title => "title",
+ TextField::CleanBody => "body",
+ TextField::Url => "url",
+ TextField::Site => "site",
+ TextField::Domain => "domain",
+ TextField::SiteNoTokenizer => "site_no_tokenizer",
+ TextField::DomainNoTokenizer => "domain_no_tokenizer",
+ TextField::BacklinkText => "backlink_text",
+ TextField::StemmedTitle => "stemmed_title",
+ TextField::StemmedCleanBody => "stemmed_body",
+ TextField::DomainIfHomepage => "domain_if_homepage",
+ TextField::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",
+ TextField::Description => "description",
+ TextField::PrimaryImage => "primary_image_uuid",
+ TextField::TitleIfHomepage => "title_if_homepage",
+ TextField::AllBody => "all_body",
+ TextField::HostTopic => "host_topic",
+ TextField::DmozDescription => "dmoz_description",
+ TextField::SchemaOrgJson => "schema_org_json",
+ TextField::FlattenedSchemaOrgJson => "flattened_schema_org_json",
+ }
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -141,6 +172,47 @@ pub enum FastField {
NumFlattenedSchemaTokens,
}
+impl FastField {
+ pub fn name(&self) -> &str {
+ match self {
+ FastField::HostCentrality => "host_centrality",
+ FastField::PageCentrality => "page_centrality",
+ FastField::IsHomepage => "is_homepage",
+ FastField::FetchTimeMs => "fetch_time_ms",
+ FastField::LastUpdated => "last_updated",
+ FastField::TrackerScore => "tracker_score",
+ FastField::Region => "region",
+ FastField::NumUrlTokens => "num_url_tokens",
+ FastField::NumTitleTokens => "num_title_tokens",
+ FastField::NumCleanBodyTokens => "num_clean_body_tokens",
+ FastField::NumDescriptionTokens => "num_description_tokens",
+ FastField::NumDomainTokens => "num_domain_tokens",
+ FastField::NumSiteTokens => "num_site_tokens",
+ FastField::NumFlattenedSchemaTokens => "num_flattened_schema_tokens",
+ FastField::SiteHash1 => "site_hash1",
+ FastField::SiteHash2 => "site_hash2",
+ FastField::UrlWithoutQueryHash1 => "url_without_query_hash1",
+ FastField::UrlWithoutQueryHash2 => "url_without_query_hash2",
+ FastField::TitleHash1 => "title_hash1",
+ FastField::TitleHash2 => "title_hash2",
+ FastField::UrlHash1 => "url_hash1",
+ FastField::UrlHash2 => "url_hash2",
+ FastField::DomainHash1 => "domain_hash1",
+ FastField::DomainHash2 => "domain_hash2",
+ FastField::PreComputedScore => "pre_computed_score",
+ FastField::HostNodeID => "host_node_id",
+ FastField::SimHash => "sim_hash",
+ FastField::CrawlStability => "crawl_stability",
+ }
+ }
+}
+
+impl From<FastField> for usize {
+ fn from(value: FastField) -> Self {
+ value as usize
+ }
+}
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Field {
Fast(FastField),
@@ -400,84 +472,8 @@ impl Field {
pub fn name(&self) -> &str {
match self {
- Field::Text(TextField::Title) => "title",
- Field::Text(TextField::CleanBody) => "body",
- Field::Text(TextField::Url) => "url",
- Field::Text(TextField::Site) => "site",
- Field::Text(TextField::Domain) => "domain",
- Field::Text(TextField::SiteNoTokenizer) => "site_no_tokenizer",
- Field::Text(TextField::DomainNoTokenizer) => "domain_no_tokenizer",
- Field::Text(TextField::BacklinkText) => "backlink_text",
- Field::Text(TextField::StemmedTitle) => "stemmed_title",
- Field::Text(TextField::StemmedCleanBody) => "stemmed_body",
- Field::Text(TextField::DomainIfHomepage) => "domain_if_homepage",
- Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
- "domain_name_if_homepage_no_tokenizer"
- }
- Field::Text(TextField::Description) => "description",
- Field::Text(TextField::PrimaryImage) => "primary_image_uuid",
- Field::Text(TextField::TitleIfHomepage) => "title_if_homepage",
- Field::Text(TextField::AllBody) => "all_body",
- Field::Text(TextField::HostTopic) => "host_topic",
- Field::Text(TextField::DmozDescription) => "dmoz_description",
- Field::Text(TextField::SchemaOrgJson) => "schema_org_json",
- Field::Text(TextField::FlattenedSchemaOrgJson) => "flattened_schema_org_json",
- // FAST FIELDS
- Field::Fast(FastField::HostCentrality) => "host_centrality",
- Field::Fast(FastField::PageCentrality) => "page_centrality",
- Field::Fast(FastField::IsHomepage) => "is_homepage",
- Field::Fast(FastField::FetchTimeMs) => "fetch_time_ms",
- Field::Fast(FastField::LastUpdated) => "last_updated",
- Field::Fast(FastField::TrackerScore) => "tracker_score",
- Field::Fast(FastField::Region) => "region",
- Field::Fast(FastField::NumUrlTokens) => "num_url_tokens",
- Field::Fast(FastField::NumTitleTokens) => "num_title_tokens",
- Field::Fast(FastField::NumCleanBodyTokens) => "num_clean_body_tokens",
- Field::Fast(FastField::NumDescriptionTokens) => "num_description_tokens",
- Field::Fast(FastField::NumDomainTokens) => "num_domain_tokens",
- Field::Fast(FastField::NumSiteTokens) => "num_site_tokens",
- Field::Fast(FastField::NumFlattenedSchemaTokens) => "num_flattened_schema_tokens",
- Field::Fast(FastField::SiteHash1) => "site_hash1",
- Field::Fast(FastField::SiteHash2) => "site_hash2",
- Field::Fast(FastField::UrlWithoutQueryHash1) => "url_without_query_hash1",
- Field::Fast(FastField::UrlWithoutQueryHash2) => "url_without_query_hash2",
- Field::Fast(FastField::TitleHash1) => "title_hash1",
- Field::Fast(FastField::TitleHash2) => "title_hash2",
- Field::Fast(FastField::UrlHash1) => "url_hash1",
- Field::Fast(FastField::UrlHash2) => "url_hash2",
- Field::Fast(FastField::DomainHash1) => "domain_hash1",
- Field::Fast(FastField::DomainHash2) => "domain_hash2",
- Field::Fast(FastField::PreComputedScore) => "pre_computed_score",
- Field::Fast(FastField::HostNodeID) => "host_node_id",
- Field::Fast(FastField::SimHash) => "sim_hash",
- Field::Fast(FastField::CrawlStability) => "crawl_stability",
- }
- }
-
- pub fn boost(&self) -> Option<f32> {
- match self {
- Field::Text(TextField::Site) => Some(2.0),
- Field::Text(TextField::TitleIfHomepage) => Some(3.0),
- Field::Text(TextField::DomainIfHomepage) => Some(6.0),
- Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => Some(30.0),
- Field::Text(TextField::StemmedCleanBody) | Field::Text(TextField::StemmedTitle) => {
- Some(0.5)
- }
- Field::Text(TextField::CleanBody) => Some(4.0),
- Field::Text(TextField::DmozDescription) => Some(3.0),
- Field::Text(TextField::Title) => Some(15.0),
- Field::Text(TextField::Url) => Some(1.0),
- Field::Text(TextField::Domain) => Some(1.0),
- Field::Text(TextField::AllBody) => Some(0.01),
- Field::Text(TextField::HostTopic) => Some(1.0),
- Field::Text(TextField::BacklinkText) => Some(4.0),
- Field::Text(TextField::SiteNoTokenizer)
- | Field::Text(TextField::DomainNoTokenizer)
- | Field::Text(TextField::Description)
- | Field::Text(TextField::FlattenedSchemaOrgJson)
- | Field::Text(TextField::SchemaOrgJson)
- | Field::Text(TextField::PrimaryImage) => None,
- Field::Fast(_) => None,
+ Field::Text(t) => t.name(),
+ Field::Fast(f) => f.name(),
}
}
@@ -495,53 +491,6 @@ impl Field {
matches!(self, Field::Fast(_))
}
- pub fn from_name(name: String) -> Option<Field> {
- match name.as_str() {
- "title" => Some(Field::Text(TextField::Title)),
- "body" => Some(Field::Text(TextField::CleanBody)),
- "url" => Some(Field::Text(TextField::Url)),
- "site" => Some(Field::Text(TextField::Site)),
- "backlink_text" => Some(Field::Text(TextField::BacklinkText)),
- "stemmed_title" => Some(Field::Text(TextField::StemmedTitle)),
- "stemmed_body" => Some(Field::Text(TextField::StemmedCleanBody)),
- "domain" => Some(Field::Text(TextField::Domain)),
- "domain_if_homepage" => Some(Field::Text(TextField::DomainIfHomepage)),
- "primary_image_uuid" => Some(Field::Text(TextField::PrimaryImage)),
- "domain_name_if_homepage_no_tokenizer" => {
- Some(Field::Text(TextField::DomainNameIfHomepageNoTokenizer))
- }
- "description" => Some(Field::Text(TextField::Description)),
- "all_body" => Some(Field::Text(TextField::AllBody)),
- "title_if_homepage" => Some(Field::Text(TextField::TitleIfHomepage)),
- "host_topic" => Some(Field::Text(TextField::HostTopic)),
- "dmoz_description" => Some(Field::Text(TextField::DmozDescription)),
- "schema_org_json" => Some(Field::Text(TextField::SchemaOrgJson)),
- "flattened_schema_org_json" => Some(Field::Text(TextField::FlattenedSchemaOrgJson)),
- "host_centrality" => Some(Field::Fast(FastField::HostCentrality)),
- "page_centrality" => Some(Field::Fast(FastField::PageCentrality)),
- "is_homepage" => Some(Field::Fast(FastField::IsHomepage)),
- "fetch_time_ms" => Some(Field::Fast(FastField::FetchTimeMs)),
- "last_updated" => Some(Field::Fast(FastField::LastUpdated)),
- "tracker_score" => Some(Field::Fast(FastField::TrackerScore)),
- "region" => Some(Field::Fast(FastField::Region)),
- "site_hash1" => Some(Field::Fast(FastField::SiteHash1)),
- "site_hash2" => Some(Field::Fast(FastField::SiteHash2)),
- "url_without_query_hash1" => Some(Field::Fast(FastField::UrlWithoutQueryHash1)),
- "url_without_query_hash2" => Some(Field::Fast(FastField::UrlWithoutQueryHash2)),
- "url_hash1" => Some(Field::Fast(FastField::UrlHash1)),
- "url_hash2" => Some(Field::Fast(FastField::UrlHash2)),
- "domain_hash1" => Some(Field::Fast(FastField::DomainHash1)),
- "domain_hash2" => Some(Field::Fast(FastField::DomainHash2)),
- "title_hash1" => Some(Field::Fast(FastField::TitleHash1)),
- "title_hash2" => Some(Field::Fast(FastField::TitleHash2)),
- "pre_computed_score" => Some(Field::Fast(FastField::PreComputedScore)),
- "host_node_id" => Some(Field::Fast(FastField::HostNodeID)),
- "sim_hash" => Some(Field::Fast(FastField::SimHash)),
- "crawl_stability" => Some(Field::Fast(FastField::CrawlStability)),
- _ => None,
- }
- }
-
pub fn as_text(&self) -> Option<TextField> {
match self {
Field::Fast(_) => None,
diff --git a/core/src/search_ctx.rs b/core/src/search_ctx.rs
index e924dcb8..2bb3af57 100644
--- a/core/src/search_ctx.rs
+++ b/core/src/search_ctx.rs
@@ -14,6 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::fastfield_reader::FastFieldReader;
+
+#[derive(Clone)]
pub struct Ctx {
pub tv_searcher: tantivy::Searcher,
+ pub fastfield_reader: FastFieldReader,
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 36b1e9f4..40a1706a 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -474,7 +474,7 @@ impl DistributedSearcher {
&self,
query: &SearchQuery,
) -> Result<Option<Vec<DisplayedWebpage>>> {
- if query.optic_program.is_some() || query.offset > 0 {
+ if query.optic_program.is_some() || query.page > 0 {
return Ok(None);
}
@@ -489,7 +489,7 @@ impl DistributedSearcher {
};
let pipeline: RankingPipeline<ScoredWebsitePointer> =
- RankingPipeline::for_query(&mut query, self.cross_encoder.clone())?;
+ RankingPipeline::reranking_for_query(&mut query, self.cross_encoder.clone())?;
let initial_results = self.search_initial(&query).await;
@@ -542,7 +542,7 @@ impl DistributedSearcher {
let mut search_query = query.clone();
let pipeline: RankingPipeline<ScoredWebsitePointer> =
- RankingPipeline::for_query(&mut search_query, self.cross_encoder.clone())?;
+ RankingPipeline::reranking_for_query(&mut search_query, self.cross_encoder.clone())?;
let initial_results = self.search_initial(&search_query).await;
@@ -565,15 +565,9 @@ impl DistributedSearcher {
.map(|result| result.local_result.num_websites)
.sum();
- let num_shard_websites: usize = initial_results
- .iter()
- .map(|res| res.local_result.websites.len())
- .sum();
-
+ let has_more_results = initial_results.iter().any(|res| res.local_result.has_more);
let top_websites = self.combine_results(initial_results, pipeline);
- let has_more_results = num_shard_websites != top_websites.len();
-
// retrieve webpages
let mut retrieved_webpages: Vec<_> = self
.retrieve_webpages(&top_websites, &query.query)
@@ -612,7 +606,7 @@ impl DistributedSearcher {
}
fn widget(&self, query: &SearchQuery) -> Option<Widget> {
- if query.offset > 0 {
+ if query.page > 0 {
return None;
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index f75e26fc..c8d117d8 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -16,7 +16,6 @@
use std::str::FromStr;
-use optics::Optic;
use uuid::Uuid;
use crate::entity_index::{EntityIndex, StoredEntity};
@@ -26,9 +25,9 @@ use crate::inverted_index::RetrievedWebpage;
use crate::query::parser::Term;
use crate::query::{self, Query};
use crate::ranking::centrality_store::SearchCentralityStore;
-use crate::ranking::optics::CreateAggregator;
-use crate::ranking::pipeline::RankingWebsite;
+use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
+use crate::schema::TextField;
use crate::search_ctx::Ctx;
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
use crate::spell::Correction;
@@ -54,6 +53,12 @@ impl From<Index> for LocalSearcher {
}
}
+struct InvertedIndexResult {
+ webpages: Vec<RankingWebsite>,
+ num_hits: usize,
+ has_more: bool,
+}
+
impl LocalSearcher {
pub fn new(index: Index) -> Self {
LocalSearcher {
@@ -76,58 +81,13 @@ impl LocalSearcher {
self.topic_centrality = Some(topic_centrality);
}
- fn parse_query(&self, query: &SearchQuery, ctx: &Ctx) -> Result<(Query, Optic)> {
- let optic = match query.optic_program.as_ref() {
- Some(program) => Some(optics::parse(program)?),
- None => None,
- };
-
- let query_aggregator = optic
- .as_ref()
- .and_then(|optic| {
- optic
- .pipeline
- .as_ref()
- .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
- })
- .unwrap_or_default();
-
- let mut parsed_query = Query::parse(
- &query.query,
- self.index.schema(),
- self.index.tokenizers(),
- &query_aggregator,
- )?;
-
- parsed_query.set_num_results(query.num_results);
- parsed_query.set_offset(query.offset);
-
- if let Some(region) = query.selected_region {
- parsed_query.set_region(region);
- }
-
- let mut optics = Vec::new();
-
- if let Some(optic) = optic {
- optics.push(optic);
- }
-
- if let Some(site_rankings) = &query.site_rankings {
- optics.push(site_rankings.clone().into_optic())
- }
-
- let mut optic = Optic::default();
-
- for o in optics {
- optic = optic.try_merge(o)?;
- }
-
- parsed_query.set_optic(&optic, &self.index, ctx);
+ fn parse_query(&self, ctx: &Ctx, query: &SearchQuery) -> Result<Query> {
+ let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;
if parsed_query.is_empty() {
Err(Error::EmptyQuery)
} else {
- Ok((parsed_query, optic))
+ Ok(parsed_query)
}
}
@@ -139,9 +99,8 @@ impl LocalSearcher {
aggregator: SignalAggregator,
) -> Result<Ranker> {
let mut ranker = Ranker::new(
- self.index.region_count.clone(),
aggregator,
- self.index.inverted_index.fastfield_reader(ctx),
+ self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),
);
if let Some(region) = query.region() {
@@ -162,7 +121,9 @@ impl LocalSearcher {
.with_max_docs(10_000, self.index.num_segments())
.with_num_results(100);
- let top_host_nodes = self.index.top_nodes(query, ctx, ranker.collector())?;
+ let top_host_nodes = self
+ .index
+ .top_nodes(query, ctx, ranker.collector(ctx.clone()))?;
if !top_host_nodes.is_empty() {
let harmonic = centrality_store
.online_harmonic
@@ -179,35 +140,33 @@ impl LocalSearcher {
fn search_inverted_index(
&self,
+ ctx: &Ctx,
query: &SearchQuery,
de_rank_similar: bool,
- ) -> Result<(Vec<RankingWebsite>, usize)> {
- let searcher = self.index.inverted_index.local_search_ctx();
-
- let (parsed_query, optic) = self.parse_query(query, &searcher)?;
+ ) -> Result<InvertedIndexResult> {
+ let mut query = query.clone();
+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::ltr_for_query(&mut query);
+ let parsed_query = self.parse_query(ctx, &query)?;
- let mut aggregator = optic
- .pipeline
- .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
- .unwrap_or_default();
+ let mut aggregator = SignalAggregator::new(Some(parsed_query.clone()));
if let Some(store) = &self.centrality_store {
aggregator.set_personal_harmonic(online_centrality_scorer(
- &optic.site_rankings,
+ parsed_query.site_rankings(),
&store.online_harmonic,
&store.node2id,
));
- let liked_sites: Vec<_> = optic
- .site_rankings
+ let liked_sites: Vec<_> = parsed_query
+ .site_rankings()
.liked
.iter()
.map(|site| Node::from(site.clone()).into_host())
.filter_map(|node| store.node2id.get(&node).copied())
.collect();
- let disliked_sites: Vec<_> = optic
- .site_rankings
+ let disliked_sites: Vec<_> = parsed_query
+ .site_rankings()
.disliked
.iter()
.map(|site| Node::from(site.clone()).into_host())
@@ -221,21 +180,50 @@ impl LocalSearcher {
)
}
- let ranker = self.ranker(&parsed_query, &searcher, de_rank_similar, aggregator)?;
+ aggregator.set_region_count(self.index.region_count.clone());
+
+ let ranker = self.ranker(&parsed_query, ctx, de_rank_similar, aggregator)?;
let res = self.index.inverted_index.search_initial(
&parsed_query,
- &searcher,
- ranker.collector(),
+ ctx,
+ ranker.collector(ctx.clone()),
)?;
+ let fastfield_reader = self.index.inverted_index.fastfield_reader(&ctx.tv_searcher);
+
let ranking_websites = self.index.inverted_index.retrieve_ranking_websites(
- &searcher,
+ ctx,
res.top_websites,
- &ranker.aggregator(),
+ ranker.aggregator(),
+ &fastfield_reader,
)?;
- Ok((ranking_websites, res.num_websites))
+ let pipe_top_n = pipeline.top_n;
+ let mut ranking_websites = pipeline.apply(ranking_websites);
+
+ let schema = self.index.schema();
+ for website in &mut ranking_websites {
+ let doc = ctx.tv_searcher.doc(website.pointer.address.into())?;
+ website.title = Some(
+ doc.get_first(schema.get_field(TextField::Title.name()).unwrap())
+ .map(|text| text.as_text().unwrap().to_string())
+ .unwrap_or_default(),
+ );
+ website.clean_body = Some(
+ doc.get_first(schema.get_field(TextField::CleanBody.name()).unwrap())
+ .map(|text| text.as_text().unwrap().to_string())
+ .unwrap_or_default(),
+ );
+ }
+
+ let has_more = pipe_top_n == ranking_websites.len();
+
+ Ok(InvertedIndexResult {
+ webpages: ranking_websites,
+ num_hits: res.num_websites,
+ has_more,
+ })
}
fn spell_correction(&self, query: &SearchQuery) -> Option<Correction> {
@@ -261,7 +249,8 @@ impl LocalSearcher {
query: &SearchQuery,
de_rank_similar: bool,
) -> Result<InitialWebsiteResult> {
- let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;
+ let ctx = self.index.inverted_index.local_search_ctx();
+ let inverted_index_result = self.search_inverted_index(&ctx, query, de_rank_similar)?;
let correction = self.spell_correction(query);
let sidebar = self
.entity_sidebar(query)
@@ -269,8 +258,9 @@ impl LocalSearcher {
Ok(InitialWebsiteResult {
spell_corrected_query: correction,
- websites,
- num_websites,
+ websites: inverted_index_result.webpages,
+ num_websites: inverted_index_result.num_hits,
+ has_more: inverted_index_result.has_more,
entity_sidebar: sidebar,
})
}
@@ -280,12 +270,12 @@ impl LocalSearcher {
websites: &[inverted_index::WebsitePointer],
query: &str,
) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- let query = Query::parse(
- query,
- self.index.schema(),
- self.index.tokenizers(),
- &SignalAggregator::default(),
- )?;
+ let ctx = self.index.inverted_index.local_search_ctx();
+ let query = SearchQuery {
+ query: query.to_string(),
+ ..Default::default()
+ };
+ let query = Query::parse(&ctx, &query, &self.index.inverted_index)?;
if query.is_empty() {
return Err(Error::EmptyQuery);
@@ -299,10 +289,7 @@ impl LocalSearcher {
use std::{sync::Arc, time::Instant};
use crate::{
- ranking::{
- models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},
- pipeline::RankingPipeline,
- },
+ ranking::models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},
search_prettifier::Sidebar,
};
@@ -310,10 +297,11 @@ impl LocalSearcher {
let mut search_query = query.clone();
let pipeline = match CrossEncoderModel::open("data/cross_encoder") {
- Ok(model) => RankingPipeline::for_query(&mut search_query, Arc::new(model))?,
- Err(_) => {
- RankingPipeline::for_query(&mut search_query, Arc::new(DummyCrossEncoder {}))?
- }
+ Ok(model) => RankingPipeline::reranking_for_query(&mut search_query, Arc::new(model))?,
+ Err(_) => RankingPipeline::reranking_for_query(
+ &mut search_query,
+ Arc::new(DummyCrossEncoder {}),
+ )?,
};
let search_result = self.search_initial(&search_query, true)?;
@@ -431,7 +419,7 @@ mod tests {
let urls: Vec<_> = searcher
.search(&SearchQuery {
query: "test".to_string(),
- offset: p * NUM_RESULTS_PER_PAGE,
+ page: p,
..Default::default()
})
.unwrap()
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 774f476f..6bce9594 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -57,7 +57,7 @@ pub struct WebsitesResult {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchQuery {
pub query: String,
- pub offset: usize,
+ pub page: usize,
pub num_results: usize,
pub selected_region: Option<Region>,
pub optic_program: Option<String>,
@@ -69,6 +69,7 @@ pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<Correction>,
pub num_websites: usize,
pub websites: Vec<RankingWebsite>,
+ pub has_more: bool,
pub entity_sidebar: Option<DisplayedEntity>,
}
@@ -79,7 +80,7 @@ impl Default for SearchQuery {
// to ensure the developer considers what the default should be.
Self {
query: Default::default(),
- offset: Default::default(),
+ page: Default::default(),
num_results: NUM_RESULTS_PER_PAGE,
selected_region: Default::default(),
optic_program: Default::default(),
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 606bc003..3ad6087a 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -237,7 +237,7 @@ pub fn generate(query: &Query, text: &str, region: &Region) -> String {
None => whatlang::detect_lang(text).unwrap_or(Lang::Eng),
};
- let snippet = snippet_string(text, &query.simple_terms(), lang);
+ let snippet = snippet_string(text, query.simple_terms(), lang);
snippet.to_html()
}
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index 7ae41d20..660256a6 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -83,10 +83,10 @@ impl TopicCentrality {
.get_field(Field::Text(TextField::CleanBody).name())
.unwrap();
- let ctx = index.inverted_index.local_search_ctx();
let mut term_count: HashMap<String, u64> = HashMap::new();
- for segment in ctx.tv_searcher.segment_readers() {
+ let tv_searcher = index.inverted_index.tv_searcher();
+ for segment in tv_searcher.segment_readers() {
let reader = segment.inverted_index(body_field).unwrap();
let mut stream = reader.terms().stream().unwrap();
@@ -100,8 +100,7 @@ impl TopicCentrality {
}
let top_topics = topics.top_topics(NUM_TOPICS);
- let topic_field = ctx
- .tv_searcher
+ let topic_field = tv_searcher
.schema()
.get_field(Field::Text(TextField::HostTopic).name())
.unwrap();
@@ -126,7 +125,7 @@ impl TopicCentrality {
let query = BooleanQuery::new(vec![(Occur::Must, topic), (Occur::Must, term)]);
- let topic_freq = ctx.tv_searcher.search(&query, &collector).unwrap();
+ let topic_freq = tv_searcher.search(&query, &collector).unwrap();
scores[i] = topic_freq as f64 / total_doc_freq as f64;
}
@@ -134,11 +133,13 @@ impl TopicCentrality {
})
.collect();
+ let mut aggregator = SignalAggregator::new(None);
+ aggregator.set_region_count(index.region_count.clone());
+
let score_tweaker = InitialScoreTweaker::new(
- Arc::new(index.region_count.clone()),
- None,
- SignalAggregator::default(),
- index.inverted_index.fastfield_reader(&ctx),
+ tv_searcher.clone(),
+ aggregator,
+ index.inverted_index.fastfield_reader(&tv_searcher),
);
let graph_node_id = index
@@ -146,8 +147,9 @@ impl TopicCentrality {
.get_field(Field::Fast(FastField::HostNodeID).name())
.unwrap();
- let collector = TopDocs::with_limit(1000, index.inverted_index.fastfield_reader(&ctx))
- .tweak_score(score_tweaker);
+ let collector =
+ TopDocs::with_limit(1000, index.inverted_index.fastfield_reader(&tv_searcher))
+ .tweak_score(score_tweaker);
let mut nodes: Vec<_> = webgraph.nodes().collect();
nodes.sort();
@@ -159,13 +161,12 @@ impl TopicCentrality {
let topic = tantivy::Term::from_facet(topic_field, &facet);
let query = TermQuery::new(topic, IndexRecordOption::Basic).box_clone();
- let top_sites: HashSet<_> = ctx
- .tv_searcher
+ let top_sites: HashSet<_> = tv_searcher
.search(&query, &collector)
.unwrap()
.into_iter()
.filter_map(|pointer| {
- let doc = ctx.tv_searcher.doc(pointer.address.into()).unwrap();
+ let doc = tv_searcher.doc(pointer.address.into()).unwrap();
let id = doc.get_first(graph_node_id).unwrap().as_u64().unwrap();
if id == u64::MAX {
None
@@ -221,7 +222,7 @@ impl TopicCentrality {
}
pub fn scorer(&self, query: &Query) -> Scorer {
- let terms = query.simple_terms();
+ let terms = query.simple_terms().to_vec();
let mut term_weights = [0.0; NUM_TOPICS];
for term in terms {
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
new file mode 100644
index 00000000..f8be72fc
--- /dev/null
+++ b/ltr/generate_data.py
@@ -0,0 +1,90 @@
+import openai
+import os
+import requests
+import random
+import json
+import pandas as pd
+from tqdm import tqdm
+
+STRACT_ENDPOINT = "localhost:3000"
+NUM_SHUFFLES = 5
+NUM_QUERIES = 100
+TOP_N_RESULTS = 20
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+
+def search(q):
+ url = f"http://{STRACT_ENDPOINT}/beta/api/search"
+ r = requests.post(
+ url, json={"query": q, "offset": 0, "num_results": TOP_N_RESULTS})
+ return r.json()['webpages']
+
+
+def prompt(query, results):
+ results = "\n".join(
+ [json.dumps({k: v for (k, v) in r.items() if k != 'url'}) for r in results])
+ return f"""You are a search engine evaluator. You will be presented with some results in json format from a search engine based on a query. Your task is to score the results from 0 to 1 for the query such that the best result gets a score of 1.0 and the worst gets a score of 0.0. A good search result answers the users query and comes from a trustworthy domain. You should not take the current ordering into account when you score the results. You should only output each score on the same line separated by ',' nothing else.
+
+query: {query}
+
+results:
+{results}
+
+scored results:"""
+
+
+def score(query, results):
+ p = prompt(query, results)
+ response = openai.ChatCompletion.create(
+ model="gpt-3.5-turbo",
+ messages=[{'role': 'user', 'content': p}],
+ temperature=0,
+ )
+ return response.choices[0].message.content
+
+
+queries = requests.get(
+ "https://s3.trystract.com/public/queries_us_big.csv").text.splitlines()
+random.shuffle(queries)
+queries = queries[:NUM_QUERIES]
+
+scores = {'query': [], 'url': [], 'score': []}
+
+for query in tqdm(queries):
+ search_result = search(query)
+
+ res = []
+
+ for r in search_result:
+ if 'Normal' not in r['snippet']:
+ continue
+
+ snip = r['snippet']['Normal']
+ res.append({"domain": r['domain'], "title": r['title'], "url": r['url'],
+ "snippet": snip['text']})
+
+ if len(res) == 0:
+ continue
+
+ for _ in range(NUM_SHUFFLES):
+ random.shuffle(res)
+ try:
+ s = score(query, res)
+ for i, s in enumerate(s.split(',')):
+ s = float(s)
+ url = res[i]['url']
+
+ scores['query'].append(query)
+ scores['url'].append(url)
+ scores['score'].append(s)
+ except:
+ pass
+
+df = pd.DataFrame(scores)
+df = df.groupby(['query', 'url'])
+df = df.mean().sort_values(by='score', ascending=False).reset_index()
+df['rank'] = df.groupby('query')['score'].rank(ascending=False)
+
+print(df.head())
+df.to_csv('ltr_scores.csv', index=False)
diff --git a/optics-lsp/Cargo.lock b/optics-lsp/Cargo.lock
index ba91e960..377d6e15 100644
--- a/optics-lsp/Cargo.lock
+++ b/optics-lsp/Cargo.lock
@@ -66,9 +66,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bumpalo"
-version = "3.11.1"
+version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
[[package]]
name = "cfg-if"
@@ -121,15 +121,15 @@ dependencies = [
[[package]]
name = "either"
-version = "1.8.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "ena"
-version = "0.14.0"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
+checksum = "b2e5d13ca2353ab7d0230988629def93914a8c4015f621f9b13ed2955614731d"
dependencies = [
"log",
]
@@ -212,15 +212,15 @@ dependencies = [
[[package]]
name = "itoa"
-version = "1.0.4"
+version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
[[package]]
name = "js-sys"
-version = "0.3.60"
+version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
dependencies = [
"wasm-bindgen",
]
@@ -259,9 +259,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.137"
+version = "0.2.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
+checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
[[package]]
name = "lock_api"
@@ -332,9 +332,9 @@ checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
[[package]]
name = "once_cell"
-version = "1.16.0"
+version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
[[package]]
name = "optics"
@@ -375,9 +375,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
-version = "0.9.5"
+version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba"
+checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
dependencies = [
"cfg-if",
"libc",
@@ -394,9 +394,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "petgraph"
-version = "0.6.2"
+version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
+checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
dependencies = [
"fixedbitset",
"indexmap",
@@ -425,18 +425,18 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
-version = "1.0.21"
+version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
dependencies = [
"proc-macro2",
]
@@ -463,9 +463,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.7.0"
+version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
dependencies = [
"aho-corasick",
"memchr",
@@ -480,15 +480,15 @@ checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]]
name = "rustversion"
-version = "1.0.9"
+version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
[[package]]
name = "ryu"
-version = "1.0.11"
+version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
[[package]]
name = "scopeguard"
@@ -498,9 +498,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
-version = "1.0.148"
+version = "1.0.154"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc"
+checksum = "8cdd151213925e7f1ab45a9bbfb129316bd00799784b174b7cc7bcd16961c49e"
dependencies = [
"serde_derive",
]
@@ -518,9 +518,9 @@ dependencies = [
[[package]]
name = "serde_derive"
-version = "1.0.148"
+version = "1.0.154"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c"
+checksum = "4fc80d722935453bcafdc2c9a73cd6fac4dc1938f0346035d84bf99fa9e33217"
dependencies = [
"proc-macro2",
"quote",
@@ -529,9 +529,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.89"
+version = "1.0.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
+checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea"
dependencies = [
"itoa",
"ryu",
@@ -540,9 +540,9 @@ dependencies = [
[[package]]
name = "serde_repr"
-version = "0.1.9"
+version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca"
+checksum = "395627de918015623b32e7669714206363a7fc00382bf477e72c1f7533e8eafc"
dependencies = [
"proc-macro2",
"quote",
@@ -563,9 +563,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
[[package]]
name = "string_cache"
-version = "0.8.4"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
+checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
dependencies = [
"new_debug_unreachable",
"once_cell",
@@ -576,9 +576,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "1.0.104"
+version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
@@ -598,18 +598,18 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "1.0.37"
+version = "1.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e"
dependencies = [
"proc-macro2",
"quote",
@@ -636,21 +636,21 @@ dependencies = [
[[package]]
name = "tinyvec_macros"
-version = "0.1.0"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "unicode-bidi"
-version = "0.3.8"
+version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "524b68aca1d05e03fdf03fcdce2c6c94b6daf6d16861ddaa7e4f2b6638a9052c"
[[package]]
name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
[[package]]
name = "unicode-normalization"
@@ -687,9 +687,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@@ -697,9 +697,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
dependencies = [
"bumpalo",
"log",
@@ -712,9 +712,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -722,9 +722,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
dependencies = [
"proc-macro2",
"quote",
@@ -735,9 +735,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.83"
+version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
[[package]]
name = "winapi"
@@ -763,9 +763,18 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
-version = "0.42.0"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
@@ -778,42 +787,42 @@ dependencies = [
[[package]]
name = "windows_aarch64_gnullvm"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
[[package]]
name = "windows_aarch64_msvc"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
[[package]]
name = "windows_i686_gnu"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
[[package]]
name = "windows_i686_msvc"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
[[package]]
name = "windows_x86_64_gnu"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
[[package]]
name = "windows_x86_64_gnullvm"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
[[package]]
name = "windows_x86_64_msvc"
-version = "0.42.0"
+version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
diff --git a/optics/src/ast.rs b/optics/src/ast.rs
index 59c43525..c1d7c514 100644
--- a/optics/src/ast.rs
+++ b/optics/src/ast.rs
@@ -16,7 +16,6 @@
use super::Error;
use super::Result as ModResult;
-use itertools::Itertools;
use lalrpop_util::lalrpop_mod;
use super::lexer;
@@ -29,50 +28,22 @@ pub static PARSER: once_cell::sync::Lazy<parser::BlocksParser> =
#[derive(Debug, PartialEq, Clone)]
pub enum RankingTarget {
Signal(String),
- Field(String),
}
#[derive(Debug, PartialEq, Clone)]
pub struct RankingCoeff {
pub target: RankingTarget,
- pub score: f64,
+ pub value: f64,
}
#[derive(Debug, PartialEq)]
pub struct RawOptic {
pub rules: Vec<RawRule>,
- pub rankings: Vec<RankingPipeline>,
+ pub rankings: Vec<RankingCoeff>,
pub site_preferences: Vec<RawSitePreference>,
pub discard_non_matching: bool,
}
-#[derive(Debug, PartialEq, Clone)]
-pub struct RankingPipeline {
- pub stages: Vec<RankingStage>,
-}
-impl RankingPipeline {
- pub(crate) fn try_merge(&mut self, other_pipeline: RankingPipeline) -> ModResult<()> {
- if self.stages.len() != other_pipeline.stages.len() {
- return Err(Error::RankingStagesMismatch);
- }
-
- for (t, mut o) in self
- .stages
- .iter_mut()
- .zip_eq(other_pipeline.stages.into_iter())
- {
- t.coefficients.append(&mut o.coefficients)
- }
-
- Ok(())
- }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct RankingStage {
- pub coefficients: Vec<RankingCoeff>,
-}
-
impl From<Vec<RawOpticBlock>> for RawOptic {
fn from(blocks: Vec<RawOpticBlock>) -> Self {
let mut rules = Vec::new();
@@ -102,7 +73,7 @@ impl From<Vec<RawOpticBlock>> for RawOptic {
pub enum RawOpticBlock {
Rule(RawRule),
SitePreference(RawSitePreference),
- Ranking(RankingPipeline),
+ Ranking(RankingCoeff),
DiscardNonMatching,
}
@@ -179,13 +150,8 @@ mod tests {
/*
this is a block comment
*/
- RankingPipeline {
- Stage {
- Ranking{Signal("host_centrality"), 3},
- Ranking{Signal("bm25"), 100},
- Ranking{Field("url"), 2},
- }
- };
+ Ranking(Signal("host_centrality"), 3);
+ Ranking(Signal("bm25"), 100);
Rule {
Matches {
@@ -220,24 +186,16 @@ mod tests {
action: None,
},
],
- rankings: vec![RankingPipeline {
- stages: vec![RankingStage {
- coefficients: vec![
- RankingCoeff {
- target: RankingTarget::Signal("host_centrality".to_string()),
- score: 3.0,
- },
- RankingCoeff {
- target: RankingTarget::Signal("bm25".to_string()),
- score: 100.0,
- },
- RankingCoeff {
- target: RankingTarget::Field("url".to_string()),
- score: 2.0,
- },
- ]
- }]
- }],
+ rankings: vec![
+ RankingCoeff {
+ target: RankingTarget::Signal("host_centrality".to_string()),
+ value: 3.0,
+ },
+ RankingCoeff {
+ target: RankingTarget::Signal("bm25".to_string()),
+ value: 100.0,
+ },
+ ],
site_preferences: vec![],
discard_non_matching: false,
}
diff --git a/optics/src/lib.rs b/optics/src/lib.rs
index d7e26a04..5b6b8ff8 100644
--- a/optics/src/lib.rs
+++ b/optics/src/lib.rs
@@ -19,7 +19,7 @@ mod lexer;
use std::convert::TryFrom;
-use ast::RankingPipeline;
+use ast::RankingCoeff;
use logos::Logos;
use serde::{Deserialize, Serialize};
use thiserror::Error;
@@ -93,7 +93,7 @@ impl TryFrom<RawOptic> for Optic {
Ok(Self {
rules,
- pipeline: raw.rankings.into_iter().next(),
+ rankings: raw.rankings,
discard_non_matching: raw.discard_non_matching,
site_rankings: SiteRankings {
liked: liked_sites,
@@ -222,25 +222,21 @@ pub enum Action {
#[derive(Debug, Default, Clone)]
pub struct Optic {
- pub pipeline: Option<RankingPipeline>,
+ pub rankings: Vec<RankingCoeff>,
pub site_rankings: SiteRankings,
pub rules: Vec<Rule>,
pub discard_non_matching: bool,
}
impl Optic {
+ pub fn parse(optic: &str) -> Result<Self> {
+ parse(optic)
+ }
+
pub fn try_merge(mut self, mut other: Self) -> Result<Self> {
self.rules.append(&mut other.rules);
- self.pipeline = match (self.pipeline, other.pipeline) {
- (None, None) => None,
- (None, Some(pipeline)) => Some(pipeline),
- (Some(pipeline), None) => Some(pipeline),
- (Some(mut self_pipeline), Some(other_pipeline)) => {
- self_pipeline.try_merge(other_pipeline)?;
- Some(self_pipeline)
- }
- };
+ self.rankings.append(&mut other.rankings);
self.discard_non_matching |= other.discard_non_matching;
diff --git a/optics/src/parser.lalrpop b/optics/src/parser.lalrpop
index 74a992f2..a9ac04f2 100644
--- a/optics/src/parser.lalrpop
+++ b/optics/src/parser.lalrpop
@@ -18,7 +18,7 @@ pub Blocks: Vec<RawOpticBlock> = <Sep<";", Block>>;
Block: RawOpticBlock = {
<Rule> => RawOpticBlock::Rule(<>),
- <RankingPipeline> => RawOpticBlock::Ranking(<>),
+ <Ranking> => RawOpticBlock::Ranking(<>),
<SitePreference> => RawOpticBlock::SitePreference(<>),
"DiscardNonMatching" => RawOpticBlock::DiscardNonMatching,
}
@@ -49,47 +49,34 @@ RawAction: RawAction= {
}
InnerAction: RawAction= {
- "Boost" "(" <l:@L> <score:Number> <r:@R> ")" =>? {
- match score.parse() {
+ "Boost" "(" <l:@L> <value:Number> <r:@R> ")" =>? {
+ match value.parse() {
Ok(n) => Ok(RawAction::Boost(n)),
Err(_) => Err(ParseError::User {
- error: crate::Error::NumberParse{ token: (l, score.to_string(), r)}
+ error: crate::Error::NumberParse{ token: (l, value.to_string(), r)}
})
}
},
- "Downrank" "(" <l:@L> <score:Number> <r:@R> ")" =>? {
- match score.parse() {
+ "Downrank" "(" <l:@L> <value:Number> <r:@R> ")" =>? {
+ match value.parse() {
Ok(n) => Ok(RawAction::Downrank(n)),
Err(_) => Err(ParseError::User {
- error: crate::Error::NumberParse{ token: (l, score.to_string(), r)}
+ error: crate::Error::NumberParse{ token: (l, value.to_string(), r)}
})
}
},
"Discard" => RawAction::Discard,
}
-RankingPipeline: RankingPipeline = {
- "RankingPipeline" "{" <Sep<",", RankingStage>> "}" => RankingPipeline {
- stages: <>
- }
-}
-
-RankingStage: RankingStage = {
- "Stage" "{" <Sep<",", Ranking>> "}" => RankingStage {
- coefficients: <>
- }
-}
-
-
Ranking: RankingCoeff = {
- "Ranking" "{" <target:RankingTarget> "," <l:@L> <score:Number> <r:@R> "}" =>? {
- match score.parse() {
- Ok(score) => Ok(RankingCoeff {
+ "Ranking" "(" <target:RankingTarget> "," <l:@L> <value:Number> <r:@R> ")" =>? {
+ match value.parse() {
+ Ok(value) => Ok(RankingCoeff {
target,
- score
+ value
}),
Err(_) => Err(ParseError::User {
- error: crate::Error::NumberParse{ token: (l, score.to_string(), r)}
+ error: crate::Error::NumberParse{ token: (l, value.to_string(), r)}
})
}
}
@@ -97,7 +84,6 @@ Ranking: RankingCoeff = {
RankingTarget: RankingTarget = {
"Signal" "(" <StringLiteral> ")" => RankingTarget::Signal(<>.to_string()),
- "Field" "(" <StringLiteral> ")" => RankingTarget::Field(<>.to_string()),
}
SitePreference: RawSitePreference = {
@@ -119,7 +105,6 @@ extern {
"DiscardNonMatching" => Token::DiscardNonMatching,
"Rule" => Token::Rule,
- "RankingPipeline" => Token::RankingPipeline,
"Ranking" => Token::Ranking,
"Stage" => Token::Stage,
"Signal" => Token::Signal,
diff --git a/optics/testcases b/optics/testcases
index 515ef07a..afa95e5d 160000
--- a/optics/testcases
+++ b/optics/testcases
@@ -1 +1 @@
-Subproject commit 515ef07a18403ff0d97d0cfa9ddf3a95d4233f04
+Subproject commit afa95e5d018e7c4b53c48bdc02cf7712de6d21ba