Skip to content

web search done right

fixed page offset bug and prepared for multi stage ranking

Mikkel Denkerf07245834a05
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index b880ab93..ce665b87 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -96,7 +96,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
}
searcher::InitialSearchResult::Bang(bang) => {
req.respond(sonic::Response::Content(
- searcher::InitialPrettifiedSearchResult::Bang(bang),
+ searcher::InitialSearchResult::Bang(bang),
))
.await
.ok();
diff --git a/core/src/frontend/api.rs b/core/src/frontend/api.rs
deleted file mode 100644
index be360bce..00000000
--- a/core/src/frontend/api.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{collections::HashMap, sync::Arc};
-
-use crate::{searcher::SearchQuery, webpage::region::Region};
-
-use super::State;
-use axum::{extract, response::IntoResponse, Extension, Json};
-
-#[allow(clippy::unused_async)]
-pub async fn search(
- extract::Query(params): extract::Query<HashMap<String, String>>,
- Extension(state): Extension<Arc<State>>,
-) -> impl IntoResponse {
- let query = params.get("q").cloned().unwrap_or_default();
-
- let skip_pages = params.get("p").and_then(|p| p.parse().ok());
-
- let selected_region = params.get("gl").and_then(|gl| {
- if let Ok(region) = Region::from_gl(gl) {
- Some(region)
- } else {
- None
- }
- });
-
- match state
- .searcher
- .search_api(&SearchQuery {
- original: query.to_string(),
- selected_region,
- skip_pages,
- ..Default::default()
- })
- .await
- {
- Ok(result) => Json(result),
- Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
- }
-}
diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rs
index b7c9475b..5275680a 100644
--- a/core/src/frontend/mod.rs
+++ b/core/src/frontend/mod.rs
@@ -32,7 +32,6 @@ use axum::{
};
mod about;
-mod api;
mod autosuggest;
mod index;
mod opensearch;
@@ -100,7 +99,6 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router
.route("/settings/optics", get(optics::route))
.route("/settings/sites", get(sites::route))
.route("/privacy-and-happy-lawyers", get(privacy::route))
- .route("/api/beta/search", get(api::search))
.route("/opensearch.xml", get(opensearch::route))
.fallback(get_service(ServeDir::new("frontend/dist/")).handle_error(
|error: std::io::Error| async move {
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index 8a97a0d3..eed51e39 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -25,7 +25,7 @@ use crate::{
thousand_sep_number, CodeOrText, DisplayedSidebar, DisplayedWebpage,
HighlightedSpellCorrection, Snippet,
},
- searcher::{self, PrettifiedSearchResult, SearchQuery},
+ searcher::{self, PrettifiedSearchResult, SearchQuery, NUM_RESULTS_PER_PAGE},
webpage::region::{Region, ALL_REGIONS},
};
@@ -71,7 +71,11 @@ pub async fn route(
) -> impl IntoResponse {
let query = params.get("q").cloned().unwrap_or_default();
- let skip_pages = params.get("p").and_then(|p| p.parse().ok());
+ let skip_pages: usize = params
+ .get("p")
+ .and_then(|p| p.parse().ok())
+ .unwrap_or_default();
+ let offset = skip_pages * NUM_RESULTS_PER_PAGE;
let mut optic = None;
let mut current_optic_url = None;
@@ -116,13 +120,13 @@ pub async fn route(
match state
.searcher
- .search_prettified(&SearchQuery {
+ .search(&SearchQuery {
original: query.clone(),
selected_region,
optic_program: optic,
- skip_pages,
+ offset,
site_rankings,
- custom_num_results: None,
+ num_results: NUM_RESULTS_PER_PAGE,
})
.await
{
@@ -151,10 +155,10 @@ pub async fn route(
})
.collect();
- let current_page = skip_pages.unwrap_or(0) + 1;
+ let current_page = skip_pages + 1;
let mut next_page_params = params.clone();
- next_page_params.insert("p".to_string(), (skip_pages.unwrap_or(0) + 1).to_string());
+ next_page_params.insert("p".to_string(), (skip_pages + 1).to_string());
let next_page_url = uri.path().to_string()
+ "?"
+ serde_urlencoded::to_string(&next_page_params)
@@ -163,8 +167,7 @@ pub async fn route(
let prev_page_url = if current_page > 1 {
let mut prev_page_params = params;
- prev_page_params
- .insert("p".to_string(), (skip_pages.unwrap_or(0) - 1).to_string());
+ prev_page_params.insert("p".to_string(), (skip_pages - 1).to_string());
Some(
uri.path().to_string()
+ "?"
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 3bd9b99c..2ab40cc7 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -16,15 +16,27 @@
use std::cmp::Ordering;
+use itertools::Itertools;
use serde::{Deserialize, Serialize};
use crate::{
inverted_index::WebsitePointer,
schema::{FastField, Field, TextField, ALL_FIELDS, FLOAT_SCALING},
+ searcher::SearchQuery,
};
use super::SignalAggregator;
+pub trait AsRankingWebsite: Clone {
+ fn as_ranking(&self) -> &RankingWebsite;
+}
+
+impl AsRankingWebsite for RankingWebsite {
+ fn as_ranking(&self) -> &RankingWebsite {
+ self
+ }
+}
+
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RankingWebsite {
pub pointer: WebsitePointer,
@@ -45,7 +57,6 @@ impl RankingWebsite {
aggregator: &SignalAggregator,
) -> Self {
let mut res = RankingWebsite {
- pointer,
host_centrality: 0.0,
page_centrality: 0.0,
topic_centrality: 0.0,
@@ -53,7 +64,8 @@ impl RankingWebsite {
query_centrality: 0.0,
title: String::new(),
clean_body: String::new(),
- score: 0.0,
+ score: pointer.score.total,
+ pointer,
};
for value in doc.field_values() {
@@ -69,7 +81,7 @@ impl RankingWebsite {
Field::Fast(FastField::HostNodeID) => {
let node = value.value.as_u64().unwrap();
- res.personal_centrality = dbg!(aggregator.personal_centrality(node));
+ res.personal_centrality = aggregator.personal_centrality(node);
res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();
res.query_centrality = aggregator.query_centrality(node).unwrap_or_default();
}
@@ -87,14 +99,14 @@ impl RankingWebsite {
}
}
-trait Scorer {
- fn score(&self, websites: &mut [RankingWebsite]);
+trait Scorer<T: AsRankingWebsite>: Send + Sync {
+ fn score(&self, websites: &mut [T]);
}
struct ReRanker {}
-impl Scorer for ReRanker {
- fn score(&self, websites: &mut [RankingWebsite]) {
+impl<T: AsRankingWebsite> Scorer<T> for ReRanker {
+ fn score(&self, websites: &mut [T]) {
// TODO: Implement actual scoring
// todo!();
}
@@ -102,26 +114,26 @@ impl Scorer for ReRanker {
struct SignalFocusText {}
-impl Scorer for SignalFocusText {
- fn score(&self, websites: &mut [RankingWebsite]) {
+impl<T: AsRankingWebsite> Scorer<T> for SignalFocusText {
+ fn score(&self, websites: &mut [T]) {
// TODO: Implement actual scoring
// todo!();
}
}
-enum Prev {
+enum Prev<T: AsRankingWebsite> {
Initial,
- Node(Box<RankingStage>),
+ Node(Box<RankingStage<T>>),
}
-struct RankingStage {
- scorer: Box<dyn Scorer>,
- prev: Prev,
+struct RankingStage<T: AsRankingWebsite> {
+ scorer: Box<dyn Scorer<T>>,
+ prev: Prev<T>,
top_n: usize,
- memory: Option<Vec<RankingWebsite>>,
+ memory: Option<(Vec<T>, Vec<T>)>,
}
-impl RankingStage {
+impl<T: AsRankingWebsite> RankingStage<T> {
fn initial_top_n(&self) -> usize {
match &self.prev {
Prev::Initial => self.top_n,
@@ -129,32 +141,90 @@ impl RankingStage {
}
}
- pub fn populate(&mut self, websites: Vec<RankingWebsite>) {
+ pub fn populate(&mut self, websites: Vec<T>) {
match &mut self.prev {
- Prev::Initial => self.memory = Some(websites),
+ Prev::Initial => {
+ let a: Vec<_> = websites.clone().into_iter().take(self.top_n).collect();
+ let b: Vec<_> = websites
+ .into_iter()
+ .skip(self.top_n)
+ .take(self.top_n)
+ .collect();
+
+ self.memory = Some((a, b));
+ }
Prev::Node(n) => n.populate(websites),
}
}
- fn apply(self, top_n: usize) -> Vec<RankingWebsite> {
- let mut chunk = match self.prev {
- Prev::Initial => self.memory.unwrap(),
- Prev::Node(n) => n.apply(self.top_n),
- };
-
- self.scorer.score(&mut chunk);
- chunk.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal));
-
- chunk.into_iter().take(top_n).collect()
+ fn apply(&self, top_n: usize, offset: usize) -> Vec<T> {
+ match &self.prev {
+ Prev::Initial => {
+ let (mut a, mut b) = self.memory.clone().unwrap();
+
+ self.scorer.score(&mut a);
+ a.sort_by(|a, b| {
+ b.as_ranking()
+ .score
+ .partial_cmp(&a.as_ranking().score)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ self.scorer.score(&mut b);
+ b.sort_by(|a, b| {
+ b.as_ranking()
+ .score
+ .partial_cmp(&a.as_ranking().score)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ a.into_iter()
+ .chain(b.into_iter())
+ .skip(offset % self.top_n)
+ .take(top_n)
+ .collect()
+ }
+ Prev::Node(n) => {
+ let k = offset / self.top_n;
+ let (mut a, mut b) = (
+ n.apply(self.top_n, k * self.top_n),
+ n.apply(self.top_n, (k + 1) * self.top_n),
+ );
+
+ self.scorer.score(&mut a);
+ self.scorer.score(&mut b);
+
+ a.sort_by(|a, b| {
+ b.as_ranking()
+ .score
+ .partial_cmp(&a.as_ranking().score)
+ .unwrap_or(Ordering::Equal)
+ });
+ b.sort_by(|a, b| {
+ b.as_ranking()
+ .score
+ .partial_cmp(&a.as_ranking().score)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ a.into_iter()
+ .chain(b.into_iter())
+ .skip(offset % self.top_n)
+ .take(top_n)
+ .collect()
+ }
+ }
}
}
-struct Pipeline {
- last_stage: RankingStage,
+pub struct RankingPipeline<T: AsRankingWebsite> {
+ last_stage: RankingStage<T>,
+ offset: usize,
+ top_n: usize,
}
-impl Default for Pipeline {
- fn default() -> Self {
+impl<T: AsRankingWebsite> RankingPipeline<T> {
+ fn create() -> Self {
let last_stage = RankingStage {
scorer: Box::new(ReRanker {}),
prev: Prev::Node(Box::new(RankingStage {
@@ -166,24 +236,43 @@ impl Default for Pipeline {
memory: None,
top_n: 50,
};
- Self { last_stage }
+
+ Self {
+ last_stage,
+ offset: 0,
+ top_n: 0,
+ }
}
-}
-impl Pipeline {
- pub fn with_offset(&mut self, offset: usize) {
- todo!();
+ pub fn for_query(query: &mut SearchQuery) -> Self {
+ let mut pipeline = Self::create();
+
+ pipeline.offset = query.offset;
+ pipeline.top_n = query.num_results;
+
+ query.num_results = pipeline.collector_top_n();
+ query.offset = pipeline.collector_offset();
+
+ pipeline
}
- pub fn apply(mut self, websites: Vec<RankingWebsite>) -> Vec<RankingWebsite> {
+ pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {
self.last_stage.populate(websites);
- self.last_stage.apply(20)
+ self.last_stage.apply(20, self.offset)
+ }
+
+ pub fn collector_top_n(&self) -> usize {
+ 2 * self.initial_top_n()
}
pub fn initial_top_n(&self) -> usize {
self.last_stage.initial_top_n()
}
+
+ pub fn collector_offset(&self) -> usize {
+ (self.offset / self.initial_top_n()) * self.initial_top_n()
+ }
}
#[cfg(test)]
@@ -222,7 +311,7 @@ mod tests {
query_centrality: 0.0,
title: String::new(),
clean_body: String::new(),
- score: 0.0,
+ score: 1.0 / i as f64,
}
})
.collect()
@@ -230,10 +319,12 @@ mod tests {
#[test]
fn simple() {
- let pipeline = Pipeline::default();
- assert_eq!(pipeline.initial_top_n(), 100);
+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {
+ ..Default::default()
+ });
+ assert_eq!(pipeline.collector_top_n(), 200);
- let sample = sample_websites(pipeline.initial_top_n());
+ let sample = sample_websites(pipeline.collector_top_n());
let res: Vec<_> = pipeline
.apply(sample)
.into_iter()
@@ -242,11 +333,93 @@ mod tests {
let expected: Vec<_> = sample_websites(100)
.into_iter()
- .rev()
.take(20)
.map(|w| w.pointer.address)
.collect();
assert_eq!(res, expected);
}
+
+ #[test]
+ fn offsets() {
+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {
+ offset: 0,
+ ..Default::default()
+ });
+
+ let sample: Vec<_> =
+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
+ .into_iter()
+ .skip(pipeline.collector_offset())
+ .collect();
+ let mut prev: Vec<_> = pipeline.apply(sample);
+
+ for offset in 1..1_000 {
+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {
+ offset,
+ ..Default::default()
+ });
+
+ let sample: Vec<_> =
+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
+ .into_iter()
+ .skip(pipeline.collector_offset())
+ .collect();
+ let res: Vec<_> = pipeline.apply(sample);
+
+ assert_eq!(res.len(), 20, "Every page should have 20 results");
+
+ if let Some(first) = prev.first() {
+ assert!(!res
+ .iter()
+ .any(|r| r.pointer.address.doc_id == first.pointer.address.doc_id));
+ }
+
+ // assert_eq!(
+ // prev.iter()
+ // .map(|p| usize::from(
+ // res.iter()
+ // .any(|r| r.pointer.address.doc_id == p.pointer.address.doc_id)
+ // ))
+ // .sum::<usize>(),
+ // 19,
+ // "Only the top result from previous offset should be removed in current ranking"
+ // );
+
+ prev = res;
+ }
+
+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {
+ offset: 0,
+ ..Default::default()
+ });
+ let sample: Vec<_> =
+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
+ .into_iter()
+ .skip(pipeline.collector_offset())
+ .collect();
+ let mut prev: Vec<_> = pipeline.apply(sample);
+ for p in 1..100 {
+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {
+ offset: p * 20,
+ ..Default::default()
+ });
+
+ let sample: Vec<_> =
+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())
+ .into_iter()
+ .skip(pipeline.collector_offset())
+ .collect();
+ let res: Vec<_> = pipeline.apply(sample).into_iter().collect();
+
+ assert_eq!(res.len(), 20, "Every page should have 20 results");
+
+ assert!(prev
+ .iter()
+ .zip_eq(res.iter())
+ .all(|(p, r)| p.pointer.address.doc_id + 20 == r.pointer.address.doc_id));
+
+ prev = res;
+ }
+ }
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index c00e2965..b991c20e 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -360,10 +360,6 @@ impl SignalAggregator {
let personal_centrality = host_id.map(|host_id| self.personal_centrality(host_id));
- if personal_centrality.unwrap() > 1.0 {
- dbg!(personal_centrality);
- }
-
let score = ALL_SIGNALS
.into_iter()
.map(|signal| {
diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rs
index 2d2f8802..ec6d3a21 100644
--- a/core/src/search_prettifier.rs
+++ b/core/src/search_prettifier.rs
@@ -23,7 +23,8 @@ use tracing::log::debug;
use crate::{
entity_index::{entity::Span, StoredEntity},
- inverted_index::{self, RetrievedWebpage},
+ inverted_index::RetrievedWebpage,
+ ranking::pipeline::RankingWebsite,
searcher::{self, LocalSearcher, Sidebar},
spell::CorrectionTerm,
webpage::{
@@ -76,6 +77,7 @@ pub fn initial(
InitialWebsiteResult {
spell_corrected_query,
websites: result.websites,
+ num_websites: result.num_websites,
sidebar,
}
}
@@ -112,7 +114,8 @@ pub struct HighlightedSpellCorrection {
#[derive(Debug, Serialize, Deserialize)]
pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
- pub websites: inverted_index::InitialSearchResult,
+ pub num_websites: usize,
+ pub websites: Vec<RankingWebsite>,
pub sidebar: Option<DisplayedSidebar>,
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index b87998c5..89471de2 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -17,9 +17,10 @@
use crate::{
collector::{self, BucketCollector},
exponential_backoff::ExponentialBackoff,
- inverted_index::{self, RetrievedWebpage},
+ inverted_index::{self},
+ ranking::pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},
search_prettifier::DisplayedWebpage,
- searcher::{PrettifiedWebsitesResult, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},
+ searcher::PrettifiedWebsitesResult,
};
use std::{net::SocketAddr, time::Instant};
@@ -31,9 +32,7 @@ use thiserror::Error;
use crate::sonic;
-use super::{
- InitialPrettifiedSearchResult, InitialSearchResult, PrettifiedSearchResult, SearchQuery,
-};
+use super::{InitialPrettifiedSearchResult, PrettifiedSearchResult, SearchQuery};
type Result<T> = std::result::Result<T, Error>;
@@ -51,25 +50,7 @@ pub enum Error {
}
impl RemoteSearcher {
- async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResult> {
- for timeout in ExponentialBackoff::from_millis(30).take(5) {
- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
- {
- if let Ok(sonic::Response::Content(body)) =
- connection.send(Request::Search(query.clone())).await
- {
- return Ok(body);
- }
- }
- }
-
- Err(Error::SearchFailed)
- }
-
- async fn search_prettified(
- &self,
- query: &SearchQuery,
- ) -> Result<InitialPrettifiedSearchResult> {
+ async fn search(&self, query: &SearchQuery) -> Result<InitialPrettifiedSearchResult> {
for timeout in ExponentialBackoff::from_millis(30).take(5) {
if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
{
@@ -89,29 +70,6 @@ impl RemoteSearcher {
&self,
pointers: &[inverted_index::WebsitePointer],
original_query: &str,
- ) -> Result<Vec<RetrievedWebpage>> {
- for timeout in ExponentialBackoff::from_millis(30).take(5) {
- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
- {
- if let Ok(sonic::Response::Content(body)) = connection
- .send(Request::RetrieveWebites {
- websites: pointers.to_vec(),
- query: original_query.to_string(),
- })
- .await
- {
- return Ok(body);
- }
- }
- }
-
- Err(Error::SearchFailed)
- }
-
- async fn retrieve_websites_prettified(
- &self,
- pointers: &[inverted_index::WebsitePointer],
- original_query: &str,
) -> Result<Vec<DisplayedWebpage>> {
for timeout in ExponentialBackoff::from_millis(30).take(5) {
if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
@@ -173,53 +131,15 @@ impl Shard {
}
}
- async fn search_prettified(
- &self,
- query: &SearchQuery,
- ) -> Result<InitialPrettifiedSearchResultShard> {
- match self
- .replicas
- .iter()
- .map(|remote| remote.search_prettified(query))
- .collect::<FuturesUnordered<_>>()
- .next()
- .await
- {
- Some(result) => Ok(InitialPrettifiedSearchResultShard {
- local_result: result?,
- shard: self.id.clone(),
- }),
- None => Err(Error::SearchFailed),
- }
- }
-
async fn retrieve_websites(
&self,
pointers: &[inverted_index::WebsitePointer],
original_query: &str,
- ) -> Result<Vec<RetrievedWebpage>> {
- match self
- .replicas
- .iter()
- .map(|remote| remote.retrieve_websites(pointers, original_query))
- .collect::<FuturesUnordered<_>>()
- .next()
- .await
- {
- Some(Ok(websites)) => Ok(websites),
- _ => Err(Error::SearchFailed),
- }
- }
-
- async fn retrieve_websites_prettified(
- &self,
- pointers: &[inverted_index::WebsitePointer],
- original_query: &str,
) -> Result<Vec<DisplayedWebpage>> {
match self
.replicas
.iter()
- .map(|remote| remote.retrieve_websites_prettified(pointers, original_query))
+ .map(|remote| remote.retrieve_websites(pointers, original_query))
.collect::<FuturesUnordered<_>>()
.next()
.await
@@ -231,11 +151,6 @@ impl Shard {
}
struct InitialSearchResultShard {
- local_result: InitialSearchResult,
- shard: ShardId,
-}
-
-struct InitialPrettifiedSearchResultShard {
local_result: InitialPrettifiedSearchResult,
shard: ShardId,
}
@@ -260,21 +175,27 @@ pub struct DistributedSearcher {
#[derive(Clone)]
struct ScoredWebsitePointer {
- local_pointer: inverted_index::WebsitePointer,
+ website: RankingWebsite,
shard: ShardId,
}
+impl AsRankingWebsite for ScoredWebsitePointer {
+ fn as_ranking(&self) -> &RankingWebsite {
+ &self.website
+ }
+}
+
impl collector::Doc for ScoredWebsitePointer {
fn score(&self) -> &f64 {
- &self.local_pointer.score.total
+ &self.website.pointer.score.total
}
fn id(&self) -> &tantivy::DocId {
- &self.local_pointer.address.doc_id
+ &self.website.pointer.address.doc_id
}
fn hashes(&self) -> collector::Hashes {
- self.local_pointer.hashes
+ self.website.pointer.hashes
}
}
@@ -283,139 +204,22 @@ impl DistributedSearcher {
Self { shards }
}
- pub async fn search_api(&self, query: &SearchQuery) -> Result<SearchResult> {
+ pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
let start = Instant::now();
if query.is_empty() {
return Err(Error::EmptyQuery);
}
- // search shards
- let initial_results = self
- .shards
- .iter()
- .map(|shard| shard.search(query))
- .collect::<FuturesUnordered<_>>()
- .collect::<Vec<_>>()
- .await
- .into_iter()
- .filter_map(|result| result.ok())
- .collect::<Vec<_>>();
-
- // check if any result has a bang hit
- if let Some(result) = initial_results
- .iter()
- .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))
- {
- if let InitialSearchResult::Bang(bang) = &result.local_result {
- return Ok(SearchResult::Bang(bang.clone()));
- }
- }
-
- let spell_corrected_query = initial_results.first().and_then(|result| {
- if let InitialSearchResult::Websites(result) = &result.local_result {
- result.spell_corrected_query.clone()
- } else {
- None
- }
- });
-
- let sidebar = initial_results.first().and_then(|result| {
- if let InitialSearchResult::Websites(result) = &result.local_result {
- result.sidebar.clone()
- } else {
- None
- }
- });
-
- let num_docs = initial_results
- .iter()
- .map(|result| {
- if let InitialSearchResult::Websites(result) = &result.local_result {
- result.websites.num_websites
- } else {
- 0
- }
- })
- .sum();
-
- // combine results
- let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);
- let mut collector = BucketCollector::new(top_n);
-
- for result in initial_results {
- if let InitialSearchResult::Websites(local_result) = result.local_result {
- for website in local_result.websites.top_websites {
- let pointer = ScoredWebsitePointer {
- local_pointer: website,
- shard: result.shard.clone(),
- };
-
- collector.insert(pointer);
- }
- }
- }
-
- let top_websites = collector
- .into_sorted_vec(true)
- .into_iter()
- .skip(query.skip_pages.unwrap_or(0))
- .take(NUM_RESULTS_PER_PAGE)
- .collect::<Vec<_>>();
-
- // retrieve webpages
- let mut retrieved_webpages = Vec::new();
-
- for _ in 0..top_websites.len() {
- retrieved_webpages.push(None);
- }
-
- for shard in &self.shards {
- let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites
- .iter()
- .enumerate()
- .filter(|(_, pointer)| pointer.shard == shard.id)
- .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))
- .unzip();
-
- if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {
- for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
- retrieved_webpages[index] = Some(website);
- }
- }
- }
-
- let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();
-
- debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
-
- if retrieved_webpages.is_empty() && !top_websites.is_empty() {
- return Err(Error::SearchFailed);
- }
-
- Ok(SearchResult::Websites(WebsitesResult {
- spell_corrected_query,
- webpages: inverted_index::SearchResult {
- num_docs,
- documents: retrieved_webpages,
- },
- sidebar,
- search_duration_ms: start.elapsed().as_millis(),
- }))
- }
-
- pub async fn search_prettified(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
- let start = Instant::now();
-
- if query.is_empty() {
- return Err(Error::EmptyQuery);
- }
+ let mut search_query = query.clone();
+ let pipeline: RankingPipeline<ScoredWebsitePointer> =
+ RankingPipeline::for_query(&mut search_query);
// search shards
let initial_results = self
.shards
.iter()
- .map(|shard| shard.search_prettified(query))
+ .map(|shard| shard.search(&search_query))
.collect::<FuturesUnordered<_>>()
.collect::<Vec<_>>()
.await
@@ -453,7 +257,7 @@ impl DistributedSearcher {
.iter()
.map(|result| {
if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.websites.num_websites
+ result.num_websites
} else {
0
}
@@ -461,14 +265,14 @@ impl DistributedSearcher {
.sum();
// combine results
- let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);
- let mut collector = BucketCollector::new(top_n);
+ let mut collector =
+ BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());
for result in initial_results {
if let InitialPrettifiedSearchResult::Websites(local_result) = result.local_result {
- for website in local_result.websites.top_websites {
+ for website in local_result.websites {
let pointer = ScoredWebsitePointer {
- local_pointer: website,
+ website,
shard: result.shard.clone(),
};
@@ -480,10 +284,12 @@ impl DistributedSearcher {
let top_websites = collector
.into_sorted_vec(true)
.into_iter()
- .skip(query.skip_pages.unwrap_or(0))
- .take(NUM_RESULTS_PER_PAGE)
+ .skip(pipeline.collector_offset())
+ .take(pipeline.collector_top_n())
.collect::<Vec<_>>();
+ let top_websites = pipeline.apply(top_websites);
+
// retrieve webpages
let mut retrieved_webpages = Vec::new();
@@ -496,13 +302,10 @@ impl DistributedSearcher {
.iter()
.enumerate()
.filter(|(_, pointer)| pointer.shard == shard.id)
- .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))
+ .map(|(idx, pointer)| (idx, pointer.website.pointer.clone()))
.unzip();
- if let Ok(websites) = shard
- .retrieve_websites_prettified(&pointers, &query.original)
- .await
- {
+ if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {
for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
retrieved_webpages[index] = Some(website);
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index fa06b391..0983ad26 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -28,6 +28,7 @@ use crate::index::Index;
use crate::query::Query;
use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::optics::CreateAggregator;
+use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
use crate::ranking::{Ranker, SignalAggregator};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
@@ -35,9 +36,7 @@ use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::{
- InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult, NUM_RESULTS_PER_PAGE,
-};
+use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult};
const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;
@@ -103,9 +102,7 @@ impl LocalSearcher {
&query_aggregator,
)?;
- if let Some(num_results) = query.custom_num_results {
- parsed_query.set_num_results(num_results)
- }
+ parsed_query.set_num_results(query.num_results);
if parsed_query.is_empty() {
Err(Error::EmptyQuery)
@@ -118,7 +115,7 @@ impl LocalSearcher {
&self,
query: &SearchQuery,
de_rank_similar: bool,
- ) -> Result<inverted_index::InitialSearchResult> {
+ ) -> Result<(Vec<RankingWebsite>, usize)> {
let optic = match query.optic_program.as_ref() {
Some(program) => Some(optics::parse(program)?),
None => None,
@@ -181,13 +178,19 @@ impl LocalSearcher {
ranker = ranker
.with_max_docs(10_000_000, self.index.num_segments())
- .with_num_results(parsed_query.num_results());
+ .with_num_results(parsed_query.num_results())
+ .with_offset(query.offset);
- if let Some(skip_pages) = query.skip_pages {
- ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);
- }
+ let res = self
+ .index
+ .search_initial(&parsed_query, ranker.collector())?;
+
+ let ranking_websites = self
+ .index
+ .inverted_index
+ .retrieve_ranking_websites(res.top_websites, &ranker.aggregator())?;
- self.index.search_initial(&parsed_query, ranker.collector())
+ Ok((ranking_websites, res.num_websites))
}
fn spell_correction(&self, query: &SearchQuery) -> Result<Option<Correction>> {
@@ -215,14 +218,14 @@ impl LocalSearcher {
fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
let mut query = query.clone();
query.optic_program = Some(include_str!("stackoverflow.optic").to_string());
- query.custom_num_results = Some(1);
+ query.num_results = 1;
- let mut res = self.search_inverted_index(&query, false)?;
+ let (mut top_websites, num_websites) = self.search_inverted_index(&query, false)?;
- if res.num_websites > 0 && !res.top_websites.is_empty() {
- let top = res.top_websites.remove(0);
- if top.score.total > self.stackoverflow_sidebar_threshold {
- let mut retrieved = self.retrieve_websites(&[top], &query.original)?;
+ if num_websites > 0 && !top_websites.is_empty() {
+ let top = top_websites.remove(0);
+ if top.score > self.stackoverflow_sidebar_threshold {
+ let mut retrieved = self.retrieve_websites(&[top.pointer], &query.original)?;
let retrieved = retrieved.remove(0);
return Ok(Some(Sidebar::StackOverflow {
schema_org: retrieved.schema_org,
@@ -255,13 +258,14 @@ impl LocalSearcher {
return Ok(InitialSearchResult::Bang(bang));
}
- let webpages = self.search_inverted_index(query, de_rank_similar)?;
+ let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;
let correction = self.spell_correction(query)?;
let sidebar = self.sidebar(query)?;
Ok(InitialSearchResult::Websites(InitialWebsiteResult {
spell_corrected_query: correction,
- websites: webpages,
+ websites,
+ num_websites,
sidebar,
}))
}
@@ -288,18 +292,25 @@ impl LocalSearcher {
pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
let start = Instant::now();
- let query_text = query.original.clone();
- let initial_result = self.search_initial(query, true)?;
+ let mut search_query = query.clone();
+
+ let pipeline = RankingPipeline::for_query(&mut search_query);
+
+ let initial_result = self.search_initial(&search_query, true)?;
match initial_result {
InitialSearchResult::Websites(search_result) => {
- let retrieved_sites =
- self.retrieve_websites(&search_result.websites.top_websites, &query_text)?;
+ let top_websites = pipeline.apply(search_result.websites);
+ let pointers: Vec<_> = top_websites
+ .into_iter()
+ .map(|website| website.pointer)
+ .collect();
+ let retrieved_sites = self.retrieve_websites(&pointers, &search_query.original)?;
Ok(SearchResult::Websites(WebsitesResult {
spell_corrected_query: search_result.spell_corrected_query,
webpages: inverted_index::SearchResult {
- num_docs: search_result.websites.num_websites,
+ num_docs: search_result.num_websites,
documents: retrieved_sites,
},
sidebar: search_result.sidebar,
@@ -337,7 +348,8 @@ impl LocalSearcher {
#[derive(Debug, Serialize, Deserialize)]
pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<Correction>,
- pub websites: inverted_index::InitialSearchResult,
+ pub num_websites: usize,
+ pub websites: Vec<RankingWebsite>,
pub sidebar: Option<Sidebar>,
}
@@ -354,13 +366,16 @@ impl SearchResult {
#[cfg(test)]
mod tests {
- use crate::webpage::{Html, Webpage};
+ use crate::{
+ searcher::NUM_RESULTS_PER_PAGE,
+ webpage::{Html, Webpage},
+ };
use super::*;
#[test]
fn offset_page() {
- const NUM_PAGES: usize = 10;
+ const NUM_PAGES: usize = 50;
const NUM_WEBSITES: usize = NUM_PAGES * NUM_RESULTS_PER_PAGE;
let mut index = Index::temporary().expect("Unable to open index");
@@ -403,7 +418,7 @@ mod tests {
let urls: Vec<_> = searcher
.search(&SearchQuery {
original: "test".to_string(),
- skip_pages: Some(p),
+ offset: p * NUM_RESULTS_PER_PAGE,
..Default::default()
})
.unwrap()
@@ -417,6 +432,8 @@ mod tests {
assert!(!urls.is_empty());
+ dbg!(&urls);
+
for (i, url) in urls.into_iter().enumerate() {
assert_eq!(
url,
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 5f4ab6fd..ce96059d 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -83,14 +83,27 @@ pub enum InitialPrettifiedSearchResult {
Bang(BangHit),
}
-#[derive(Debug, Serialize, Deserialize, Clone, Default)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchQuery {
pub original: String,
+ pub offset: usize,
+ pub num_results: usize,
pub selected_region: Option<Region>,
pub optic_program: Option<String>,
- pub skip_pages: Option<usize>,
pub site_rankings: Option<SiteRankings>,
- pub custom_num_results: Option<usize>,
+}
+
+impl Default for SearchQuery {
+ fn default() -> Self {
+ Self {
+ original: Default::default(),
+ offset: Default::default(),
+ num_results: NUM_RESULTS_PER_PAGE,
+ selected_region: Default::default(),
+ optic_program: Default::default(),
+ site_rankings: Default::default(),
+ }
+ }
}
impl SearchQuery {