Skip to content

web search done right

[api-searcher] refactor functionality into helper structs to keep the searcher a bit more clean

Mikkel Denker9dce1a3931c7
diff --git a/core/src/bangs.rs b/core/src/bangs.rs
index 7ac34e7d..70b3b57e 100644
--- a/core/src/bangs.rs
+++ b/core/src/bangs.rs
@@ -28,7 +28,7 @@ use utoipa::ToSchema;
use crate::query::parser::Term;
-pub const BANG_PREFIX: char = '!';
+pub const BANG_PREFIXES: [char; 2] = ['!', '!'];
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 0f1f2840..b2b2f43f 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -147,6 +147,29 @@ impl IndexingWorker {
.into_iter()
.map(|edge| edge.label)
.filter(|label| !label.is_empty())
+ .filter(|label| {
+ let label = label.to_lowercase();
+ let stopwords = [
+ "click",
+ "click here",
+ "here",
+ "link",
+ "website",
+ "webpage",
+ "page",
+ "site",
+ "url",
+ "web",
+ "visit",
+ "more",
+ "info",
+ "information",
+ "read",
+ "read more",
+ ];
+
+ !stopwords.contains(&label.as_str())
+ })
.collect()
})
.unwrap_or_default();
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 444f0235..56c6abc2 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -589,6 +589,7 @@ pub struct RetrievedWebpage {
pub region: Region,
pub likely_has_ads: bool,
pub likely_has_paywall: bool,
+ pub recipe_first_ingredient_tag_id: Option<String>,
}
impl RetrievedWebpage {
pub fn description(&self) -> Option<&String> {
@@ -688,6 +689,18 @@ impl From<TantivyDocument> for RetrievedWebpage {
webpage.likely_has_paywall =
value.value().as_value().as_u64().unwrap_or_default() != 0;
}
+ Field::Text(TextField::RecipeFirstIngredientTagId) => {
+ let tag_id = value
+ .value()
+ .as_value()
+ .as_str()
+ .expect("Recipe first ingredient tag id field should be stored as text")
+ .to_string();
+
+ if !tag_id.is_empty() {
+ webpage.recipe_first_ingredient_tag_id = Some(tag_id);
+ }
+ }
_ => {}
}
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index f621040c..c875889a 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -1084,7 +1084,9 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "avengers endgame".to_string(),
- optic: Some(Optic::parse(include_str!("../searcher/discussions.optic")).unwrap()),
+ optic: Some(
+ Optic::parse(include_str!("../searcher/api/discussions.optic")).unwrap(),
+ ),
..Default::default()
})
.unwrap()
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index c9a478f5..18b70777 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -20,7 +20,7 @@ use tantivy::{
};
use crate::{
- bangs::BANG_PREFIX,
+ bangs::BANG_PREFIXES,
floor_char_boundary,
schema::{Field, TextField, ALL_FIELDS},
};
@@ -87,7 +87,7 @@ impl ToString for Term {
Term::Title(title) => "intitle:".to_string() + title.as_str(),
Term::Body(body) => "inbody:".to_string() + body.as_str(),
Term::Url(url) => "inurl:".to_string() + url.as_str(),
- Term::PossibleBang(bang) => BANG_PREFIX.to_string() + bang.as_str(),
+ Term::PossibleBang(bang) => BANG_PREFIXES[0].to_string() + bang.as_str(),
}
}
}
@@ -233,7 +233,7 @@ impl Term {
Term::PossibleBang(text) => {
let mut term = String::new();
- term.push(BANG_PREFIX);
+ term.push(BANG_PREFIXES[0]);
term.push_str(text);
simple_into_tantivy(&term.into(), &[], fields)
@@ -372,9 +372,13 @@ fn parse_term(term: &str) -> Box<Term> {
} else {
Box::new(Term::Simple(term.to_string().into()))
}
- } else if let Some(bang) = term.strip_prefix(BANG_PREFIX) {
- Box::new(Term::PossibleBang(bang.to_string()))
} else {
+ for bang_prefix in BANG_PREFIXES {
+ if let Some(bang) = term.strip_prefix(bang_prefix) {
+ return Box::new(Term::PossibleBang(bang.to_string()));
+ }
+ }
+
Box::new(Term::Simple(term.to_string().into()))
}
}
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 3970de21..06d55e93 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -22,7 +22,7 @@ use crate::{
collector::{self, BucketCollector},
config::CollectorConfig,
enum_map::EnumMap,
- inverted_index::WebsitePointer,
+ inverted_index::{RetrievedWebpage, WebsitePointer},
searcher::SearchQuery,
Result,
};
@@ -72,12 +72,44 @@ impl lambdamart::AsValue for SignalScore {
}
}
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct RetrievedWebpageRanking {
+ retrieved_webpage: RetrievedWebpage,
+ ranking: RankingWebsite,
+}
+
+impl RetrievedWebpageRanking {
+ pub fn new(retrieved_webpage: RetrievedWebpage, ranking: RankingWebsite) -> Self {
+ let mut ranking = ranking;
+ ranking.snippet = Some(retrieved_webpage.snippet.unhighlighted_string());
+
+ Self {
+ retrieved_webpage,
+ ranking,
+ }
+ }
+
+ pub fn into_retrieved_webpage(self) -> RetrievedWebpage {
+ self.retrieved_webpage
+ }
+}
+
+impl AsRankingWebsite for RetrievedWebpageRanking {
+ fn as_ranking(&self) -> &RankingWebsite {
+ &self.ranking
+ }
+
+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
+ &mut self.ranking
+ }
+}
+
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RankingWebsite {
pub pointer: WebsitePointer,
pub signals: EnumMap<Signal, SignalScore>,
pub title: Option<String>,
- pub clean_body: Option<String>,
+ pub snippet: Option<String>,
pub optic_boost: Option<f64>,
pub score: f64,
}
@@ -87,9 +119,9 @@ impl RankingWebsite {
let mut res = RankingWebsite {
signals: EnumMap::new(),
title: None,
- clean_body: None,
score: pointer.score.total,
optic_boost: None,
+ snippet: None,
pointer: pointer.clone(),
};
@@ -129,36 +161,51 @@ impl<M: CrossEncoder> ReRanker<M> {
}
fn crossencoder_score_websites<T: AsRankingWebsite>(&self, websites: &mut [T]) {
- let mut bodies = Vec::with_capacity(websites.len());
+ let mut snippets = Vec::with_capacity(websites.len());
+ let mut titles = Vec::with_capacity(websites.len());
for website in websites.iter_mut() {
- let website = website.as_mut_ranking();
- let title = website.title.clone().unwrap_or_default();
- let body = website.clean_body.clone().unwrap_or_default();
- let text = title + ". " + body.as_str();
- bodies.push(text);
+ titles.push(website.as_mut_ranking().title.clone().unwrap_or_default());
+ snippets.push(website.as_mut_ranking().snippet.clone().unwrap_or_default());
}
let query = &self.query.as_ref().unwrap().query;
- let scores = self.crossencoder.run(query, &bodies);
+ let snippet_scores = self.crossencoder.run(query, &snippets);
+ let title_scores = self.crossencoder.run(query, &titles);
- for (website, score) in websites.iter_mut().zip(scores) {
+ for ((website, snippet), title) in websites.iter_mut().zip(snippet_scores).zip(title_scores)
+ {
let website = website.as_mut_ranking();
website.signals.insert(
- Signal::CrossEncoder,
+ Signal::CrossEncoderSnippet,
+ SignalScore {
+ coefficient: self.crossencoder_snippet_coeff(),
+ value: snippet,
+ },
+ );
+
+ website.signals.insert(
+ Signal::CrossEncoderTitle,
SignalScore {
- coefficient: self.crossencoder_coeff(),
- value: score,
+ coefficient: self.crossencoder_title_coeff(),
+ value: title,
},
);
}
}
- fn crossencoder_coeff(&self) -> f64 {
+ fn crossencoder_snippet_coeff(&self) -> f64 {
self.signal_coefficients
.as_ref()
- .and_then(|coeffs| coeffs.get(&Signal::CrossEncoder))
- .unwrap_or(Signal::CrossEncoder.default_coefficient())
+ .and_then(|coeffs| coeffs.get(&Signal::CrossEncoderSnippet))
+ .unwrap_or(Signal::CrossEncoderSnippet.default_coefficient())
+ }
+
+ fn crossencoder_title_coeff(&self) -> f64 {
+ self.signal_coefficients
+ .as_ref()
+ .and_then(|coeffs| coeffs.get(&Signal::CrossEncoderSnippet))
+ .unwrap_or(Signal::CrossEncoderSnippet.default_coefficient())
}
}
@@ -293,10 +340,11 @@ pub struct RankingPipeline<T: AsRankingWebsite> {
}
impl<T: AsRankingWebsite> RankingPipeline<T> {
- fn create_reranking<M: CrossEncoder + 'static>(
+ fn create_second_stage<M: CrossEncoder + 'static>(
crossencoder: Option<Arc<M>>,
lambda: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
+ top_n_considered: usize,
) -> Result<Self> {
let scorer = match crossencoder {
Some(cross_encoder) => {
@@ -307,7 +355,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
let stage = RankingStage {
scorer,
- stage_top_n: 20,
+ stage_top_n: top_n_considered,
derank_similar: true,
};
@@ -319,25 +367,31 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
})
}
- pub fn reranking_for_query<M: CrossEncoder + 'static>(
+ pub fn reranker<M: CrossEncoder + 'static>(
query: &mut SearchQuery,
crossencoder: Option<Arc<M>>,
lambda: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
+ top_n_considered: usize,
) -> Result<Self> {
- let mut pipeline = Self::create_reranking(crossencoder, lambda, collector_config)?;
+ let mut pipeline =
+ Self::create_second_stage(crossencoder, lambda, collector_config, top_n_considered)?;
pipeline.set_query_info(query);
Ok(pipeline)
}
- fn create_ltr(model: Option<Arc<LambdaMART>>, collector_config: CollectorConfig) -> Self {
+ fn create_first_stage(
+ model: Option<Arc<LambdaMART>>,
+ collector_config: CollectorConfig,
+ stage_top_n: usize,
+ ) -> Self {
let last_stage = RankingStage {
scorer: Box::new(Initial {
model,
signal_coefficients: None,
}),
- stage_top_n: 100,
+ stage_top_n,
derank_similar: true,
};
@@ -349,12 +403,13 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
}
- pub fn initial_for_query(
+ pub fn first_stage(
query: &mut SearchQuery,
model: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
+ top_n_considered: usize,
) -> Self {
- let mut pipeline = Self::create_ltr(model, collector_config);
+ let mut pipeline = Self::create_first_stage(model, collector_config, top_n_considered);
pipeline.set_query_info(query);
pipeline
@@ -436,7 +491,7 @@ mod tests {
signals,
optic_boost: None,
title: None,
- clean_body: None,
+ snippet: None,
score: 1.0 / i as f64,
}
})
@@ -445,13 +500,14 @@ mod tests {
#[test]
fn simple() {
- let pipeline = RankingPipeline::reranking_for_query(
+ let pipeline = RankingPipeline::reranker(
&mut SearchQuery {
..Default::default()
},
Some(Arc::new(DummyCrossEncoder {})),
None,
CollectorConfig::default(),
+ 20,
)
.unwrap();
assert_eq!(pipeline.collector_top_n(), 20);
@@ -475,7 +531,7 @@ mod tests {
#[test]
fn top_n() {
let num_results = 100;
- let pipeline = RankingPipeline::reranking_for_query(
+ let pipeline = RankingPipeline::reranker(
&mut SearchQuery {
num_results,
..Default::default()
@@ -483,6 +539,7 @@ mod tests {
Some(Arc::new(DummyCrossEncoder {})),
None,
CollectorConfig::default(),
+ num_results,
)
.unwrap();
@@ -508,7 +565,7 @@ mod tests {
#[test]
fn offsets() {
let num_results = 20;
- let pipeline = RankingPipeline::reranking_for_query(
+ let pipeline = RankingPipeline::reranker(
&mut SearchQuery {
page: 0,
num_results,
@@ -517,13 +574,14 @@ mod tests {
Some(Arc::new(DummyCrossEncoder {})),
None,
CollectorConfig::default(),
+ num_results,
)
.unwrap();
let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
let mut prev: Vec<_> = pipeline.apply(sample);
for p in 1..1_000 {
- let pipeline = RankingPipeline::reranking_for_query(
+ let pipeline = RankingPipeline::reranker(
&mut SearchQuery {
page: p,
..Default::default()
@@ -531,6 +589,7 @@ mod tests {
Some(Arc::new(DummyCrossEncoder {})),
None,
CollectorConfig::default(),
+ num_results,
)
.unwrap();
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index f6c556df..c769a502 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -98,8 +98,10 @@ pub enum Signal {
Bm25TitleIfHomepage,
#[serde(rename = "bm25_backlink_text")]
Bm25BacklinkText,
- #[serde(rename = "cross_encoder")]
- CrossEncoder,
+ #[serde(rename = "cross_encoder_snippet")]
+ CrossEncoderSnippet,
+ #[serde(rename = "cross_encoder_title")]
+ CrossEncoderTitle,
#[serde(rename = "host_centrality")]
HostCentrality,
#[serde(rename = "page_centrality")]
@@ -134,7 +136,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 34] = [
+pub const ALL_SIGNALS: [Signal; 35] = [
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
Signal::Bm25TitleTrigrams,
@@ -155,7 +157,8 @@ pub const ALL_SIGNALS: [Signal; 34] = [
Signal::Bm25DomainIfHomepageNoTokenizer,
Signal::Bm25TitleIfHomepage,
Signal::Bm25BacklinkText,
- Signal::CrossEncoder,
+ Signal::CrossEncoderSnippet,
+ Signal::CrossEncoderTitle,
Signal::HostCentrality,
Signal::PageCentrality,
Signal::IsHomepage,
@@ -278,7 +281,8 @@ impl Signal {
Signal::Bm25DomainIfHomepageNoTokenizer => 0.0036,
Signal::Bm25TitleIfHomepage => 0.00022,
Signal::Bm25BacklinkText => 0.003,
- Signal::CrossEncoder => 0.17,
+ Signal::CrossEncoderSnippet => 0.17,
+ Signal::CrossEncoderTitle => 0.17,
Signal::HostCentrality => 0.5,
Signal::PageCentrality => 0.25,
Signal::QueryCentrality => 0.0,
@@ -453,7 +457,8 @@ impl Signal {
.unwrap_or(0.0)
}),
- Signal::CrossEncoder => None, // this is calculated in a later step
+ Signal::CrossEncoderSnippet => None, // this is calculated in a later step
+ Signal::CrossEncoderTitle => None, // this is calculated in a later step
Signal::LambdaMART => None,
};
@@ -553,7 +558,8 @@ impl Signal {
| Signal::Bm25DomainIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
| Signal::Bm25BacklinkText
- | Signal::CrossEncoder
+ | Signal::CrossEncoderSnippet
+ | Signal::CrossEncoderTitle
| Signal::InboundSimilarity
| Signal::LambdaMART
| Signal::QueryCentrality => {
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 5db77aa3..63b87caa 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -60,6 +60,7 @@ pub enum TextField {
/// can either be NSFW or SFW (see safety classifier)
SafetyClassification,
InsertionTimestamp,
+ RecipeFirstIngredientTagId,
}
impl From<TextField> for usize {
@@ -111,6 +112,7 @@ impl TextField {
TextField::MicroformatTags => Tokenizer::default(),
TextField::SafetyClassification => Tokenizer::Identity(Identity {}),
TextField::InsertionTimestamp => Tokenizer::Identity(Identity {}),
+ TextField::RecipeFirstIngredientTagId => Tokenizer::Identity(Identity {}),
}
}
@@ -154,6 +156,7 @@ impl TextField {
TextField::MicroformatTags => true,
TextField::SafetyClassification => false,
TextField::InsertionTimestamp => false,
+ TextField::RecipeFirstIngredientTagId => false,
}
}
@@ -189,6 +192,7 @@ impl TextField {
TextField::MicroformatTags => "microformat_tags",
TextField::SafetyClassification => "safety_classification",
TextField::InsertionTimestamp => "insertion_timestamp",
+ TextField::RecipeFirstIngredientTagId => "recipe_first_ingredient_tag_id",
}
}
}
@@ -459,6 +463,9 @@ impl Field {
Field::Text(TextField::SafetyClassification) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::RecipeFirstIngredientTagId) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
Field::Text(TextField::InsertionTimestamp) => {
IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())
}
@@ -634,6 +641,7 @@ impl Field {
| Field::Text(TextField::SiteWithout) // will match url
| Field::Text(TextField::Domain) // will match url
| Field::Text(TextField::InsertionTimestamp)
+ | Field::Text(TextField::RecipeFirstIngredientTagId)
) && !self.is_fast()
}
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
deleted file mode 100644
index 85135ba2..00000000
--- a/core/src/searcher/api.rs
+++ /dev/null
@@ -1,733 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::cmp::Ordering;
-use std::collections::HashMap;
-use std::ops::Range;
-use std::sync::Arc;
-use std::time::Instant;
-
-use itertools::{intersperse, Itertools};
-use optics::Optic;
-use url::Url;
-
-use crate::bangs::{Bang, BangHit};
-use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};
-use crate::image_store::Image;
-use crate::inverted_index::RetrievedWebpage;
-#[cfg(not(feature = "libtorch"))]
-use crate::ranking::models::cross_encoder::DummyCrossEncoder;
-use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite};
-use crate::ranking::ALL_SIGNALS;
-use crate::search_prettifier::{
- create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,
- DisplayedWebpage, HighlightedSpellCorrection,
-};
-use crate::web_spell::SpellChecker;
-use crate::widgets::{Widget, Widgets};
-use crate::{
- bangs::Bangs,
- collector::BucketCollector,
- distributed::cluster::Cluster,
- ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},
-};
-use crate::{ceil_char_boundary, floor_char_boundary, query, Result};
-#[cfg(feature = "libtorch")]
-use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};
-
-use super::live::LiveSearcher;
-use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};
-
-#[derive(Clone)]
-enum ScoredWebsitePointer {
- Normal(distributed::ScoredWebsitePointer),
- Live(live::ScoredWebsitePointer),
-}
-
-impl AsRankingWebsite for ScoredWebsitePointer {
- fn as_ranking(&self) -> &RankingWebsite {
- match self {
- ScoredWebsitePointer::Normal(p) => &p.website,
- ScoredWebsitePointer::Live(p) => &p.website,
- }
- }
-
- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
- match self {
- ScoredWebsitePointer::Normal(p) => &mut p.website,
- ScoredWebsitePointer::Live(p) => &mut p.website,
- }
- }
-}
-
-#[cfg(feature = "libtorch")]
-pub struct ApiSearcher {
- distributed_searcher: DistributedSearcher,
- live_searcher: LiveSearcher,
- cross_encoder: Option<Arc<CrossEncoderModel>>,
- lambda_model: Option<Arc<LambdaMART>>,
- qa_model: Option<Arc<QaModel>>,
- bangs: Bangs,
- collector_config: CollectorConfig,
- thresholds: ApiThresholds,
- widgets: Widgets,
- spell_checker: Option<SpellChecker>,
-}
-
-#[cfg(not(feature = "libtorch"))]
-pub struct ApiSearcher {
- distributed_searcher: DistributedSearcher,
- live_searcher: LiveSearcher,
- lambda_model: Option<Arc<LambdaMART>>,
- bangs: Bangs,
- collector_config: CollectorConfig,
- thresholds: ApiThresholds,
- widgets: Widgets,
- spell_checker: Option<SpellChecker>,
-}
-
-impl ApiSearcher {
- #[cfg(feature = "libtorch")]
- pub fn new(
- cluster: Arc<Cluster>,
- cross_encoder: Option<CrossEncoderModel>,
- lambda_model: Option<LambdaMART>,
- qa_model: Option<QaModel>,
- bangs: Bangs,
- config: ApiConfig,
- ) -> Self {
- Self {
- distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),
- live_searcher: LiveSearcher::new(Arc::clone(&cluster)),
- cross_encoder: cross_encoder.map(Arc::new),
- lambda_model: lambda_model.map(Arc::new),
- qa_model: qa_model.map(Arc::new),
- bangs,
- collector_config: config.collector,
- thresholds: config.thresholds,
- widgets: Widgets::new(config.widgets).unwrap(),
- spell_checker: config
- .spell_checker_path
- .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
- }
- }
-
- #[cfg(not(feature = "libtorch"))]
- pub fn new(
- cluster: Arc<Cluster>,
- lambda_model: Option<LambdaMART>,
- bangs: Bangs,
- config: ApiConfig,
- ) -> Self {
- Self {
- distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),
- live_searcher: LiveSearcher::new(Arc::clone(&cluster)),
- lambda_model: lambda_model.map(Arc::new),
- bangs,
- collector_config: config.collector,
- thresholds: config.thresholds,
- widgets: Widgets::new(config.widgets).unwrap(),
- spell_checker: config
- .spell_checker_path
- .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
- }
- }
-
- fn combine_results(
- &self,
- initial_results: Vec<distributed::InitialSearchResultShard>,
- live_results: Vec<live::InitialSearchResultSplit>,
- pipeline: RankingPipeline<ScoredWebsitePointer>,
- ) -> (Vec<ScoredWebsitePointer>, bool) {
- let mut collector =
- BucketCollector::new(pipeline.collector_top_n(), self.collector_config.clone());
-
- let mut num_sites: usize = 0;
- for result in initial_results {
- for website in result.local_result.websites {
- num_sites += 1;
- let pointer = distributed::ScoredWebsitePointer {
- website,
- shard: result.shard,
- };
-
- let pointer = ScoredWebsitePointer::Normal(pointer);
-
- collector.insert(pointer);
- }
- }
-
- for result in live_results {
- for website in result.local_result.websites {
- num_sites += 1;
- let pointer = live::ScoredWebsitePointer {
- website,
- split_id: result.split_id.clone(),
- };
-
- let pointer = ScoredWebsitePointer::Live(pointer);
-
- collector.insert(pointer);
- }
- }
-
- let top_websites = collector
- .into_sorted_vec(true)
- .into_iter()
- .take(pipeline.collector_top_n())
- .collect::<Vec<_>>();
-
- let offset = pipeline.offset();
-
- let res = pipeline.apply(top_websites);
- let has_more = num_sites.saturating_sub(offset) > res.len();
-
- (res, has_more)
- }
-
- async fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<DisplayedSidebar>> {
- let query = SearchQuery {
- query: query.query.clone(),
- num_results: 1,
- optic: Some(Optic::parse(include_str!("stackoverflow.optic")).unwrap()),
- ..Default::default()
- };
-
- let mut results: Vec<_> = self
- .distributed_searcher
- .search_initial(&query)
- .await
- .into_iter()
- .filter_map(|result| {
- result
- .local_result
- .websites
- .first()
- .cloned()
- .map(|website| (result.shard, website))
- })
- .collect();
-
- results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));
-
- if let Some((shard, website)) = results.pop() {
- tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");
- if website.score > self.thresholds.stackoverflow {
- let scored_websites =
- vec![(0, distributed::ScoredWebsitePointer { website, shard })];
- let mut retrieved = self
- .distributed_searcher
- .retrieve_webpages(&scored_websites, &query.query)
- .await;
-
- if let Some((_, res)) = retrieved.pop() {
- return Ok(Some(create_stackoverflow_sidebar(
- res.schema_org,
- Url::parse(&res.url).unwrap(),
- )?));
- }
- }
- }
-
- Ok(None)
- }
-
- fn sidebar(
- &self,
- initial_results: &[distributed::InitialSearchResultShard],
- stackoverflow: Option<DisplayedSidebar>,
- ) -> Option<DisplayedSidebar> {
- let entity = initial_results
- .iter()
- .filter_map(|res| res.local_result.entity_sidebar.clone())
- .map(DisplayedEntity::from)
- .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)
- .max_by(|a, b| {
- a.match_score
- .partial_cmp(&b.match_score)
- .unwrap_or(Ordering::Equal)
- });
-
- match entity {
- Some(entity) => Some(DisplayedSidebar::Entity(entity)),
- None => stackoverflow,
- }
- }
-
- async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {
- let parsed_terms = query::parser::parse(&query.query);
-
- if parsed_terms.iter().any(|term| match term.as_ref() {
- query::parser::Term::PossibleBang(t) => t.is_empty(),
- _ => false,
- }) {
- let q: String = intersperse(
- parsed_terms
- .iter()
- .filter(|term| !matches!(term.as_ref(), query::parser::Term::PossibleBang(_)))
- .map(|term| term.to_string()),
- " ".to_string(),
- )
- .collect();
-
- let mut query = query.clone();
- query.query = q;
-
- let res = self.search_websites(&query).await?;
-
- return Ok(res.webpages.first().map(|webpage| BangHit {
- bang: Bang {
- category: None,
- sub_category: None,
- domain: None,
- ranking: None,
- site: None,
- tag: String::new(),
- url: webpage.url.clone(),
- },
- redirect_to: Url::parse(&webpage.url).unwrap().into(),
- }));
- }
-
- Ok(self.bangs.get(&parsed_terms))
- }
-
- async fn discussions_widget(
- &self,
- query: &SearchQuery,
- ) -> Result<Option<Vec<DisplayedWebpage>>> {
- if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {
- return Ok(None);
- }
-
- const NUM_RESULTS: usize = 10;
-
- let mut query = SearchQuery {
- query: query.query.clone(),
- num_results: NUM_RESULTS,
- optic: Some(Optic::parse(include_str!("discussions.optic")).unwrap()),
- site_rankings: query.site_rankings.clone(),
- return_ranking_signals: query.return_ranking_signals,
- ..Default::default()
- };
-
- #[cfg(feature = "libtorch")]
- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::reranking_for_query(
- &mut query,
- self.cross_encoder.as_ref().map(Arc::clone),
- self.lambda_model.clone(),
- self.collector_config.clone(),
- )?;
-
- #[cfg(not(feature = "libtorch"))]
- let pipeline: RankingPipeline<ScoredWebsitePointer> =
- RankingPipeline::reranking_for_query::<DummyCrossEncoder>(
- &mut query,
- None,
- self.lambda_model.clone(),
- self.collector_config.clone(),
- )?;
-
- let initial_results = self.distributed_searcher.search_initial(&query).await;
-
- if initial_results.is_empty() {
- return Ok(None);
- }
-
- let num_results: usize = initial_results
- .iter()
- .map(|res| res.local_result.websites.len())
- .sum();
-
- if num_results < NUM_RESULTS / 2 {
- return Ok(None);
- }
-
- let (top_websites, _) = self.combine_results(initial_results, vec![], pipeline);
-
- let scores: Vec<_> = top_websites
- .iter()
- .map(|pointer| pointer.as_ranking().score)
- .collect();
-
- let median = if scores.len() % 2 == 0 {
- (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0
- } else {
- scores[scores.len() / 2]
- };
-
- if median < self.thresholds.discussions_widget {
- return Ok(None);
- }
-
- let mut result: Vec<DisplayedWebpage> = self
- .distributed_searcher
- .retrieve_webpages(
- &top_websites
- .clone()
- .into_iter()
- .filter_map(|pointer| {
- if let ScoredWebsitePointer::Normal(p) = pointer {
- Some(p)
- } else {
- None
- }
- })
- .enumerate()
- .collect_vec(),
- &query.query,
- )
- .await
- .into_iter()
- .map(|(_, webpage)| webpage)
- .map(DisplayedWebpage::from)
- .collect();
-
- if query.return_ranking_signals {
- self.add_ranking_signals(&mut result, &top_websites);
- }
-
- Ok(Some(result))
- }
-
- fn add_ranking_signals(
- &self,
- websites: &mut [DisplayedWebpage],
- pointers: &[ScoredWebsitePointer],
- ) {
- for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {
- let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
-
- for signal in ALL_SIGNALS {
- if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {
- signals.insert(signal, *signal_value);
- }
- }
-
- website.ranking_signals = Some(signals);
- }
- }
-
- async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {
- let start = Instant::now();
-
- if query.is_empty() {
- return Err(distributed::Error::EmptyQuery.into());
- }
-
- let mut search_query = query.clone();
- #[cfg(feature = "libtorch")]
- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::reranking_for_query(
- &mut search_query,
- self.cross_encoder.as_ref().map(Arc::clone),
- self.lambda_model.clone(),
- self.collector_config.clone(),
- )?;
-
- #[cfg(not(feature = "libtorch"))]
- let pipeline: RankingPipeline<ScoredWebsitePointer> =
- RankingPipeline::reranking_for_query::<DummyCrossEncoder>(
- &mut search_query,
- None,
- self.lambda_model.clone(),
- self.collector_config.clone(),
- )?;
-
- let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(
- self.distributed_searcher.search_initial(&search_query),
- self.live_searcher.search_initial(&search_query),
- self.widget(query),
- self.discussions_widget(query),
- self.stackoverflow_sidebar(query),
- );
-
- let discussions = discussions?;
- let stackoverflow = stackoverflow?;
- let sidebar = self.sidebar(&initial_results, stackoverflow);
-
- let spell_corrected_query = if widget.is_none() {
- self.spell_checker
- .as_ref()
- .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))
- .map(HighlightedSpellCorrection::from)
- } else {
- None
- };
-
- let num_docs = initial_results
- .iter()
- .map(|result| result.local_result.num_websites)
- .sum();
-
- let (top_websites, has_more_results) =
- self.combine_results(initial_results, live_results, pipeline);
-
- // retrieve webpages
- let normal: Vec<_> = top_websites
- .iter()
- .enumerate()
- .filter_map(|(i, pointer)| {
- if let ScoredWebsitePointer::Normal(p) = pointer {
- Some((i, p.clone()))
- } else {
- None
- }
- })
- .collect();
-
- let live: Vec<_> = top_websites
- .iter()
- .enumerate()
- .filter_map(|(i, pointer)| {
- if let ScoredWebsitePointer::Live(p) = pointer {
- Some((i, p.clone()))
- } else {
- None
- }
- })
- .collect();
-
- let (retrieved_normal, retrieved_live) = tokio::join!(
- self.distributed_searcher
- .retrieve_webpages(&normal, &query.query),
- self.live_searcher.retrieve_webpages(&live, &query.query),
- );
-
- let mut retrieved_webpages: Vec<_> =
- retrieved_normal.into_iter().chain(retrieved_live).collect();
- retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));
-
- let mut retrieved_webpages: Vec<_> = retrieved_webpages
- .into_iter()
- .map(|(_, webpage)| webpage)
- .map(DisplayedWebpage::from)
- .collect();
-
- if retrieved_webpages.len() != top_websites.len() {
- return Err(distributed::Error::SearchFailed.into());
- }
-
- if query.return_ranking_signals {
- self.add_ranking_signals(&mut retrieved_webpages, &top_websites);
- }
-
- let direct_answer = self.answer(&query.query, &mut retrieved_webpages);
-
- let search_duration_ms = start.elapsed().as_millis();
-
- Ok(WebsitesResult {
- spell_corrected_query,
- num_hits: num_docs,
- webpages: retrieved_webpages,
- direct_answer,
- sidebar,
- widget,
- discussions,
- search_duration_ms,
- has_more_results,
- })
- }
-
- pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
- if let Some(bang) = self.check_bangs(query).await? {
- return Ok(SearchResult::Bang(Box::new(bang)));
- }
-
- Ok(SearchResult::Websites(self.search_websites(query).await?))
- }
-
- async fn widget(&self, query: &SearchQuery) -> Option<Widget> {
- if query.page > 0 {
- return None;
- }
-
- let parsed_terms = query::parser::parse(&query.query);
-
- self.widgets.widget(
- parsed_terms
- .into_iter()
- .filter_map(|term| {
- if let query::parser::Term::Simple(simple) = *term {
- Some(String::from(simple))
- } else {
- None
- }
- })
- .join(" ")
- .as_str(),
- )
- }
-
- #[cfg(feature = "libtorch")]
- fn answer(&self, query: &str, webpages: &mut Vec<DisplayedWebpage>) -> Option<DisplayedAnswer> {
- self.qa_model.as_ref().and_then(|qa_model| {
- let contexts: Vec<_> = webpages
- .iter()
- .take(1)
- .filter_map(|webpage| webpage.snippet.text())
- .map(|t| t.fragments.iter().map(|f| f.text()).join(""))
- .collect();
-
- match qa_model.run(query, &contexts) {
- Some(answer) => {
- let answer_webpage = webpages.remove(answer.context_idx);
- let snip = answer_webpage
- .snippet
- .text()
- .unwrap()
- .fragments
- .iter()
- .map(|f| f.text())
- .join("");
- Some(DisplayedAnswer {
- title: answer_webpage.title,
- url: answer_webpage.url,
- pretty_url: answer_webpage.pretty_url,
- snippet: generate_answer_snippet(&snip, answer.offset.clone()),
- answer: snip[answer.offset].to_string(),
- })
- }
- None => None,
- }
- })
- }
-
- #[cfg(not(feature = "libtorch"))]
- fn answer(&self, query: &str, webpages: &mut Vec<DisplayedWebpage>) -> Option<DisplayedAnswer> {
- None
- }
-
- pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- self.distributed_searcher.get_webpage(url).await
- }
-
- pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
- self.distributed_searcher.get_entity_image(image_id).await
- }
-}
-
-fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {
- let mut best_start = 0;
- let mut best_end = 0;
- const SNIPPET_LENGTH: usize = 200;
-
- if body.is_empty() || answer_offset.start > body.len() - 1 {
- return body.to_string();
- }
-
- for (idx, _) in body.char_indices().filter(|(_, c)| *c == '.') {
- if idx > answer_offset.end + SNIPPET_LENGTH {
- break;
- }
-
- if idx < answer_offset.start {
- best_start = idx;
- }
-
- if idx > answer_offset.end {
- best_end = idx;
- }
- }
-
- if (answer_offset.end - best_start > SNIPPET_LENGTH) || (best_start >= best_end) {
- if answer_offset.end - answer_offset.start >= SNIPPET_LENGTH {
- let end = floor_char_boundary(body, answer_offset.start + SNIPPET_LENGTH);
-
- return "<b>".to_string() + &body[answer_offset.start..end] + "</b>";
- }
-
- let chars_either_side = (SNIPPET_LENGTH - (answer_offset.end - answer_offset.start)) / 2;
-
- let start = ceil_char_boundary(
- body,
- answer_offset
- .start
- .checked_sub(chars_either_side)
- .unwrap_or_default(),
- );
- let mut end = ceil_char_boundary(body, answer_offset.end + chars_either_side);
-
- if end >= body.len() {
- end = floor_char_boundary(body, body.len());
- }
-
- body[start..answer_offset.start].to_string()
- + "<b>"
- + &body[answer_offset.clone()]
- + "</b>"
- + &body[answer_offset.end..end]
- } else {
- let mut res = body[best_start..answer_offset.start].to_string()
- + "<b>"
- + &body[answer_offset.clone()]
- + "</b>";
-
- let remaining_chars = SNIPPET_LENGTH - (res.len() - 7);
- let end = ceil_char_boundary(body, (remaining_chars + answer_offset.end).min(best_end));
-
- res += &body[answer_offset.end..end];
-
- res
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- #[test]
- fn test_generate_answer_snippet() {
- assert_eq!(
- generate_answer_snippet("this is a test", 0..4),
- "<b>this</b> is a test".to_string()
- );
-
- assert_eq!(
- generate_answer_snippet("this is a test", 0..1000),
- "<b>this is a test</b>".to_string()
- );
- assert_eq!(
- generate_answer_snippet("this is a test", 1000..2000),
- "this is a test".to_string()
- );
- let input = r#"
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
- "#;
-
- let res = generate_answer_snippet(input, 0..500);
- assert!(!res.is_empty());
- assert!(res.len() > 100);
- assert!(res.len() < input.len());
- assert!(res.starts_with("<b>"));
- assert!(res.ends_with("</b>"));
-
- assert_eq!(generate_answer_snippet("", 0..2000), "".to_string());
- }
-}
diff --git a/core/src/searcher/discussions.optic b/core/src/searcher/api/discussions.optic
similarity index 100%
rename from core/src/searcher/discussions.optic
rename to core/src/searcher/api/discussions.optic
diff --git a/core/src/searcher/api/mod.rs b/core/src/searcher/api/mod.rs
new file mode 100644
index 00000000..c233e63c
--- /dev/null
+++ b/core/src/searcher/api/mod.rs
@@ -0,0 +1,450 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod sidebar;
+mod widget;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Instant;
+
+use itertools::intersperse;
+use url::Url;
+
+use crate::bangs::{Bang, BangHit};
+use crate::config::{ApiConfig, CollectorConfig};
+use crate::image_store::Image;
+use crate::inverted_index::RetrievedWebpage;
+#[cfg(not(feature = "libtorch"))]
+use crate::ranking::models::cross_encoder::DummyCrossEncoder;
+use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite, RetrievedWebpageRanking};
+use crate::ranking::ALL_SIGNALS;
+use crate::search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection};
+use crate::web_spell::SpellChecker;
+use crate::widgets::Widgets;
+use crate::{
+ bangs::Bangs,
+ collector::BucketCollector,
+ distributed::cluster::Cluster,
+ ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},
+};
+#[cfg(feature = "libtorch")]
+use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};
+use crate::{query, Result};
+
+use self::sidebar::SidebarManager;
+use self::widget::WidgetManager;
+
+use super::live::LiveSearcher;
+use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};
+
+#[derive(Clone)]
+pub enum ScoredWebsitePointer {
+ Normal(distributed::ScoredWebsitePointer),
+ Live(live::ScoredWebsitePointer),
+}
+
+impl AsRankingWebsite for ScoredWebsitePointer {
+ fn as_ranking(&self) -> &RankingWebsite {
+ match self {
+ ScoredWebsitePointer::Normal(p) => &p.website,
+ ScoredWebsitePointer::Live(p) => &p.website,
+ }
+ }
+
+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
+ match self {
+ ScoredWebsitePointer::Normal(p) => &mut p.website,
+ ScoredWebsitePointer::Live(p) => &mut p.website,
+ }
+ }
+}
+
+pub fn combine_results(
+ collector_config: CollectorConfig,
+ initial_results: Vec<distributed::InitialSearchResultShard>,
+ live_results: Vec<live::InitialSearchResultSplit>,
+ pipeline: RankingPipeline<ScoredWebsitePointer>,
+) -> (Vec<ScoredWebsitePointer>, bool) {
+ let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);
+
+ let mut num_sites: usize = 0;
+ for result in initial_results {
+ for website in result.local_result.websites {
+ num_sites += 1;
+ let pointer = distributed::ScoredWebsitePointer {
+ website,
+ shard: result.shard,
+ };
+
+ let pointer = ScoredWebsitePointer::Normal(pointer);
+
+ collector.insert(pointer);
+ }
+ }
+
+ for result in live_results {
+ for website in result.local_result.websites {
+ num_sites += 1;
+ let pointer = live::ScoredWebsitePointer {
+ website,
+ split_id: result.split_id.clone(),
+ };
+
+ let pointer = ScoredWebsitePointer::Live(pointer);
+
+ collector.insert(pointer);
+ }
+ }
+
+ let top_websites = collector
+ .into_sorted_vec(true)
+ .into_iter()
+ .take(pipeline.collector_top_n())
+ .collect::<Vec<_>>();
+
+ let offset = pipeline.offset();
+
+ let res = pipeline.apply(top_websites);
+ let has_more = num_sites.saturating_sub(offset) > res.len();
+
+ (res, has_more)
+}
+pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[ScoredWebsitePointer]) {
+ for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {
+ let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
+
+ for signal in ALL_SIGNALS {
+ if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {
+ signals.insert(signal, *signal_value);
+ }
+ }
+
+ website.ranking_signals = Some(signals);
+ }
+}
+
+#[cfg(feature = "libtorch")]
+pub struct ApiSearcher {
+ distributed_searcher: Arc<DistributedSearcher>,
+ sidebar_manager: SidebarManager,
+ live_searcher: LiveSearcher,
+ cross_encoder: Option<Arc<CrossEncoderModel>>,
+ lambda_model: Option<Arc<LambdaMART>>,
+ bangs: Bangs,
+ collector_config: CollectorConfig,
+ widget_manager: WidgetManager,
+ spell_checker: Option<SpellChecker>,
+}
+
+#[cfg(not(feature = "libtorch"))]
+pub struct ApiSearcher {
+ distributed_searcher: Arc<DistributedSearcher>,
+ live_searcher: LiveSearcher,
+ sidebar_manager: SidebarManager,
+ lambda_model: Option<Arc<LambdaMART>>,
+ bangs: Bangs,
+ collector_config: CollectorConfig,
+ widget_manager: WidgetManager,
+ spell_checker: Option<SpellChecker>,
+}
+
+impl ApiSearcher {
+ #[cfg(feature = "libtorch")]
+ pub fn new(
+ cluster: Arc<Cluster>,
+ cross_encoder: Option<CrossEncoderModel>,
+ lambda_model: Option<LambdaMART>,
+ qa_model: Option<QaModel>,
+ bangs: Bangs,
+ config: ApiConfig,
+ ) -> Self {
+ let dist_searcher = Arc::new(DistributedSearcher::new(Arc::clone(&cluster)));
+ let sidebar_manager =
+ SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());
+
+ let lambda_model = lambda_model.map(Arc::new);
+ let qa_model = qa_model.map(Arc::new);
+
+ let widget_manager = WidgetManager::new(
+ Widgets::new(config.widgets).unwrap(),
+ Arc::clone(&dist_searcher),
+ config.thresholds.clone(),
+ config.collector.clone(),
+ lambda_model.clone(),
+ qa_model,
+ );
+
+ Self {
+ distributed_searcher: dist_searcher,
+ sidebar_manager,
+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),
+ cross_encoder: cross_encoder.map(Arc::new),
+ lambda_model,
+ bangs,
+ collector_config: config.collector,
+ widget_manager,
+ spell_checker: config
+ .spell_checker_path
+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
+ }
+ }
+
+ #[cfg(not(feature = "libtorch"))]
+ pub fn new(
+ cluster: Arc<Cluster>,
+ lambda_model: Option<LambdaMART>,
+ bangs: Bangs,
+ config: ApiConfig,
+ ) -> Self {
+ let dist_searcher = Arc::new(DistributedSearcher::new(Arc::clone(&cluster)));
+ let sidebar_manager =
+ SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());
+ let lambda_model = lambda_model.map(Arc::new);
+
+ let widget_manager = WidgetManager::new(
+ Widgets::new(config.widgets).unwrap(),
+ Arc::clone(&dist_searcher),
+ config.thresholds.clone(),
+ config.collector.clone(),
+ lambda_model.clone(),
+ );
+
+ Self {
+ distributed_searcher: dist_searcher,
+ sidebar_manager,
+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),
+ lambda_model,
+ bangs,
+ collector_config: config.collector,
+ widget_manager,
+ spell_checker: config
+ .spell_checker_path
+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),
+ }
+ }
+
+ async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {
+ let parsed_terms = query::parser::parse(&query.query);
+
+ if parsed_terms.iter().any(|term| match term.as_ref() {
+ query::parser::Term::PossibleBang(t) => t.is_empty(),
+ _ => false,
+ }) {
+ let q: String = intersperse(
+ parsed_terms
+ .iter()
+ .filter(|term| !matches!(term.as_ref(), query::parser::Term::PossibleBang(_)))
+ .map(|term| term.to_string()),
+ " ".to_string(),
+ )
+ .collect();
+
+ let mut query = query.clone();
+ query.query = q;
+
+ let res = self.search_websites(&query).await?;
+
+ return Ok(res.webpages.first().map(|webpage| BangHit {
+ bang: Bang {
+ category: None,
+ sub_category: None,
+ domain: None,
+ ranking: None,
+ site: None,
+ tag: String::new(),
+ url: webpage.url.clone(),
+ },
+ redirect_to: Url::parse(&webpage.url).unwrap().into(),
+ }));
+ }
+
+ Ok(self.bangs.get(&parsed_terms))
+ }
+
+ async fn retrieve_webpages(
+ &self,
+ query: &str,
+ top_websites: &[ScoredWebsitePointer],
+ ) -> Vec<RetrievedWebpageRanking> {
+ let normal: Vec<_> = top_websites
+ .iter()
+ .enumerate()
+ .filter_map(|(i, pointer)| {
+ if let ScoredWebsitePointer::Normal(p) = pointer {
+ Some((i, p.clone()))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let live: Vec<_> = top_websites
+ .iter()
+ .enumerate()
+ .filter_map(|(i, pointer)| {
+ if let ScoredWebsitePointer::Live(p) = pointer {
+ Some((i, p.clone()))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let (retrieved_normal, retrieved_live) = tokio::join!(
+ self.distributed_searcher.retrieve_webpages(&normal, query),
+ self.live_searcher.retrieve_webpages(&live, query),
+ );
+
+ let mut retrieved_webpages: Vec<_> =
+ retrieved_normal.into_iter().chain(retrieved_live).collect();
+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+ retrieved_webpages
+ .into_iter()
+ .map(|(_, webpage)| webpage)
+ .collect::<Vec<_>>()
+ }
+
+ async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {
+ let start = Instant::now();
+
+ if query.is_empty() {
+ return Err(distributed::Error::EmptyQuery.into());
+ }
+
+ let mut search_query = query.clone();
+
+ // This pipeline should be created before the first search is performed
+ // so the query knows how many results to fetch from the indices
+ let first_pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::first_stage(
+ &mut search_query,
+ self.lambda_model.clone(),
+ self.collector_config.clone(),
+ 20,
+ );
+
+ let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(
+ self.distributed_searcher.search_initial(&search_query),
+ self.live_searcher.search_initial(&search_query),
+ self.widget_manager.widget(query),
+ self.widget_manager.discussions(query),
+ self.sidebar_manager.stackoverflow(query),
+ );
+
+ let discussions = discussions?;
+ let stackoverflow = stackoverflow?;
+ let sidebar = self
+ .sidebar_manager
+ .sidebar(&initial_results, stackoverflow);
+
+ let spell_corrected_query = if widget.is_none() {
+ self.spell_checker
+ .as_ref()
+ .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))
+ .map(HighlightedSpellCorrection::from)
+ } else {
+ None
+ };
+
+ let num_docs = initial_results
+ .iter()
+ .map(|result| result.local_result.num_websites)
+ .sum();
+
+ let (top_websites, has_more_results) = combine_results(
+ self.collector_config.clone(),
+ initial_results,
+ live_results,
+ first_pipeline,
+ );
+
+ let retrieved_webpages = self
+ .retrieve_webpages(&search_query.query, &top_websites)
+ .await;
+
+ let mut search_query = query.clone();
+
+ #[cfg(feature = "libtorch")]
+ let pipeline: RankingPipeline<RetrievedWebpageRanking> = RankingPipeline::reranker(
+ &mut search_query,
+ self.cross_encoder.clone(),
+ self.lambda_model.clone(),
+ self.collector_config.clone(),
+ query.num_results,
+ )?;
+
+ #[cfg(not(feature = "libtorch"))]
+ let pipeline: RankingPipeline<RetrievedWebpageRanking> =
+ RankingPipeline::reranker::<DummyCrossEncoder>(
+ &mut search_query,
+ None,
+ self.lambda_model.clone(),
+ self.collector_config.clone(),
+ query.num_results,
+ )?;
+
+ let retrieved_webpages = pipeline.apply(retrieved_webpages);
+
+ let mut retrieved_webpages: Vec<_> = retrieved_webpages
+ .into_iter()
+ .map(|webpage| webpage.into_retrieved_webpage())
+ .map(DisplayedWebpage::from)
+ .collect();
+
+ if retrieved_webpages.len() != top_websites.len() {
+ return Err(distributed::Error::SearchFailed.into());
+ }
+
+ if query.return_ranking_signals {
+ add_ranking_signals(&mut retrieved_webpages, &top_websites);
+ }
+
+ let direct_answer = self
+ .widget_manager
+ .answer(&query.query, &mut retrieved_webpages);
+
+ let search_duration_ms = start.elapsed().as_millis();
+
+ Ok(WebsitesResult {
+ spell_corrected_query,
+ num_hits: num_docs,
+ webpages: retrieved_webpages,
+ direct_answer,
+ sidebar,
+ widget,
+ discussions,
+ search_duration_ms,
+ has_more_results,
+ })
+ }
+
+ pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
+ if let Some(bang) = self.check_bangs(query).await? {
+ return Ok(SearchResult::Bang(Box::new(bang)));
+ }
+
+ Ok(SearchResult::Websites(self.search_websites(query).await?))
+ }
+
+ pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
+ self.distributed_searcher.get_webpage(url).await
+ }
+
+ pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
+ self.distributed_searcher.get_entity_image(image_id).await
+ }
+}
diff --git a/core/src/searcher/api/sidebar.rs b/core/src/searcher/api/sidebar.rs
new file mode 100644
index 00000000..205bfac4
--- /dev/null
+++ b/core/src/searcher/api/sidebar.rs
@@ -0,0 +1,113 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{config::ApiThresholds, Result};
+use std::{cmp::Ordering, sync::Arc};
+
+use optics::Optic;
+use url::Url;
+
+use crate::{
+ search_prettifier::{create_stackoverflow_sidebar, DisplayedEntity, DisplayedSidebar},
+ searcher::{distributed, DistributedSearcher, SearchQuery},
+};
+
+pub struct SidebarManager {
+ distributed_searcher: Arc<DistributedSearcher>,
+ thresholds: ApiThresholds,
+}
+
+impl SidebarManager {
+ pub fn new(
+ distributed_searcher: Arc<DistributedSearcher>,
+ thresholds: ApiThresholds,
+ ) -> SidebarManager {
+ Self {
+ distributed_searcher,
+ thresholds,
+ }
+ }
+
+ pub async fn stackoverflow(&self, query: &SearchQuery) -> Result<Option<DisplayedSidebar>> {
+ let query = SearchQuery {
+ query: query.query.clone(),
+ num_results: 1,
+ optic: Some(Optic::parse(include_str!("stackoverflow.optic")).unwrap()),
+ ..Default::default()
+ };
+
+ let mut results: Vec<_> = self
+ .distributed_searcher
+ .search_initial(&query)
+ .await
+ .into_iter()
+ .filter_map(|result| {
+ result
+ .local_result
+ .websites
+ .first()
+ .cloned()
+ .map(|website| (result.shard, website))
+ })
+ .collect();
+
+ results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));
+
+ if let Some((shard, website)) = results.pop() {
+ tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");
+ if website.score > self.thresholds.stackoverflow {
+ let scored_websites =
+ vec![(0, distributed::ScoredWebsitePointer { website, shard })];
+ let mut retrieved = self
+ .distributed_searcher
+ .retrieve_webpages(&scored_websites, &query.query)
+ .await;
+
+ if let Some((_, res)) = retrieved.pop() {
+ let res = res.into_retrieved_webpage();
+ return Ok(Some(create_stackoverflow_sidebar(
+ res.schema_org,
+ Url::parse(&res.url).unwrap(),
+ )?));
+ }
+ }
+ }
+
+ Ok(None)
+ }
+
+ pub fn sidebar(
+ &self,
+ initial_results: &[distributed::InitialSearchResultShard],
+ stackoverflow: Option<DisplayedSidebar>,
+ ) -> Option<DisplayedSidebar> {
+ let entity = initial_results
+ .iter()
+ .filter_map(|res| res.local_result.entity_sidebar.clone())
+ .map(DisplayedEntity::from)
+ .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)
+ .max_by(|a, b| {
+ a.match_score
+ .partial_cmp(&b.match_score)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ match entity {
+ Some(entity) => Some(DisplayedSidebar::Entity(entity)),
+ None => stackoverflow,
+ }
+ }
+}
diff --git a/core/src/searcher/stackoverflow.optic b/core/src/searcher/api/stackoverflow.optic
similarity index 100%
rename from core/src/searcher/stackoverflow.optic
rename to core/src/searcher/api/stackoverflow.optic
diff --git a/core/src/searcher/api/widget.rs b/core/src/searcher/api/widget.rs
new file mode 100644
index 00000000..f27e4bcf
--- /dev/null
+++ b/core/src/searcher/api/widget.rs
@@ -0,0 +1,367 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::ops::Range;
+use std::sync::Arc;
+
+use itertools::Itertools;
+use optics::Optic;
+
+#[cfg(feature = "libtorch")]
+use crate::qa_model::QaModel;
+use crate::ranking::models::lambdamart::LambdaMART;
+use crate::ranking::pipeline::{AsRankingWebsite, RankingPipeline};
+use crate::search_prettifier::DisplayedAnswer;
+use crate::search_prettifier::DisplayedWebpage;
+use crate::searcher::api::{add_ranking_signals, combine_results, ScoredWebsitePointer};
+use crate::searcher::{DistributedSearcher, SearchQuery};
+use crate::widgets::Widget;
+use crate::{ceil_char_boundary, floor_char_boundary, query, Result};
+use crate::{
+ config::{ApiThresholds, CollectorConfig},
+ widgets::Widgets,
+};
+
+#[cfg(feature = "libtorch")]
+pub struct WidgetManager {
+ widgets: Widgets,
+ distributed_searcher: Arc<DistributedSearcher>,
+ thresholds: ApiThresholds,
+ collector_config: CollectorConfig,
+ lambda_model: Option<Arc<LambdaMART>>,
+ qa_model: Option<Arc<QaModel>>,
+}
+
+#[cfg(not(feature = "libtorch"))]
+pub struct WidgetManager {
+ widgets: Widgets,
+ distributed_searcher: Arc<DistributedSearcher>,
+ thresholds: ApiThresholds,
+ collector_config: CollectorConfig,
+ lambda_model: Option<Arc<LambdaMART>>,
+}
+
+impl WidgetManager {
+ #[cfg(feature = "libtorch")]
+ pub fn new(
+ widgets: Widgets,
+ distributed_searcher: Arc<DistributedSearcher>,
+ thresholds: ApiThresholds,
+ collector_config: CollectorConfig,
+ lambda_model: Option<Arc<LambdaMART>>,
+ qa_model: Option<Arc<QaModel>>,
+ ) -> WidgetManager {
+ Self {
+ widgets,
+ distributed_searcher,
+ thresholds,
+ collector_config,
+ lambda_model,
+ qa_model,
+ }
+ }
+
+ #[cfg(not(feature = "libtorch"))]
+ pub fn new(
+ widgets: Widgets,
+ distributed_searcher: Arc<DistributedSearcher>,
+ thresholds: ApiThresholds,
+ collector_config: CollectorConfig,
+ lambda_model: Option<Arc<LambdaMART>>,
+ ) -> WidgetManager {
+ Self {
+ widgets,
+ distributed_searcher,
+ thresholds,
+ collector_config,
+ lambda_model,
+ }
+ }
+
+ pub async fn discussions(&self, query: &SearchQuery) -> Result<Option<Vec<DisplayedWebpage>>> {
+ if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {
+ return Ok(None);
+ }
+
+ const NUM_RESULTS: usize = 10;
+
+ let mut query = SearchQuery {
+ query: query.query.clone(),
+ num_results: NUM_RESULTS,
+ optic: Some(Optic::parse(include_str!("discussions.optic")).unwrap()),
+ site_rankings: query.site_rankings.clone(),
+ return_ranking_signals: query.return_ranking_signals,
+ ..Default::default()
+ };
+
+ // This pipeline should be created before the first search is performed
+ // so the query knows how many results to fetch from the indices
+ let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::first_stage(
+ &mut query,
+ self.lambda_model.clone(),
+ self.collector_config.clone(),
+ NUM_RESULTS,
+ );
+
+ let initial_results = self.distributed_searcher.search_initial(&query).await;
+
+ if initial_results.is_empty() {
+ return Ok(None);
+ }
+
+ let num_results: usize = initial_results
+ .iter()
+ .map(|res| res.local_result.websites.len())
+ .sum();
+
+ if num_results < NUM_RESULTS / 2 {
+ return Ok(None);
+ }
+
+ let (top_websites, _) = combine_results(
+ self.collector_config.clone(),
+ initial_results,
+ vec![],
+ pipeline,
+ );
+
+ let scores: Vec<_> = top_websites
+ .iter()
+ .map(|pointer| pointer.as_ranking().score)
+ .collect();
+
+ let median = if scores.len() % 2 == 0 {
+ (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0
+ } else {
+ scores[scores.len() / 2]
+ };
+
+ if median < self.thresholds.discussions_widget {
+ return Ok(None);
+ }
+
+ let mut result: Vec<DisplayedWebpage> = self
+ .distributed_searcher
+ .retrieve_webpages(
+ &top_websites
+ .clone()
+ .into_iter()
+ .filter_map(|pointer| {
+ if let ScoredWebsitePointer::Normal(p) = pointer {
+ Some(p)
+ } else {
+ None
+ }
+ })
+ .enumerate()
+ .collect_vec(),
+ &query.query,
+ )
+ .await
+ .into_iter()
+ .map(|(_, webpage)| webpage.into_retrieved_webpage())
+ .map(DisplayedWebpage::from)
+ .collect();
+
+ if query.return_ranking_signals {
+ add_ranking_signals(&mut result, &top_websites);
+ }
+
+ Ok(Some(result))
+ }
+
+ pub async fn widget(&self, query: &SearchQuery) -> Option<Widget> {
+ if query.page > 0 {
+ return None;
+ }
+
+ let parsed_terms = query::parser::parse(&query.query);
+
+ self.widgets.widget(
+ parsed_terms
+ .into_iter()
+ .filter_map(|term| {
+ if let query::parser::Term::Simple(simple) = *term {
+ Some(String::from(simple))
+ } else {
+ None
+ }
+ })
+ .join(" ")
+ .as_str(),
+ )
+ }
+
+ #[cfg(feature = "libtorch")]
+ pub fn answer(
+ &self,
+ query: &str,
+ webpages: &mut Vec<DisplayedWebpage>,
+ ) -> Option<DisplayedAnswer> {
+ self.qa_model.as_ref().and_then(|qa_model| {
+ let contexts: Vec<_> = webpages
+ .iter()
+ .take(1)
+ .filter_map(|webpage| webpage.snippet.text())
+ .map(|t| t.fragments.iter().map(|f| f.text()).join(""))
+ .collect();
+
+ match qa_model.run(query, &contexts) {
+ Some(answer) => {
+ let answer_webpage = webpages.remove(answer.context_idx);
+ let snip = answer_webpage
+ .snippet
+ .text()
+ .unwrap()
+ .fragments
+ .iter()
+ .map(|f| f.text())
+ .join("");
+ Some(DisplayedAnswer {
+ title: answer_webpage.title,
+ url: answer_webpage.url,
+ pretty_url: answer_webpage.pretty_url,
+ snippet: generate_answer_snippet(&snip, answer.offset.clone()),
+ answer: snip[answer.offset].to_string(),
+ })
+ }
+ None => None,
+ }
+ })
+ }
+
+ #[cfg(not(feature = "libtorch"))]
+ pub fn answer(
+ &self,
+ query: &str,
+ webpages: &mut Vec<DisplayedWebpage>,
+ ) -> Option<DisplayedAnswer> {
+ None
+ }
+}
+
+fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {
+ let mut best_start = 0;
+ let mut best_end = 0;
+ const SNIPPET_LENGTH: usize = 200;
+
+ if body.is_empty() || answer_offset.start > body.len() - 1 {
+ return body.to_string();
+ }
+
+ for (idx, _) in body.char_indices().filter(|(_, c)| *c == '.') {
+ if idx > answer_offset.end + SNIPPET_LENGTH {
+ break;
+ }
+
+ if idx < answer_offset.start {
+ best_start = idx;
+ }
+
+ if idx > answer_offset.end {
+ best_end = idx;
+ }
+ }
+
+ if (answer_offset.end - best_start > SNIPPET_LENGTH) || (best_start >= best_end) {
+ if answer_offset.end - answer_offset.start >= SNIPPET_LENGTH {
+ let end = floor_char_boundary(body, answer_offset.start + SNIPPET_LENGTH);
+
+ return "<b>".to_string() + &body[answer_offset.start..end] + "</b>";
+ }
+
+ let chars_either_side = (SNIPPET_LENGTH - (answer_offset.end - answer_offset.start)) / 2;
+
+ let start = ceil_char_boundary(
+ body,
+ answer_offset
+ .start
+ .checked_sub(chars_either_side)
+ .unwrap_or_default(),
+ );
+ let mut end = ceil_char_boundary(body, answer_offset.end + chars_either_side);
+
+ if end >= body.len() {
+ end = floor_char_boundary(body, body.len());
+ }
+
+ body[start..answer_offset.start].to_string()
+ + "<b>"
+ + &body[answer_offset.clone()]
+ + "</b>"
+ + &body[answer_offset.end..end]
+ } else {
+ let mut res = body[best_start..answer_offset.start].to_string()
+ + "<b>"
+ + &body[answer_offset.clone()]
+ + "</b>";
+
+ let remaining_chars = SNIPPET_LENGTH - (res.len() - 7);
+ let end = ceil_char_boundary(body, (remaining_chars + answer_offset.end).min(best_end));
+
+ res += &body[answer_offset.end..end];
+
+ res
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn test_generate_answer_snippet() {
+ assert_eq!(
+ generate_answer_snippet("this is a test", 0..4),
+ "<b>this</b> is a test".to_string()
+ );
+
+ assert_eq!(
+ generate_answer_snippet("this is a test", 0..1000),
+ "<b>this is a test</b>".to_string()
+ );
+ assert_eq!(
+ generate_answer_snippet("this is a test", 1000..2000),
+ "this is a test".to_string()
+ );
+ let input = r#"
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
+ "#;
+
+ let res = generate_answer_snippet(input, 0..500);
+ assert!(!res.is_empty());
+ assert!(res.len() > 100);
+ assert!(res.len() < input.len());
+ assert!(res.starts_with("<b>"));
+ assert!(res.ends_with("</b>"));
+
+ assert_eq!(generate_answer_snippet("", 0..2000), "".to_string());
+ }
+}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index eb107a16..39135a3f 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -26,12 +26,13 @@ use crate::{
entrypoint::search_server::{self, SearchService},
image_store::Image,
inverted_index::{RetrievedWebpage, WebsitePointer},
- ranking::pipeline::RankingWebsite,
+ ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},
Result,
};
use std::{collections::HashMap, sync::Arc};
+use fnv::FnvHashMap;
use futures::future::join_all;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
@@ -161,7 +162,8 @@ impl DistributedSearcher {
&self,
top_websites: &[(usize, ScoredWebsitePointer)],
query: &str,
- ) -> Vec<(usize, RetrievedWebpage)> {
+ ) -> Vec<(usize, RetrievedWebpageRanking)> {
+ let mut rankings = FnvHashMap::default();
let mut pointers: HashMap<_, Vec<_>> = HashMap::new();
for (i, pointer) in top_websites {
@@ -169,6 +171,8 @@ impl DistributedSearcher {
.entry(pointer.shard)
.or_default()
.push((*i, pointer.website.pointer.clone()));
+
+ rankings.insert(*i, pointer.website.clone());
}
let client = self.client().await;
@@ -179,7 +183,10 @@ impl DistributedSearcher {
let mut retrieved_webpages = Vec::new();
for pages in join_all(futures).await {
- retrieved_webpages.extend(pages);
+ for (i, page) in pages {
+ retrieved_webpages
+ .push((i, RetrievedWebpageRanking::new(page, rankings[&i].clone())));
+ }
}
debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
diff --git a/core/src/searcher/live.rs b/core/src/searcher/live.rs
index 6593d10b..1e90ebd2 100644
--- a/core/src/searcher/live.rs
+++ b/core/src/searcher/live.rs
@@ -26,11 +26,12 @@ use crate::{
entrypoint::search_server::{self, SearchService},
feed::scheduler::SplitId,
inverted_index::{RetrievedWebpage, WebsitePointer},
- ranking::pipeline::RankingWebsite,
+ ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},
};
use std::{collections::HashMap, sync::Arc};
+use fnv::FnvHashMap;
use futures::future::join_all;
use itertools::Itertools;
@@ -142,7 +143,8 @@ impl LiveSearcher {
&self,
top_websites: &[(usize, ScoredWebsitePointer)],
query: &str,
- ) -> Vec<(usize, RetrievedWebpage)> {
+ ) -> Vec<(usize, RetrievedWebpageRanking)> {
+ let mut rankings = FnvHashMap::default();
let mut pointers: HashMap<_, Vec<_>> = HashMap::new();
for (i, pointer) in top_websites {
@@ -150,17 +152,22 @@ impl LiveSearcher {
.entry(pointer.split_id.clone())
.or_default()
.push((*i, pointer.website.pointer.clone()));
+
+ rankings.insert(*i, pointer.website.clone());
}
let client = self.client().await;
let mut futures = Vec::new();
- for (split_id, pointers) in pointers {
- futures.push(self.retrieve_webpages_from_shard(split_id, &client, query, pointers));
+ for (shard, pointers) in pointers {
+ futures.push(self.retrieve_webpages_from_shard(shard, &client, query, pointers));
}
let mut retrieved_webpages = Vec::new();
for pages in join_all(futures).await {
- retrieved_webpages.extend(pages);
+ for (i, page) in pages {
+ retrieved_webpages
+ .push((i, RetrievedWebpageRanking::new(page, rankings[&i].clone())));
+ }
}
debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index d47185a5..b85c0d32 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -17,8 +17,6 @@
use std::collections::HashMap;
use std::sync::{Arc, RwLockReadGuard};
-use tantivy::schema::Value;
-use tantivy::TantivyDocument;
use url::Url;
use crate::config::{CollectorConfig, SnippetConfig};
@@ -34,7 +32,6 @@ use crate::ranking::models::lambdamart::LambdaMART;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};
-use crate::schema::TextField;
use crate::search_ctx::Ctx;
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
use crate::webgraph::Node;
@@ -116,7 +113,6 @@ impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {
pub struct LocalSearcher<I: SearchableIndex> {
index: I,
- // spell: Option<Spell>,
entity_index: Option<EntityIndex>,
inbound_similarity: Option<InboundSimilarity>,
linear_regression: Option<Arc<LinearRegression>>,
@@ -257,10 +253,11 @@ where
de_rank_similar: bool,
) -> Result<InvertedIndexResult> {
let mut query = query.clone();
- let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::initial_for_query(
+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::first_stage(
&mut query,
self.lambda_model.clone(),
self.collector_config.clone(),
+ 100,
);
let parsed_query = self.parse_query(ctx, guard, &query)?;
@@ -319,26 +316,7 @@ where
)?;
let pipe_top_n = pipeline.top_n;
- let mut ranking_websites = pipeline.apply(ranking_websites);
-
- let schema = guard.inverted_index().schema();
- for website in &mut ranking_websites {
- let doc: TantivyDocument = ctx.tv_searcher.doc(website.pointer.address.into())?;
- website.title = Some(
- doc.get_first(schema.get_field(TextField::Title.name()).unwrap())
- .map(|text| text.as_value().as_str().unwrap().to_string())
- .unwrap_or_default(),
- );
- website.clean_body = Some(
- doc.get_first(
- schema
- .get_field(TextField::StemmedCleanBody.name())
- .unwrap(),
- )
- .map(|text| text.as_value().as_str().unwrap().to_string())
- .unwrap_or_default(),
- );
- }
+ let ranking_websites = pipeline.apply(ranking_websites);
let has_more = pipe_top_n == ranking_websites.len();
@@ -409,17 +387,19 @@ where
let pipeline = {
use crate::ranking::models::cross_encoder::CrossEncoderModel;
match CrossEncoderModel::open("data/cross_encoder") {
- Ok(model) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(
+ Ok(model) => RankingPipeline::reranker::<CrossEncoderModel>(
&mut search_query,
Some(Arc::new(model)),
None,
self.collector_config.clone(),
+ query.num_results,
)?,
- Err(_) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(
+ Err(_) => RankingPipeline::reranker::<CrossEncoderModel>(
&mut search_query,
None,
None,
self.collector_config.clone(),
+ query.num_results,
)?,
}
};
@@ -427,11 +407,12 @@ where
#[cfg(not(feature = "libtorch"))]
let pipeline = {
use crate::ranking::models::cross_encoder::DummyCrossEncoder;
- RankingPipeline::reranking_for_query::<DummyCrossEncoder>(
+ RankingPipeline::reranker::<DummyCrossEncoder>(
&mut search_query,
None,
None,
self.collector_config.clone(),
+ query.num_results,
)?
};
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 7b4568a4..b0498f90 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -77,9 +77,9 @@ impl SimilarSitesFinder {
}
pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {
- const SIMILAR_DOMAINS_BUFFER: usize = 30;
+ const DEAD_LINKS_BUFFER: usize = 30;
let orig_limit = limit.min(self.max_similar_sites);
- let limit = orig_limit + nodes.len() + SIMILAR_DOMAINS_BUFFER;
+ let limit = orig_limit + nodes.len() + DEAD_LINKS_BUFFER;
let nodes: Vec<_> = nodes
.iter()
@@ -155,6 +155,9 @@ impl SimilarSitesFinder {
scored_nodes
.into_iter()
+ .filter(|ScoredNodeID { node_id, score: _ }| {
+ !self.webgraph.raw_outgoing_edges(node_id).is_empty()
+ })
.filter_map(|ScoredNodeID { node_id, score }| {
let node = self.webgraph.id2node(&node_id).unwrap();
match Url::parse(&format!("http://{}", &node.name))
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 263db069..3f0c9f57 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -72,6 +72,16 @@ pub struct TextSnippet {
pub fragments: Vec<TextSnippetFragment>,
}
+impl TextSnippet {
+ pub fn unhighlighted_string(&self) -> String {
+ self.fragments
+ .iter()
+ .map(|f| f.text.clone())
+ .collect::<Vec<_>>()
+ .join("")
+ }
+}
+
struct SnippetBuilder {
fragment: String,
highlights: Vec<Range<usize>>,
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index eb11fc7d..855a26e1 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -582,6 +582,10 @@ impl WebgraphWriter {
}
pub fn insert(&mut self, from: Node, to: Node, label: String) {
+ if from == to {
+ return;
+ }
+
let (from_id, to_id) = (
self.id_or_assign(from.clone()),
self.id_or_assign(to.clone()),
diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rs
index 0aebd97c..607d3da1 100644
--- a/core/src/webpage/just_text.rs
+++ b/core/src/webpage/just_text.rs
@@ -412,7 +412,7 @@ impl JustText {
res
}
- fn contextual_classification(&self, paragraphs: &mut Vec<ClassifiedParagraph>) {
+ fn contextual_classification(&self, paragraphs: &mut [ClassifiedParagraph]) {
// good headings
let num_paragraphs = paragraphs.len();
for i in 0..num_paragraphs {
@@ -602,7 +602,7 @@ impl JustText {
.initial_classification(paragraphs, lang)
.into_iter()
.filter(|par| par.paragraph.text.chars().any(|c| !c.is_whitespace()))
- .collect();
+ .collect::<Vec<_>>();
self.contextual_classification(&mut classified);
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index c895adb9..99920dd1 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -1015,6 +1015,8 @@ impl Html {
.to_string();
let schemas: Vec<_> = self.schema_org();
+ let first_ingredient_tag_id =
+ find_recipe_first_ingredient_tag_id(&schemas, &self.root).unwrap_or_default();
let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
@@ -1255,6 +1257,9 @@ impl Html {
Field::Text(TextField::AllBody) => {
doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
}
+ Field::Text(TextField::RecipeFirstIngredientTagId) => {
+ doc.add_text(tantivy_field, first_ingredient_tag_id.clone());
+ }
Field::Text(TextField::SchemaOrgJson) => {
doc.add_text(tantivy_field, schema_json.clone());
}
@@ -1713,6 +1718,52 @@ fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {
}
}
+fn find_recipe_first_ingredient_tag_id(
+ schemas: &[schema_org::Item],
+ root: &NodeRef,
+) -> Option<String> {
+ schemas
+ .iter()
+ .filter_map(|schema| {
+ if let Some(ingredients) = schema.properties.get("recipeIngredient") {
+ if let Some(ingredient) = ingredients.clone().many().first() {
+ if let Some(ingredient) = ingredient.try_into_string() {
+ let ingredient = ingredient.trim();
+ // find first occurrence in html
+ if let Some(ingredient_node) = root
+ .select("body")
+ .unwrap()
+ .flat_map(|node| node.as_node().descendants())
+ .find(|node| {
+ if let Some(text) = node.as_text() {
+ text.borrow().trim() == ingredient
+ } else {
+ false
+ }
+ })
+ {
+ // find first parent that has an id
+ if let Some(id) = ingredient_node
+ .ancestors()
+ .filter_map(|node| {
+ node.as_element().and_then(|e| {
+ e.attributes.borrow().get("id").map(|s| s.to_string())
+ })
+ })
+ .next()
+ {
+ return Some(id.to_string());
+ }
+ }
+ }
+ }
+ }
+
+ None
+ })
+ .next()
+}
+
fn stem_tokens(tokens: &mut [tantivy::tokenizer::Token], lang: Lang) {
let stemmer = stemmer_from_lang(&lang);
for token in tokens {
@@ -2592,4 +2643,65 @@ mod tests {
assert!(!microformats.contains(Microformat::HCard));
assert!(!microformats.contains(Microformat::HProduct));
}
+
+ #[test]
+ fn recipe_first_ingredient_tag() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ <script type="application/ld+json">
+ {
+ "@context": "https://schema.org",
+ "@type": "Recipe",
+ "author": "John Smith",
+ "cookTime": "PT1H",
+ "datePublished": "2009-05-08",
+ "description": "This classic banana bread recipe comes from my mom -- the walnuts add a nice texture and flavor to the banana bread.",
+ "image": "bananabread.jpg",
+ "recipeIngredient": [
+ "3 or 4 ripe bananas, smashed",
+ "1 egg",
+ "3/4 cup of sugar"
+ ],
+ "interactionStatistic": {
+ "@type": "InteractionCounter",
+ "interactionType": "https://schema.org/Comment",
+ "userInteractionCount": "140"
+ },
+ "name": "Mom's World Famous Banana Bread",
+ "nutrition": {
+ "@type": "NutritionInformation",
+ "calories": "240 calories",
+ "fatContent": "9 grams fat"
+ },
+ "prepTime": "PT15M",
+ "recipeInstructions": "Preheat the oven to 350 degrees. Mix in the ingredients in a bowl. Add the flour last. Pour the mixture into a loaf pan and bake for one hour.",
+ "recipeYield": "1 loaf",
+ "suitableForDiet": "https://schema.org/LowFatDiet"
+ }
+ </script>
+
+ <div id="ingredients">
+ <h2>Ingredients</h2>
+ <ul>
+ <li>3 or 4 ripe bananas, smashed</li>
+ <li>1 egg</li>
+ <li>3/4 cup of sugar</li>
+ </ul>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/",
+ ).unwrap();
+
+ let schemas = html.schema_org();
+
+ assert_eq!(
+ find_recipe_first_ingredient_tag_id(&schemas, &html.root),
+ Some("ingredients".to_string())
+ );
+ }
}
diff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rs
index f7012127..3e99b6e3 100644
--- a/core/src/widgets/calculator.rs
+++ b/core/src/widgets/calculator.rs
@@ -159,6 +159,11 @@ impl Calculator {
pub fn try_calculate(&self, expr: &str) -> Result<Calculation, Error> {
let expr = expr.replace(['"', '\''], "");
+ // check if expr has at least one digit
+ if !expr.chars().any(|c| c.is_ascii_digit()) {
+ return Err(Error::CalculatorParse);
+ }
+
// if expr starts with "d[0-9]+", wrap it in "roll(...)"
let expr = if DICE_REGEX.is_match(&expr) {
format!("roll({})", expr)
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index f690696c..a7b802d8 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -143,10 +143,10 @@
{#if results.type == 'websites'}
<div
- class="m-0 grid w-full gap-y-6 px-5 pt-4 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] md:gap-x-12 md:pl-20 lg:px-36"
+ class="m-0 grid w-full gap-y-6 px-5 pt-4 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] md:gap-x-12 md:pl-20 lg:pl-28"
style="text-rendering:optimizeLegibility;font-smooth:always;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;"
>
- <div class="flex max-w-2xl flex-col space-y-3">
+ <div class="flex max-w-2xl flex-col space-y-2">
<div class="w-full">
<Searchbar {query} />
</div>
@@ -233,7 +233,7 @@
{#if results.sidebar}
<div
- class="row-start-2 mx-auto max-w-[90vw] justify-center md:col-start-2 md:row-span-2 md:row-start-1 md:pt-10"
+ class="row-start-2 mx-auto max-w-[90vw] md:max-w-[30vw] justify-center md:col-start-2 md:row-span-2 md:row-start-1 md:pt-5"
>
<Sidebar sidebar={results.sidebar} />
</div>
diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelte
index 142efeca..8123d2fe 100644
--- a/frontend/src/routes/search/Discussions.svelte
+++ b/frontend/src/routes/search/Discussions.svelte
@@ -51,7 +51,7 @@
</div>
{/each}
<div class="noscript:hidden mt-2">
- <Button kind="neutral" pale on:click={() => (showMore = !showMore)}
+ <Button _class="py-1" kind="neutral" pale on:click={() => (showMore = !showMore)}
>{showMore ? 'Show less' : 'Show more'}</Button
>
</div>
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index 5cf3e43c..e8050ac7 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -20,9 +20,9 @@
const dispatch = createEventDispatcher<{ modal: HTMLButtonElement }>();
</script>
-<div class="flex min-w-0 grow flex-col space-y-1">
+<div class="flex min-w-0 grow flex-col space-y-0.5">
<div class="flex min-w-0">
- <div class="flex min-w-0 grow flex-col space-y-1">
+ <div class="flex min-w-0 grow flex-col space-y-0.5">
<div class="flex items-center text-sm">
<a
class="max-w-[calc(100%-100px)] truncate text-neutral-focus"
@@ -46,7 +46,7 @@
bind:this={button}
on:click|stopPropagation={() => dispatch('modal', button)}
>
- <AdjustVertical class="text-xl" />
+ <AdjustVertical class="text-md" />
</button>
</div>
<div class="text-sm font-normal text-neutral-focus [&>b]:font-bold">
diff --git a/frontend/src/routes/search/StackOverflow.svelte b/frontend/src/routes/search/StackOverflow.svelte
index e3683b76..d903908c 100644
--- a/frontend/src/routes/search/StackOverflow.svelte
+++ b/frontend/src/routes/search/StackOverflow.svelte
@@ -20,7 +20,7 @@
</div>
</div>
</div>
- <div class="flex grow justify-between space-x-2">
+ <div class="flex grow justify-between space-x-2 text-sm">
<div>
<a href={answer.url}>{answer.url}</a>
</div>
@@ -28,7 +28,7 @@
</div>
</div>
<hr class="border-stone-700" />
- <div class="flex flex-col space-y-3">
+ <div class="flex flex-col space-y-3 text-sm">
{#each answer.body as part}
{#if part.type == 'code'}
<Code code={part.value} />