web search done right
[ranking] dampen scores of ngram fields if a larger ngram already matched for the document. for instance, if a document matches on 'example text' in the title, it should count 'example' and 'text' bm25 scores less for that particular document. this allows us to have better control over the scoring and not have text signals dominate the scores when a trigram matches a document
Mikkel Denkeree1db747f6c0
diff --git a/core/benches/search_preindexed.rs b/core/benches/search_preindexed.rsindex 7b177ccc..60329f15 100644--- a/core/benches/search_preindexed.rs+++ b/core/benches/search_preindexed.rs@@ -1,49 +1,170 @@-use std::path::Path;-+#[cfg(not(feature = "libtorch"))]use criterion::{criterion_group, criterion_main, Criterion};+#[cfg(not(feature = "libtorch"))]use optics::HostRankings;+#[cfg(not(feature = "libtorch"))]use stract::{+ bangs::Bangs,+ config::{+ ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, SnippetConfig, WidgetsConfig,+ },+ image_store::Image,index::Index,- ranking::inbound_similarity::InboundSimilarity,- searcher::{LocalSearcher, SearchQuery},+ inverted_index::RetrievedWebpage,+ ranking::{inbound_similarity::InboundSimilarity, pipeline::RetrievedWebpageRanking},+ searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery, ShardId},+ Result,};+#[cfg(not(feature = "libtorch"))]+struct Searcher(LocalSearcher<Index>);++#[cfg(not(feature = "libtorch"))]+impl stract::searcher::distributed::SearchClient for Searcher {+ async fn search_initial(+ &self,+ query: &SearchQuery,+ ) -> Vec<stract::searcher::InitialSearchResultShard> {+ let res = self.0.search_initial(query, true).unwrap();++ vec![stract::searcher::InitialSearchResultShard {+ local_result: res,+ shard: ShardId::new(0),+ }]+ }-const INDEX_PATH: &str = "data/index";-const CENTRALITY_PATH: &str = "data/centrality";+ async fn retrieve_webpages(+ &self,+ top_websites: &[(usize, stract::searcher::ScoredWebsitePointer)],+ query: &str,+ ) -> Vec<(usize, stract::ranking::pipeline::RetrievedWebpageRanking)> {+ let pointers = top_websites+ .iter()+ .map(|(_, p)| p.website.pointer.clone())+ .collect::<Vec<_>>();++ let res = self+ .0+ .retrieve_websites(&pointers, query)+ .unwrap()+ .into_iter()+ .zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))+ .map(|(ret, (i, ran))| (i, RetrievedWebpageRanking::new(ret, ran)))+ .collect::<Vec<_>>();++ res+ }++ async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {+ Ok(self.0.get_webpage(url))+ }++ async fn get_homepage_descriptions(+ &self,+ urls: &[url::Url],+ ) -> std::collections::HashMap<url::Url, String> {+ let mut res = std::collections::HashMap::new();++ for url in urls {+ if let Some(homepage) = self.0.get_homepage(url) {+ if let Some(desc) = homepage.description() {+ res.insert(url.clone(), desc.clone());+ }+ }+ }++ res+ }+ async fn get_entity_image(+ &self,+ _image_id: &str,+ _max_height: Option<u64>,+ _max_width: Option<u64>,+ ) -> Result<Option<Image>> {+ Ok(None)+ }+}++#[cfg(not(feature = "libtorch"))]macro_rules! bench {($query:tt, $searcher:ident, $c:ident) => {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .build()+ .unwrap();+let mut desc = "search '".to_string();desc.push_str($query);desc.push('\'');$c.bench_function(desc.as_str(), |b| {b.iter(|| {- $searcher- .search(&SearchQuery {- query: $query.to_string(),- host_rankings: Some(HostRankings {- liked: vec![- "docs.rs".to_string(),- "ycombinator.com".to_string(),- "nih.gov".to_string(),- ],- disliked: vec!["pinterest.com".to_string()],- blocked: vec![],- }),- ..Default::default()- })- .unwrap()+ runtime.block_on(async {+ $searcher+ .search(&SearchQuery {+ query: $query.to_string(),+ host_rankings: Some(HostRankings {+ liked: vec![+ "docs.rs".to_string(),+ "ycombinator.com".to_string(),+ "nih.gov".to_string(),+ ],+ disliked: vec!["pinterest.com".to_string()],+ blocked: vec![],+ }),+ ..Default::default()+ })+ .await+ .unwrap()+ })})});};}+#[cfg(not(feature = "libtorch"))]pub fn criterion_benchmark(c: &mut Criterion) {- let index = Index::open(INDEX_PATH).unwrap();+ let index = Index::open("data/index").unwrap();++ let collector_conf = CollectorConfig {+ ..Default::default()+ };++ let config = ApiConfig {+ queries_csv_path: "data/queries_us.csv".to_string(),+ host: "0.0.0.0:8000".parse().unwrap(),+ prometheus_host: "0.0.0.0:8001".parse().unwrap(),+ crossencoder_model_path: None,+ lambda_model_path: None,+ qa_model_path: None,+ spell_checker_path: Some("data/web_spell".to_string()),+ bangs_path: "data/bangs.json".to_string(),+ summarizer_path: "data/summarizer".to_string(),+ query_store_db_host: None,+ cluster_id: "api".to_string(),+ gossip_seed_nodes: None,+ gossip_addr: "0.0.0.0:8002".parse().unwrap(),+ collector: collector_conf.clone(),+ thresholds: ApiThresholds::default(),+ widgets: WidgetsConfig {+ thesaurus_paths: vec!["data/english-wordnet-2022-subset.ttl".to_string()],+ calculator_fetch_currencies_exchange: false,+ },+ correction_config: CorrectionConfig::default(),+ };+let mut searcher = LocalSearcher::new(index);searcher.set_inbound_similarity(- InboundSimilarity::open(Path::new(CENTRALITY_PATH).join("inbound_similarity")).unwrap(),+ InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),);+ searcher.set_collector_config(collector_conf);+ searcher.set_snippet_config(SnippetConfig {+ ..Default::default()+ });+ let bangs = Bangs::from_path(&config.bangs_path);++ let searcher = Searcher(searcher);++ let searcher: ApiSearcher<Searcher, LiveSearcher> =+ ApiSearcher::new(searcher, None, None, bangs, config);for _ in 0..1000 {bench!("the", searcher, c);@@ -57,5 +178,10 @@ pub fn criterion_benchmark(c: &mut Criterion) {}}+#[cfg(not(feature = "libtorch"))]criterion_group!(benches, criterion_benchmark);+#[cfg(not(feature = "libtorch"))]criterion_main!(benches);++#[cfg(feature = "libtorch")]+fn main() {}diff --git a/core/benches/search_preindexed_optic.rs b/core/benches/search_preindexed_optic.rsindex 05c57624..18cecb7a 100644--- a/core/benches/search_preindexed_optic.rs+++ b/core/benches/search_preindexed_optic.rs@@ -1,44 +1,171 @@+#[cfg(not(feature = "libtorch"))]use criterion::{criterion_group, criterion_main, Criterion};+#[cfg(not(feature = "libtorch"))]use optics::Optic;+#[cfg(not(feature = "libtorch"))]use stract::{+ bangs::Bangs,+ config::{ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, WidgetsConfig},+ image_store::Image,index::Index,- searcher::{LocalSearcher, SearchQuery},+ inverted_index::RetrievedWebpage,+ ranking::{inbound_similarity::InboundSimilarity, pipeline::RetrievedWebpageRanking},+ searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery, ShardId},+ Result,};+#[cfg(not(feature = "libtorch"))]+struct Searcher(LocalSearcher<Index>);-const INDEX_PATH: &str = "data/index";+#[cfg(not(feature = "libtorch"))]+impl stract::searcher::distributed::SearchClient for Searcher {+ async fn search_initial(+ &self,+ query: &SearchQuery,+ ) -> Vec<stract::searcher::InitialSearchResultShard> {+ let res = self.0.search_initial(query, true).unwrap();+ vec![stract::searcher::InitialSearchResultShard {+ local_result: res,+ shard: ShardId::new(0),+ }]+ }++ async fn retrieve_webpages(+ &self,+ top_websites: &[(usize, stract::searcher::ScoredWebsitePointer)],+ query: &str,+ ) -> Vec<(usize, stract::ranking::pipeline::RetrievedWebpageRanking)> {+ let pointers = top_websites+ .iter()+ .map(|(_, p)| p.website.pointer.clone())+ .collect::<Vec<_>>();++ let res = self+ .0+ .retrieve_websites(&pointers, query)+ .unwrap()+ .into_iter()+ .zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))+ .map(|(ret, (i, ran))| (i, RetrievedWebpageRanking::new(ret, ran)))+ .collect::<Vec<_>>();++ res+ }++ async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {+ Ok(self.0.get_webpage(url))+ }++ async fn get_homepage_descriptions(+ &self,+ urls: &[url::Url],+ ) -> std::collections::HashMap<url::Url, String> {+ let mut res = std::collections::HashMap::new();++ for url in urls {+ if let Some(homepage) = self.0.get_homepage(url) {+ if let Some(desc) = homepage.description() {+ res.insert(url.clone(), desc.clone());+ }+ }+ }++ res+ }++ async fn get_entity_image(+ &self,+ _image_id: &str,+ _max_height: Option<u64>,+ _max_width: Option<u64>,+ ) -> Result<Option<Image>> {+ Ok(None)+ }+}++#[cfg(not(feature = "libtorch"))]macro_rules! bench {($query:tt, $searcher:ident, $optic:ident, $c:ident) => {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .build()+ .unwrap();+let mut desc = "search '".to_string();desc.push_str($query);desc.push('\'');desc.push_str(" with optic");$c.bench_function(desc.as_str(), |b| {b.iter(|| {- $searcher- .search(&SearchQuery {- query: $query.to_string(),- optic: Some(Optic::parse($optic).unwrap()),- ..Default::default()- })- .unwrap()+ runtime.block_on(async {+ $searcher+ .search(&SearchQuery {+ query: $query.to_string(),+ optic: Some(Optic::parse($optic).unwrap()),+ ..Default::default()+ })+ .await+ .unwrap()+ })})});};}+#[cfg(not(feature = "libtorch"))]pub fn criterion_benchmark(c: &mut Criterion) {- let index = Index::open(INDEX_PATH).unwrap();- let searcher = LocalSearcher::new(index);+ let index = Index::open("data/index").unwrap();let optic = include_str!("../../optics/testcases/samples/discussions.optic");- for _ in 0..10_000 {+ let config = ApiConfig {+ queries_csv_path: "data/queries_us.csv".to_string(),+ host: "0.0.0.0:8000".parse().unwrap(),+ prometheus_host: "0.0.0.0:8001".parse().unwrap(),+ crossencoder_model_path: None,+ lambda_model_path: None,+ qa_model_path: None,+ spell_checker_path: Some("data/web_spell".to_string()),+ bangs_path: "data/bangs.json".to_string(),+ summarizer_path: "data/summarizer".to_string(),+ query_store_db_host: None,+ cluster_id: "api".to_string(),+ gossip_seed_nodes: None,+ gossip_addr: "0.0.0.0:8002".parse().unwrap(),+ collector: CollectorConfig::default(),+ thresholds: ApiThresholds::default(),+ widgets: WidgetsConfig {+ thesaurus_paths: vec!["data/english-wordnet-2022-subset.ttl".to_string()],+ calculator_fetch_currencies_exchange: false,+ },+ correction_config: CorrectionConfig::default(),+ };++ let mut searcher = LocalSearcher::new(index);+ searcher.set_inbound_similarity(+ InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),+ );+ let bangs = Bangs::from_path(&config.bangs_path);++ let searcher = Searcher(searcher);++ let searcher: ApiSearcher<Searcher, LiveSearcher> =+ ApiSearcher::new(searcher, None, None, bangs, config);++ for _ in 0..1000 {bench!("the", searcher, optic, c);bench!("dtu", searcher, optic, c);bench!("the best", searcher, optic, c);bench!("the circle of life", searcher, optic, c);+ bench!("what", searcher, optic, c);+ bench!("a", searcher, optic, c);+ bench!("sun", searcher, optic, c);+ bench!("what a sun", searcher, optic, c);}}+#[cfg(not(feature = "libtorch"))]criterion_group!(benches, criterion_benchmark);+#[cfg(not(feature = "libtorch"))]criterion_main!(benches);++#[cfg(feature = "libtorch")]+fn main() {}diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 7404d341..9b49db18 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -32,7 +32,7 @@ use crate::{improvement::{store_improvements_loop, ImprovementEvent},leaky_queue::LeakyQueue,ranking::models::lambdamart::LambdaMART,- searcher::api::ApiSearcher,+ searcher::{api::ApiSearcher, live::LiveSearcher, DistributedSearcher},};#[cfg(feature = "libtorch")]@@ -76,7 +76,7 @@ pub struct Counters {#[cfg(feature = "libtorch")]pub struct State {pub config: ApiConfig,- pub searcher: ApiSearcher,+ pub searcher: ApiSearcher<DistributedSearcher, LiveSearcher>,pub remote_webgraph: RemoteWebgraph,pub autosuggest: Autosuggest,pub counters: Counters,@@ -88,7 +88,7 @@ pub struct State {#[cfg(not(feature = "libtorch"))]pub struct State {pub config: ApiConfig,- pub searcher: ApiSearcher,+ pub searcher: ApiSearcher<DistributedSearcher, LiveSearcher>,pub remote_webgraph: RemoteWebgraph,pub autosuggest: Autosuggest,pub counters: Counters,@@ -135,6 +135,9 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {);let remote_webgraph = RemoteWebgraph::new(cluster.clone());+ let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster));+ let live_searcher = LiveSearcher::new(Arc::clone(&cluster));+#[cfg(feature = "libtorch")]let state = {let mut cross_encoder = None;@@ -149,7 +152,8 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {};let searcher = ApiSearcher::new(- cluster.clone(),+ dist_searcher,+ Some(live_searcher),cross_encoder,lambda_model,qa_model,@@ -171,7 +175,13 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {#[cfg(not(feature = "libtorch"))]let state = {- let searcher = ApiSearcher::new(cluster.clone(), lambda_model, bangs, config.clone());+ let searcher = ApiSearcher::new(+ dist_searcher,+ Some(live_searcher),+ lambda_model,+ bangs,+ config.clone(),+ );Arc::new(State {config: config.clone(),diff --git a/core/src/collector.rs b/core/src/collector.rsindex b3ec02c1..c32375bf 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -36,7 +36,7 @@ use crate::{pub type MainCollector = TweakedScoreTopCollector<InitialScoreTweaker>;-#[derive(Clone)]+#[derive(Clone, Debug)]pub struct MaxDocsConsidered {pub total_docs: usize,pub segments: usize,@@ -168,6 +168,7 @@ impl TopSegmentCollector {false}}+fn collect(&mut self, doc: DocId, score: Score) {if self.is_done() {return;diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex b44829b7..0d66c3a9 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -114,14 +114,6 @@ impl WebgraphServer {}}-pub struct SearchServer;--impl SearchServer {- pub fn build_spell_dictionary() -> bool {- true- }-}-pub struct SearchQuery;impl SearchQuery {@@ -161,3 +153,11 @@ impl Correction {5.77}}++pub struct Widgets;++impl Widgets {+ pub fn calculator_fetch_currencies_exchange() -> bool {+ true+ }+}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 659b8fae..8e365610 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -371,6 +371,9 @@ pub struct WebgraphServerConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct WidgetsConfig {pub thesaurus_paths: Vec<String>,++ #[serde(default = "defaults::Widgets::calculator_fetch_currencies_exchange")]+ pub calculator_fetch_currencies_exchange: bool,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 68c3a840..f8efd44e 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -266,6 +266,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {info!("processing {}", name);let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();+ index.prepare_writer().unwrap();let warc_files = download_all_warc_files(&job.warc_paths, &job.source_config);pin!(warc_files);diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex c0e94793..e4607fe9 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -32,6 +32,7 @@ use crate::distributed::sonic;use crate::distributed::sonic::service::Message;use crate::ranking::inbound_similarity::InboundSimilarity;use crate::searcher::DistributedSearcher;+use crate::searcher::SearchClient;use crate::similar_hosts::SimilarHostsFinder;use crate::sonic_service;use crate::webgraph::Compression;diff --git a/core/src/enum_map.rs b/core/src/enum_map.rsindex e325152b..ed3cf07a 100644--- a/core/src/enum_map.rs+++ b/core/src/enum_map.rs@@ -68,6 +68,10 @@ where}}+ pub fn contains_key(&self, key: K) -> bool {+ self.get(key).is_some()+ }+pub fn values_mut(&mut self) -> impl Iterator<Item = &mut V> {self.inner.iter_mut().filter_map(|value| value.as_mut())}diff --git a/core/src/index.rs b/core/src/index.rsindex 7a7c5129..a9062c69 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -75,7 +75,11 @@ impl Index {#[cfg(test)]pub fn temporary() -> Result<Self> {let path = crate::gen_temp_path();- Self::open(path)+ let mut s = Self::open(path)?;++ s.prepare_writer()?;++ Ok(s)}pub fn insert(&self, webpage: Webpage) -> Result<()> {@@ -145,6 +149,10 @@ impl Index {Self::open(&self.path).expect("failed to open index")}++ pub(crate) fn prepare_writer(&mut self) -> Result<()> {+ self.inverted_index.prepare_writer()+ }}#[cfg(test)]diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex ff47af9c..907defca 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -158,7 +158,7 @@ struct SegmentMergeCandidate {pub struct InvertedIndex {pub path: String,tantivy_index: tantivy::Index,- writer: IndexWriter,+ writer: Option<IndexWriter>,reader: IndexReader,schema: Arc<Schema>,snippet_config: SnippetConfig,@@ -220,15 +220,10 @@ impl InvertedIndex {.tokenizers().register(tokenizer.as_str(), tokenizer);- let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;-- let merge_policy = NoMergePolicy;- writer.set_merge_policy(Box::new(merge_policy));-let reader: IndexReader = tantivy_index.reader_builder().try_into()?;Ok(InvertedIndex {- writer,+ writer: None,reader,schema: Arc::new(schema),path: path.as_ref().to_str().unwrap().to_string(),@@ -237,13 +232,29 @@ impl InvertedIndex {})}+ pub fn prepare_writer(&mut self) -> Result<()> {+ let writer = self+ .tantivy_index+ .writer_with_num_threads(1, 1_000_000_000)?;++ let merge_policy = NoMergePolicy;+ writer.set_merge_policy(Box::new(merge_policy));++ self.writer = Some(writer);++ Ok(())+ }+pub fn set_snippet_config(&mut self, config: SnippetConfig) {self.snippet_config = config;}pub fn set_auto_merge_policy(&mut self) {let merge_policy = tantivy::merge_policy::LogMergePolicy::default();- self.writer.set_merge_policy(Box::new(merge_policy));+ self.writer+ .as_mut()+ .expect("writer has not been prepared")+ .set_merge_policy(Box::new(merge_policy));}pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {@@ -257,24 +268,36 @@ impl InvertedIndex {#[cfg(test)]pub fn temporary() -> Result<Self> {let path = crate::gen_temp_path();- Self::open(path)+ let mut s = Self::open(path)?;++ s.prepare_writer()?;++ Ok(s)}pub fn insert(&self, webpage: Webpage) -> Result<()> {self.writer+ .as_ref()+ .expect("writer has not been prepared").add_document(webpage.into_tantivy(&self.schema)?)?;Ok(())}pub fn commit(&mut self) -> Result<()> {- self.writer.commit()?;+ self.writer+ .as_mut()+ .expect("writer has not been prepared")+ .commit()?;self.reader.reload()?;Ok(())}fn delete(&self, query: Box<dyn tantivy::query::Query>) -> Result<()> {- self.writer.delete_query(query)?;+ self.writer+ .as_ref()+ .expect("writer has not been prepared")+ .delete_query(query)?;Ok(())}@@ -433,7 +456,12 @@ impl InvertedIndex {.into_iter().collect();- merge_tantivy_segments(&mut self.writer, segments, base_path, max_num_segments)?;+ merge_tantivy_segments(+ self.writer.as_mut().expect("writer has not been prepared"),+ segments,+ base_path,+ max_num_segments,+ )?;Ok(())}@@ -466,11 +494,20 @@ impl InvertedIndex {let other_path = other.path.clone();let other_path = Path::new(other_path.as_str());- other.writer.wait_merging_threads().unwrap();+ other+ .writer+ .take()+ .expect("writer has not been prepared")+ .wait_merging_threads()+ .unwrap();let path = self.path.clone();let self_path = Path::new(path.as_str());- self.writer.wait_merging_threads().unwrap();+ self.writer+ .take()+ .expect("writer has not been prepared")+ .wait_merging_threads()+ .unwrap();let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();@@ -506,8 +543,12 @@ impl InvertedIndex {Self::open(path).expect("failed to open index")}- pub fn stop(self) {- self.writer.wait_merging_threads().unwrap()+ pub fn stop(mut self) {+ self.writer+ .take()+ .expect("writer has not been prepared")+ .wait_merging_threads()+ .unwrap()}pub fn schema(&self) -> Arc<Schema> {@@ -1169,6 +1210,7 @@ mod tests {.expect("failed to insert webpage");let mut index = index1.merge(index2);+ index.prepare_writer().unwrap();index.commit().unwrap();let ctx = index.local_search_ctx();diff --git a/core/src/lib.rs b/core/src/lib.rsindex c6cba649..860068af 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -27,7 +27,7 @@ use std::path::PathBuf;use thiserror::Error;pub mod entrypoint;-mod inverted_index;+pub mod inverted_index;pub mod mapreduce;@@ -35,7 +35,7 @@ pub mod mapreduce;pub mod alice;mod api;mod autosuggest;-mod bangs;+pub mod bangs;mod bloom;mod collector;pub mod config;@@ -49,7 +49,7 @@ mod fastfield_reader;pub mod feed;mod human_website_annotations;pub mod hyperloglog;-mod image_store;+pub mod image_store;mod improvement;pub mod index;mod intmap;@@ -117,7 +117,7 @@ pub enum Error {UnknownRobotsMetaTag,}-pub(crate) type Result<T, E = anyhow::Error> = std::result::Result<T, E>;+pub type Result<T, E = anyhow::Error> = std::result::Result<T, E>;// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445pub fn gen_temp_path() -> PathBuf {diff --git a/core/src/live_index.rs b/core/src/live_index.rsindex 53427be2..e659dc8e 100644--- a/core/src/live_index.rs+++ b/core/src/live_index.rs@@ -228,6 +228,7 @@ pub struct Index {impl Index {pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {let mut search_index = crate::index::Index::open(path.as_ref().join("index"))?;+ search_index.prepare_writer()?;search_index.set_auto_merge_policy();let search_index = Arc::new(RwLock::new(search_index));diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 1e4a5ee1..26d813d1 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -315,6 +315,7 @@ impl Signal {fn host_id(&self, aggregator: &SignalAggregator, doc: DocId) -> Option<NodeID> {aggregator.segment_reader.as_ref().and_then(|reader| {let node_id: Option<u64> = reader+ .borrow_mut().fastfield_reader.get_field_reader(&FastField::HostNodeID).get(&doc)@@ -333,6 +334,7 @@ impl Signal {aggregator.segment_reader.as_ref().and_then(|reader| {self.as_fastfield().map(|fast_field| {reader+ .borrow_mut().fastfield_reader.get_field_reader(&fast_field).get(&doc)@@ -340,11 +342,7 @@ impl Signal {})}- fn compute(- self,- signal_aggregator: &mut SignalAggregator,- doc: DocId,- ) -> Option<ComputedSignal> {+ fn compute(self, signal_aggregator: &SignalAggregator, doc: DocId) -> Option<ComputedSignal> {let coefficient = signal_aggregator.coefficient(&self);if coefficient == 0.0 {return None;@@ -460,8 +458,9 @@ impl Signal {| Signal::Bm25DomainNameIfHomepageNoTokenizer| Signal::Bm25DomainIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage- | Signal::Bm25BacklinkText => signal_aggregator.segment_reader.as_mut().map(|reader| {+ | Signal::Bm25BacklinkText => signal_aggregator.segment_reader.as_ref().map(|reader| {reader+ .borrow_mut().text_fields.get_mut(self.as_textfield().unwrap()).map(|field| bm25(field, doc))@@ -723,7 +722,7 @@ struct QueryData {pub struct SignalAggregator {query_data: Option<QueryData>,query_signal_coefficients: Option<SignalCoefficient>,- segment_reader: Option<SegmentReader>,+ segment_reader: Option<RefCell<SegmentReader>>,inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,@@ -731,6 +730,7 @@ pub struct SignalAggregator {region_count: Option<Arc<RegionCount>>,current_timestamp: Option<usize>,linear_regression: Option<Arc<LinearRegression>>,+ order: SignalOrder,}impl Clone for SignalAggregator {@@ -756,6 +756,7 @@ impl Clone for SignalAggregator {region_count: self.region_count.clone(),current_timestamp: self.current_timestamp,linear_regression: self.linear_regression.clone(),+ order: self.order.clone(),}}}@@ -813,8 +814,10 @@ impl SignalAggregator {current_timestamp: None,linear_regression: None,query_data: query,+ order: SignalOrder::empty(),};+ s.order = SignalOrder::new(&s);s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);s@@ -923,13 +926,13 @@ impl SignalAggregator {let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);- self.segment_reader = Some(SegmentReader {+ self.segment_reader = Some(RefCell::new(SegmentReader {text_fields,fastfield_reader: fastfield_segment_reader,optic_boosts: OpticBoosts {rules: optic_rule_boosts,},- });+ }));Ok(())}@@ -977,21 +980,16 @@ impl SignalAggregator {/// scores calculated for their text related signals. The wrong ranking will most likely/// be returned./// This function also assumes that the segment reader has been set.- pub fn compute_signals(- &mut self,- doc: DocId,- ) -> impl Iterator<Item = Option<ComputedSignal>> + '_ {- ALL_SIGNALS- .into_iter()- .map(move |signal| signal.compute(self, doc))+ pub fn compute_signals(&self, doc: DocId) -> impl Iterator<Item = Option<ComputedSignal>> + '_ {+ self.order.compute(doc, self)}pub fn boosts(&mut self, doc: DocId) -> Option<f64> {- self.segment_reader.as_mut().map(|segment_reader| {+ self.segment_reader.as_ref().map(|segment_reader| {let mut downrank = 0.0;let mut boost = 0.0;- for rule in &mut segment_reader.optic_boosts.rules {+ for rule in &mut segment_reader.borrow_mut().optic_boosts.rules {if rule.docset.doc() > doc {continue;}@@ -1047,3 +1045,106 @@ pub struct SignalScore {pub coefficient: f64,pub value: f64,}++#[derive(Clone)]+pub struct SignalOrder {+ text_signals: EnumMap<TextField, NGramSignalOrder>,+ other_signals: Vec<Signal>,+}++impl SignalOrder {+ pub fn empty() -> Self {+ Self {+ text_signals: EnumMap::new(),+ other_signals: Vec::new(),+ }+ }++ pub fn new(signal_aggregator: &SignalAggregator) -> Self {+ let mut text_signals = EnumMap::new();+ let mut other_signals = Vec::new();++ for signal in ALL_SIGNALS {+ if signal_aggregator.coefficient(&signal) == 0.0 {+ continue;+ }++ if let Some(text_field) = signal.as_textfield() {+ let mono = text_field.monogram_field();++ if !text_signals.contains_key(mono) {+ text_signals.insert(mono, NGramSignalOrder::default());+ }++ let ngram = text_field.ngram_size();+ text_signals.get_mut(mono).unwrap().push(signal, ngram);+ } else {+ other_signals.push(signal);+ }+ }++ Self {+ text_signals,+ other_signals,+ }+ }++ fn compute<'a>(+ &'a self,+ doc: DocId,+ signal_aggregator: &'a SignalAggregator,+ ) -> impl Iterator<Item = Option<ComputedSignal>> + 'a {+ self.text_signals+ .values()+ .flat_map(move |ngram| ngram.compute(doc, signal_aggregator))+ .map(Some)+ .chain(+ self.other_signals+ .iter()+ .map(move |signal| signal.compute(signal_aggregator, doc)),+ )+ }+}++/// If an ngram of size n matches the query for a given document in a given field,+/// the score of all ngrams where n' < n is dampened by NGRAM_DAMPENING.+///+/// A dampening factor of 0.0 means that we ignore all ngrams where n' < n. A dampening factor of 1.0+/// does not dampen any ngrams.+const NGRAM_DAMPENING: f64 = 0.4;++#[derive(Debug, Default, Clone)]+pub struct NGramSignalOrder {+ /// ordered by descending ngram size. e.g. [title_bm25_trigram, title_bm25_bigram, title_bm25]+ signals: Vec<(usize, Signal)>,+}++impl NGramSignalOrder {+ fn push(&mut self, signal: Signal, ngram: usize) {+ self.signals.push((ngram, signal));+ self.signals.sort_unstable_by(|(a, _), (b, _)| b.cmp(a));+ }++ fn compute<'a>(+ &'a self,+ doc: DocId,+ signal_aggregator: &'a SignalAggregator,+ ) -> impl Iterator<Item = ComputedSignal> + 'a {+ let mut hits = 0.0;++ self.signals+ .iter()+ .map(|(_, s)| s)+ .filter_map(move |signal| {+ signal.compute(signal_aggregator, doc).map(|mut c| {+ c.score.coefficient *= NGRAM_DAMPENING.powf(hits);++ if c.score.value > 0.0 {+ hits += 1.0;+ }++ c+ })+ })+ }+}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 8ca974f7..d2035d32 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -70,6 +70,80 @@ impl From<TextField> for usize {}impl TextField {+ pub fn ngram_size(&self) -> usize {+ match self {+ TextField::Title => 1,+ TextField::CleanBody => 1,+ TextField::StemmedTitle => 1,+ TextField::StemmedCleanBody => 1,+ TextField::AllBody => 1,+ TextField::Url => 1,+ TextField::UrlNoTokenizer => 1,+ TextField::UrlForSiteOperator => 1,+ TextField::SiteWithout => 1,+ TextField::Domain => 1,+ TextField::SiteNoTokenizer => 1,+ TextField::DomainNoTokenizer => 1,+ TextField::DomainNameNoTokenizer => 1,+ TextField::SiteIfHomepageNoTokenizer => 1,+ TextField::DomainIfHomepage => 1,+ TextField::DomainNameIfHomepageNoTokenizer => 1,+ TextField::DomainIfHomepageNoTokenizer => 1,+ TextField::TitleIfHomepage => 1,+ TextField::BacklinkText => 1,+ TextField::Description => 1,+ TextField::DmozDescription => 1,+ TextField::SchemaOrgJson => 1,+ TextField::FlattenedSchemaOrgJson => 1,+ TextField::CleanBodyBigrams => 2,+ TextField::TitleBigrams => 2,+ TextField::CleanBodyTrigrams => 3,+ TextField::TitleTrigrams => 3,+ TextField::MicroformatTags => 1,+ TextField::SafetyClassification => 1,+ TextField::InsertionTimestamp => 1,+ TextField::RecipeFirstIngredientTagId => 1,+ }+ }++ pub fn monogram_field(&self) -> TextField {+ match self {+ TextField::Title => TextField::Title,+ TextField::CleanBody => TextField::CleanBody,+ TextField::StemmedTitle => TextField::StemmedTitle,+ TextField::StemmedCleanBody => TextField::StemmedCleanBody,+ TextField::AllBody => TextField::AllBody,+ TextField::Url => TextField::Url,+ TextField::UrlNoTokenizer => TextField::UrlNoTokenizer,+ TextField::UrlForSiteOperator => TextField::UrlForSiteOperator,+ TextField::SiteWithout => TextField::SiteWithout,+ TextField::Domain => TextField::Domain,+ TextField::SiteNoTokenizer => TextField::SiteNoTokenizer,+ TextField::DomainNoTokenizer => TextField::DomainNoTokenizer,+ TextField::DomainNameNoTokenizer => TextField::DomainNameNoTokenizer,+ TextField::SiteIfHomepageNoTokenizer => TextField::SiteIfHomepageNoTokenizer,+ TextField::DomainIfHomepage => TextField::DomainIfHomepage,+ TextField::DomainNameIfHomepageNoTokenizer => {+ TextField::DomainNameIfHomepageNoTokenizer+ }+ TextField::DomainIfHomepageNoTokenizer => TextField::DomainIfHomepageNoTokenizer,+ TextField::TitleIfHomepage => TextField::TitleIfHomepage,+ TextField::BacklinkText => TextField::BacklinkText,+ TextField::Description => TextField::Description,+ TextField::DmozDescription => TextField::DmozDescription,+ TextField::SchemaOrgJson => TextField::SchemaOrgJson,+ TextField::FlattenedSchemaOrgJson => TextField::FlattenedSchemaOrgJson,+ TextField::CleanBodyBigrams => TextField::CleanBody,+ TextField::TitleBigrams => TextField::Title,+ TextField::CleanBodyTrigrams => TextField::CleanBody,+ TextField::TitleTrigrams => TextField::Title,+ TextField::MicroformatTags => TextField::MicroformatTags,+ TextField::SafetyClassification => TextField::SafetyClassification,+ TextField::InsertionTimestamp => TextField::InsertionTimestamp,+ TextField::RecipeFirstIngredientTagId => TextField::RecipeFirstIngredientTagId,+ }+ }+pub fn query_tokenizer(&self) -> Tokenizer {match self {TextField::TitleBigrams => Tokenizer::default(),diff --git a/core/src/searcher/api/mod.rs b/core/src/searcher/api/mod.rsindex 7f42e962..4f922093 100644--- a/core/src/searcher/api/mod.rs+++ b/core/src/searcher/api/mod.rs@@ -38,7 +38,6 @@ use crate::widgets::Widgets;use crate::{bangs::Bangs,collector::BucketCollector,- distributed::cluster::Cluster,ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},};#[cfg(feature = "libtorch")]@@ -48,8 +47,7 @@ use crate::{query, Result};use self::sidebar::SidebarManager;use self::widget::WidgetManager;-use super::live::LiveSearcher;-use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};+use super::{distributed, live, SearchQuery, SearchResult, WebsitesResult};#[derive(Clone)]pub enum ScoredWebsitePointer {@@ -138,41 +136,46 @@ pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[Scored}#[cfg(feature = "libtorch")]-pub struct ApiSearcher {- distributed_searcher: Arc<DistributedSearcher>,- sidebar_manager: SidebarManager,- live_searcher: LiveSearcher,+pub struct ApiSearcher<S, L> {+ distributed_searcher: Arc<S>,+ sidebar_manager: SidebarManager<S>,+ live_searcher: Option<L>,cross_encoder: Option<Arc<CrossEncoderModel>>,lambda_model: Option<Arc<LambdaMART>>,bangs: Bangs,collector_config: CollectorConfig,- widget_manager: WidgetManager,+ widget_manager: WidgetManager<S>,spell_checker: Option<SpellChecker>,}#[cfg(not(feature = "libtorch"))]-pub struct ApiSearcher {- distributed_searcher: Arc<DistributedSearcher>,- live_searcher: LiveSearcher,- sidebar_manager: SidebarManager,+pub struct ApiSearcher<S, L> {+ distributed_searcher: Arc<S>,+ live_searcher: Option<L>,+ sidebar_manager: SidebarManager<S>,lambda_model: Option<Arc<LambdaMART>>,bangs: Bangs,collector_config: CollectorConfig,- widget_manager: WidgetManager,+ widget_manager: WidgetManager<S>,spell_checker: Option<SpellChecker>,}-impl ApiSearcher {+impl<S, L> ApiSearcher<S, L>+where+ S: distributed::SearchClient,+ L: live::SearchClient,+{#[cfg(feature = "libtorch")]pub fn new(- cluster: Arc<Cluster>,+ dist_searcher: S,+ live_searcher: Option<L>,cross_encoder: Option<CrossEncoderModel>,lambda_model: Option<LambdaMART>,qa_model: Option<QaModel>,bangs: Bangs,config: ApiConfig,) -> Self {- let dist_searcher = Arc::new(DistributedSearcher::new(Arc::clone(&cluster)));+ let dist_searcher = Arc::new(dist_searcher);let sidebar_manager =SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());@@ -191,7 +194,7 @@ impl ApiSearcher {Self {distributed_searcher: dist_searcher,sidebar_manager,- live_searcher: LiveSearcher::new(Arc::clone(&cluster)),+ live_searcher,cross_encoder: cross_encoder.map(Arc::new),lambda_model,bangs,@@ -205,12 +208,13 @@ impl ApiSearcher {#[cfg(not(feature = "libtorch"))]pub fn new(- cluster: Arc<Cluster>,+ dist_searcher: S,+ live_searcher: Option<L>,lambda_model: Option<LambdaMART>,bangs: Bangs,config: ApiConfig,) -> Self {- let dist_searcher = Arc::new(DistributedSearcher::new(Arc::clone(&cluster)));+ let dist_searcher = Arc::new(dist_searcher);let sidebar_manager =SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());let lambda_model = lambda_model.map(Arc::new);@@ -226,7 +230,7 @@ impl ApiSearcher {Self {distributed_searcher: dist_searcher,sidebar_manager,- live_searcher: LiveSearcher::new(Arc::clone(&cluster)),+ live_searcher,lambda_model,bangs,collector_config: config.collector,@@ -306,7 +310,7 @@ impl ApiSearcher {let (retrieved_normal, retrieved_live) = tokio::join!(self.distributed_searcher.retrieve_webpages(&normal, query),- self.live_searcher.retrieve_webpages(&live, query),+ self.retrieve_webpages_from_live(&live, query),);let mut retrieved_webpages: Vec<_> =@@ -319,6 +323,27 @@ impl ApiSearcher {.collect::<Vec<_>>()}+ async fn search_initial_from_live(+ &self,+ query: &SearchQuery,+ ) -> Option<Vec<live::InitialSearchResultSplit>> {+ match &self.live_searcher {+ Some(searcher) => Some(searcher.search_initial(query).await),+ None => None,+ }+ }++ async fn retrieve_webpages_from_live(+ &self,+ pointers: &[(usize, live::ScoredWebsitePointer)],+ query: &str,+ ) -> Vec<(usize, RetrievedWebpageRanking)> {+ match &self.live_searcher {+ Some(searcher) => searcher.retrieve_webpages(pointers, query).await,+ None => vec![],+ }+ }+async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {let start = Instant::now();@@ -336,10 +361,9 @@ impl ApiSearcher {self.collector_config.clone(),20,);-let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(self.distributed_searcher.search_initial(&search_query),- self.live_searcher.search_initial(&search_query),+ self.search_initial_from_live(&search_query),self.widget_manager.widget(query),self.widget_manager.discussions(query),self.sidebar_manager.stackoverflow(query),@@ -368,7 +392,7 @@ impl ApiSearcher {let (top_websites, has_more_results) = combine_results(self.collector_config.clone(),initial_results,- live_results,+ live_results.unwrap_or_default(),recall_pipeline,);diff --git a/core/src/searcher/api/sidebar.rs b/core/src/searcher/api/sidebar.rsindex 205bfac4..bc138c76 100644--- a/core/src/searcher/api/sidebar.rs+++ b/core/src/searcher/api/sidebar.rs@@ -22,19 +22,19 @@ use url::Url;use crate::{search_prettifier::{create_stackoverflow_sidebar, DisplayedEntity, DisplayedSidebar},- searcher::{distributed, DistributedSearcher, SearchQuery},+ searcher::{distributed, SearchQuery},};-pub struct SidebarManager {- distributed_searcher: Arc<DistributedSearcher>,+pub struct SidebarManager<S> {+ distributed_searcher: Arc<S>,thresholds: ApiThresholds,}-impl SidebarManager {- pub fn new(- distributed_searcher: Arc<DistributedSearcher>,- thresholds: ApiThresholds,- ) -> SidebarManager {+impl<S> SidebarManager<S>+where+ S: distributed::SearchClient,+{+ pub fn new(distributed_searcher: Arc<S>, thresholds: ApiThresholds) -> SidebarManager<S> {Self {distributed_searcher,thresholds,diff --git a/core/src/searcher/api/widget.rs b/core/src/searcher/api/widget.rsindex 20664b92..53d312c5 100644--- a/core/src/searcher/api/widget.rs+++ b/core/src/searcher/api/widget.rs@@ -27,7 +27,7 @@ use crate::ranking::pipeline::{AsRankingWebsite, RankingPipeline};use crate::search_prettifier::DisplayedAnswer;use crate::search_prettifier::DisplayedWebpage;use crate::searcher::api::{add_ranking_signals, combine_results, ScoredWebsitePointer};-use crate::searcher::{DistributedSearcher, SearchQuery};+use crate::searcher::{distributed, SearchQuery};use crate::widgets::Widget;use crate::{ceil_char_boundary, floor_char_boundary, query, Result};use crate::{@@ -36,9 +36,9 @@ use crate::{};#[cfg(feature = "libtorch")]-pub struct WidgetManager {+pub struct WidgetManager<S> {widgets: Widgets,- distributed_searcher: Arc<DistributedSearcher>,+ distributed_searcher: Arc<S>,thresholds: ApiThresholds,collector_config: CollectorConfig,lambda_model: Option<Arc<LambdaMART>>,@@ -46,24 +46,27 @@ pub struct WidgetManager {}#[cfg(not(feature = "libtorch"))]-pub struct WidgetManager {+pub struct WidgetManager<S> {widgets: Widgets,- distributed_searcher: Arc<DistributedSearcher>,+ distributed_searcher: Arc<S>,thresholds: ApiThresholds,collector_config: CollectorConfig,lambda_model: Option<Arc<LambdaMART>>,}-impl WidgetManager {+impl<S> WidgetManager<S>+where+ S: distributed::SearchClient,+{#[cfg(feature = "libtorch")]pub fn new(widgets: Widgets,- distributed_searcher: Arc<DistributedSearcher>,+ distributed_searcher: Arc<S>,thresholds: ApiThresholds,collector_config: CollectorConfig,lambda_model: Option<Arc<LambdaMART>>,qa_model: Option<Arc<QaModel>>,- ) -> WidgetManager {+ ) -> WidgetManager<S> {Self {widgets,distributed_searcher,@@ -77,11 +80,11 @@ impl WidgetManager {#[cfg(not(feature = "libtorch"))]pub fn new(widgets: Widgets,- distributed_searcher: Arc<DistributedSearcher>,+ distributed_searcher: Arc<S>,thresholds: ApiThresholds,collector_config: CollectorConfig,lambda_model: Option<Arc<LambdaMART>>,- ) -> WidgetManager {+ ) -> WidgetManager<S> {Self {widgets,distributed_searcher,diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 1699a6f4..205bab7a 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -36,6 +36,7 @@ use fnv::FnvHashMap;use futures::future::join_all;use itertools::Itertools;use serde::{Deserialize, Serialize};+use std::future::Future;use thiserror::Error;use url::Url;@@ -62,6 +63,12 @@ pub struct ScoredWebsitePointer {#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]pub struct ShardId(u64);+impl ShardId {+ pub fn new(id: u64) -> Self {+ Self(id)+ }+}+impl ShardIdentifier for ShardId {}#[derive(Debug)]@@ -99,33 +106,6 @@ impl DistributedSearcher {ShardedClient::new(shard_clients)}- pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {- let client = self.client().await;- let mut results = Vec::new();-- if let Ok(res) = client- .send(- &search_server::Search {- query: query.clone(),- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- {- for (shard_id, mut res) in res {- if let Some(Some(res)) = res.pop() {- results.push(InitialSearchResultShard {- local_result: res,- shard: shard_id,- });- }- }- }-- results- }-async fn retrieve_webpages_from_shard(&self,shard: ShardId,@@ -157,8 +137,37 @@ impl DistributedSearcher {_ => vec![],}}+}++impl SearchClient for DistributedSearcher {+ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {+ let client = self.client().await;+ let mut results = Vec::new();++ if let Ok(res) = client+ .send(+ &search_server::Search {+ query: query.clone(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ {+ for (shard_id, mut res) in res {+ if let Some(Some(res)) = res.pop() {+ results.push(InitialSearchResultShard {+ local_result: res,+ shard: shard_id,+ });+ }+ }+ }++ results+ }- pub async fn retrieve_webpages(+ async fn retrieve_webpages(&self,top_websites: &[(usize, ScoredWebsitePointer)],query: &str,@@ -196,7 +205,7 @@ impl DistributedSearcher {retrieved_webpages}- pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {+ async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {let client = self.client().await;let res = client@@ -217,7 +226,7 @@ impl DistributedSearcher {}}- pub async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {+ async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {let client = self.client().await;let res = client@@ -236,7 +245,7 @@ impl DistributedSearcher {}}- pub async fn get_entity_image(+ async fn get_entity_image(&self,image_id: &str,max_height: Option<u64>,@@ -262,3 +271,29 @@ impl DistributedSearcher {Ok(res.pop().flatten())}}++pub trait SearchClient {+ fn search_initial(+ &self,+ query: &SearchQuery,+ ) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;+ fn retrieve_webpages(+ &self,+ top_websites: &[(usize, ScoredWebsitePointer)],+ query: &str,+ ) -> impl Future<Output = Vec<(usize, RetrievedWebpageRanking)>> + Send;+ fn get_webpage(+ &self,+ url: &str,+ ) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;+ fn get_homepage_descriptions(+ &self,+ urls: &[Url],+ ) -> impl Future<Output = HashMap<Url, String>> + Send;+ fn get_entity_image(+ &self,+ image_id: &str,+ max_height: Option<u64>,+ max_width: Option<u64>,+ ) -> impl Future<Output = Result<Option<Image>>> + Send;+}diff --git a/core/src/searcher/live.rs b/core/src/searcher/live.rsindex 1e90ebd2..b3b44eec 100644--- a/core/src/searcher/live.rs+++ b/core/src/searcher/live.rs@@ -29,6 +29,7 @@ use crate::{ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},};+use std::future::Future;use std::{collections::HashMap, sync::Arc};use fnv::FnvHashMap;@@ -80,33 +81,6 @@ impl LiveSearcher {ShardedClient::new(shard_clients)}- pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultSplit> {- let client = self.client().await;- let mut results = Vec::new();-- if let Ok(res) = client- .send(- &search_server::Search {- query: query.clone(),- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- {- for (shard_id, mut res) in res {- if let Some(Some(res)) = res.pop() {- results.push(InitialSearchResultSplit {- local_result: res,- split_id: shard_id,- });- }- }- }-- results- }-async fn retrieve_webpages_from_shard(&self,split: SplitId,@@ -138,8 +112,37 @@ impl LiveSearcher {_ => vec![],}}+}++impl SearchClient for LiveSearcher {+ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultSplit> {+ let client = self.client().await;+ let mut results = Vec::new();- pub async fn retrieve_webpages(+ if let Ok(res) = client+ .send(+ &search_server::Search {+ query: query.clone(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ {+ for (shard_id, mut res) in res {+ if let Some(Some(res)) = res.pop() {+ results.push(InitialSearchResultSplit {+ local_result: res,+ split_id: shard_id,+ });+ }+ }+ }++ results+ }++ async fn retrieve_webpages(&self,top_websites: &[(usize, ScoredWebsitePointer)],query: &str,@@ -177,3 +180,15 @@ impl LiveSearcher {retrieved_webpages}}++pub trait SearchClient {+ fn search_initial(+ &self,+ query: &SearchQuery,+ ) -> impl Future<Output = Vec<InitialSearchResultSplit>> + Send;+ fn retrieve_webpages(+ &self,+ top_websites: &[(usize, ScoredWebsitePointer)],+ query: &str,+ ) -> impl Future<Output = Vec<(usize, RetrievedWebpageRanking)>> + Send;+}diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 5e83236e..283c0ee6 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -16,7 +16,7 @@pub mod api;pub mod distributed;-mod live;+pub mod live;pub mod local;pub use distributed::*;diff --git a/core/src/tokenizer/mod.rs b/core/src/tokenizer/mod.rsindex ad31a6fd..690ee285 100644--- a/core/src/tokenizer/mod.rs+++ b/core/src/tokenizer/mod.rs@@ -339,7 +339,7 @@ impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {}}-struct NGramTokenStream<'a, const N: usize> {+pub struct NGramTokenStream<'a, const N: usize> {inner: BoxTokenStream<'a>,token: tantivy::tokenizer::Token,token_window: [tantivy::tokenizer::Token; N],@@ -347,7 +347,7 @@ struct NGramTokenStream<'a, const N: usize> {}impl<'a, const N: usize> NGramTokenStream<'a, N> {- fn new(inner: BoxTokenStream<'a>) -> Self {+ pub fn new(inner: BoxTokenStream<'a>) -> Self {Self {inner,token: tantivy::tokenizer::Token::default(),diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rsindex 2fcea1f0..57d1d4d1 100644--- a/core/src/widgets/mod.rs+++ b/core/src/widgets/mod.rs@@ -50,8 +50,14 @@ impl Widgets {None};+ let exchange_update = if config.calculator_fetch_currencies_exchange {+ calculator::ExchangeUpdate::AsyncTokio+ } else {+ calculator::ExchangeUpdate::None+ };+Ok(Self {- calculator: Calculator::new(calculator::ExchangeUpdate::AsyncTokio),+ calculator: Calculator::new(exchange_update),thesaurus,})}diff --git a/frontend/src/lib/components/Header.svelte b/frontend/src/lib/components/Header.svelteindex f489ff01..13f614ef 100644--- a/frontend/src/lib/components/Header.svelte+++ b/frontend/src/lib/components/Header.svelte@@ -32,7 +32,7 @@<ahref={url}class={twJoin(- 'relative z-10 border-b p-2',+ 'relative z-10 border-b p-2 text-neutral-focus',$page.url.pathname.startsWith(url.split('?')[0]) ? 'border-accent' : 'border-transparent',)}>