web search done right
Ftr/distributed search (#59)
Mikkel Denker3cc7c84a3299
diff --git a/Cargo.lock b/Cargo.lockindex 5a97aca8..5d383a45 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -127,9 +127,9 @@ dependencies = [[[package]]name = "async-trait"-version = "0.1.56"+version = "0.1.57"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716"+checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f"dependencies = ["proc-macro2","quote",@@ -674,6 +674,7 @@ version = "0.1.0"dependencies = ["anyhow","askama",+ "async-trait","axum","axum-extra","base64",diff --git a/Cargo.toml b/Cargo.tomlindex 386b2918..fa4d1402 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -57,6 +57,7 @@ serde_urlencoded = "0.7.1"rust-stemmers = "1.2.0"min-max-heap = "1.3.0"base64 = "0.13.0"+async-trait = "0.1.57"[dev-dependencies]criterion = "0.3.6"diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rsindex 85ae6c87..9258cdc5 100644--- a/benches/search-preindexed-goggle.rs+++ b/benches/search-preindexed-goggle.rs@@ -1,5 +1,5 @@use criterion::{criterion_group, criterion_main, Criterion};-use cuely::{index::Index, searcher::Searcher};+use cuely::{index::Index, searcher::LocalSearcher};const INDEX_PATH: &str = "data/index";@@ -21,7 +21,7 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();- let searcher = Searcher::new(index, None, None);+ let searcher = LocalSearcher::new(index, None, None);let goggle = include_str!("../testcases/goggles/hacker_news.goggle");// for _ in 0..10 {diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rsindex 0033eca2..bbcdee67 100644--- a/benches/search-preindexed.rs+++ b/benches/search-preindexed.rs@@ -1,5 +1,5 @@use criterion::{criterion_group, criterion_main, Criterion};-use cuely::{index::Index, searcher::Searcher};+use cuely::{index::Index, searcher::LocalSearcher};const INDEX_PATH: &str = "data/index";@@ -16,7 +16,7 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();- let searcher = Searcher::new(index, None, None);+ let searcher = LocalSearcher::new(index, None, None);// for _ in 0..100 {bench!("the", searcher, c);diff --git a/configs/frontend.toml b/configs/frontend.tomlnew file mode 100644index 00000000..11006110--- /dev/null+++ b/configs/frontend.toml@@ -0,0 +1,3 @@+queries_csv_path = "data/queries_us.csv"+host = "0.0.0.0:3000"+search_servers = [["0.0.0.0:3001"]]\ No newline at end of filediff --git a/configs/search_server.toml b/configs/search_server.tomlnew file mode 100644index 00000000..089bf194--- /dev/null+++ b/configs/search_server.toml@@ -0,0 +1,4 @@+host = "0.0.0.0:3001"+index_path = "data/index"+entity_index_path = "data/entity"+bangs_path = "data/bangs.json"\ No newline at end of filediff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsindex f8206863..e13fa4a5 100644--- a/examples/mapreduce_manager.rs+++ b/examples/mapreduce_manager.rs@@ -1,6 +1,6 @@use std::net::SocketAddr;-use cuely::mapreduce::{Map, MapReduce, Reduce, StatelessWorker};+use cuely::mapreduce::{Manager, Map, Reduce, StatelessWorker};use serde::{Deserialize, Serialize};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -14,7 +14,7 @@ struct Job {struct Count(usize);impl Map<StatelessWorker, Count> for Job {- fn map(self, _worker: &StatelessWorker) -> Count {+ fn map(&self, _worker: &StatelessWorker) -> Count {std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(1)}@@ -26,7 +26,8 @@ impl Reduce<Count> for Count {}}-fn main() {+#[tokio::main]+async fn main() {let subscriber = FmtSubscriber::builder().with_max_level(Level::DEBUG).finish();@@ -41,9 +42,8 @@ fn main() {Job { id: 5 },];- let res: Count = jobs- .into_iter()- .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])- .unwrap();+ let manager = Manager::new(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()]);+ let res: Count = manager.run(jobs.into_iter()).await.unwrap();+println!("{:?}", res.0);}diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsindex fdca5e9c..9d136537 100644--- a/examples/mapreduce_worker.rs+++ b/examples/mapreduce_worker.rs@@ -14,7 +14,7 @@ struct Job {struct Count(usize);impl Map<StatelessWorker, Count> for Job {- fn map(self, _worker: &StatelessWorker) -> Count {+ fn map(&self, _worker: &StatelessWorker) -> Count {std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(1)}@@ -26,7 +26,8 @@ impl Reduce<Count> for Count {}}-fn main() {+#[tokio::main]+async fn main() {let subscriber = FmtSubscriber::builder().with_max_level(Level::DEBUG).finish();@@ -36,5 +37,6 @@ fn main() {StatelessWorker::default().run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())+ .await.expect("failed to run worker");}diff --git a/justfile b/justfileindex 1c52da5d..2d323990 100644--- a/justfile+++ b/justfile@@ -21,7 +21,7 @@# time ((cd frontend; npm run build) && cargo build)# cargo run -- frontend data/index data/queries_us.csv data/entity data/bangs.jsoncd frontend; npm run build- cargo run -- frontend data/index data/queries_us.csv data/entity data/bangs.json+ bash scripts/run_frontend.sh@frontend:cd frontend; npm installdiff --git a/nohup.out b/nohup.outnew file mode 100644index 00000000..0da8ba7d--- /dev/null+++ b/nohup.out@@ -0,0 +1,20 @@+ Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)+ Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)+ Finished dev [unoptimized + debuginfo] target(s) in 4.40s+ Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`+[2m2022-09-28T09:43:26.587009Z[0m [32m INFO[0m [2mcuely::entrypoint::frontend[0m[2m:[0m listening on 0.0.0.0:3000+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on build directory+ Finished dev [unoptimized + debuginfo] target(s) in 4.31s+ Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`+thread 'main' panicked at 'unable to open rocks db: Error { message: "IO error: While lock file: data/entity/attribute_occurrences/LOCK: Resource temporarily unavailable" }', src/kv/rocksdb_store.rs:39:41+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Blocking waiting for file lock on package cache+ Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)+ Finished dev [unoptimized + debuginfo] target(s) in 4.22s+ Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`+[2m2022-09-28T09:44:05.809083Z[0m [32m INFO[0m [2mcuely::entrypoint::frontend[0m[2m:[0m listening on 0.0.0.0:3000diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.shnew file mode 100644index 00000000..f7c69417--- /dev/null+++ b/scripts/run_frontend.sh@@ -0,0 +1,4 @@+#!/usr/bin/bash+trap 'kill $(jobs -p)' EXIT+cargo run -- search-server configs/search_server.toml &+cargo run -- frontend configs/frontend.tomldiff --git a/src/bangs.rs b/src/bangs.rsindex 21a86c22..18a41903 100644--- a/src/bangs.rs+++ b/src/bangs.rs@@ -50,7 +50,7 @@ pub struct Bang {url: String,}-#[derive(Debug, Serialize, PartialEq, Eq)]+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]pub struct BangHit {bang: Bang,pub redirect_to: Url,diff --git a/src/collector.rs b/src/collector.rsindex 6df5752c..833c4969 100644--- a/src/collector.rs+++ b/src/collector.rs@@ -18,10 +18,11 @@ use min_max_heap::MinMaxHeap;use tantivy::{collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},fastfield::MultiValuedFastFieldReader,- DocAddress, DocId, Score, SegmentOrdinal, SegmentReader,+ DocId, Score, SegmentOrdinal, SegmentReader,};use crate::{+ inverted_index::{DocAddress, WebsitePointer},prehashed::{combine_u64s, PrehashMap, Prehashed},schema::Field,};@@ -37,10 +38,23 @@ pub struct MaxDocsConsidered {pub segments: usize,}+pub trait Doc: PartialEq + Eq + PartialOrd + Ord + Clone {+ fn score(&self) -> &f64;+ fn key(&self) -> &Prehashed;+ fn id(&self) -> &DocId;++ fn set_score(&mut self, score: f64);+ fn set_key(&mut self, key: Prehashed);+ fn set_id(&mut self, id: DocId);++ fn update_values_from(&mut self, other: &Self);+}+pub struct TopDocs {top_n: usize,offset: usize,max_docs: Option<MaxDocsConsidered>,+ de_rank_similar: bool,}impl TopDocs {@@ -49,6 +63,7 @@ impl TopDocs {top_n,offset: 0,max_docs: None,+ de_rank_similar: false,}}@@ -63,10 +78,16 @@ impl TopDocs {self}+ pub fn and_de_rank_similar(mut self) -> Self {+ self.de_rank_similar = true;++ self+ }+pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(self,score_tweaker: TScoreTweaker,- ) -> impl Collector<Fruit = Vec<(f64, DocAddress)>>+ ) -> impl Collector<Fruit = Vec<WebsitePointer>>whereTScoreSegmentTweaker: ScoreSegmentTweaker<f64> + 'static,TScoreTweaker: ScoreTweaker<f64, Child = TScoreSegmentTweaker> + Send + Sync,@@ -76,7 +97,7 @@ impl TopDocs {}impl Collector for TopDocs {- type Fruit = Vec<(f64, DocAddress)>;+ type Fruit = Vec<WebsitePointer>;type Child = TopSegmentCollector;@@ -123,17 +144,16 @@ impl Collector for TopDocs {}Ok(collector- .into_sorted_vec(true)+ .into_sorted_vec(self.de_rank_similar).into_iter().skip(self.offset)- .map(|doc| {- (- doc.score,- DocAddress {- segment_ord: doc.segment,- doc_id: doc.id,- },- )+ .map(|doc| WebsitePointer {+ score: doc.score,+ site_hash: doc.key,+ address: DocAddress {+ segment: doc.segment,+ doc_id: doc.id,+ },}).collect())}@@ -144,11 +164,11 @@ pub struct TopSegmentCollector {max_docs: Option<usize>,num_docs_taken: usize,segment_ord: SegmentOrdinal,- bucket_collector: BucketCollector,+ bucket_collector: BucketCollector<SegmentDoc>,}impl SegmentCollector for TopSegmentCollector {- type Fruit = Vec<Doc>;+ type Fruit = Vec<SegmentDoc>;fn collect(&mut self, doc: DocId, score: Score) {if let Some(max_docs) = &self.max_docs {@@ -165,7 +185,7 @@ impl SegmentCollector for TopSegmentCollector {let keys = [keys[0], keys[1]];let key = combine_u64s(keys);- self.bucket_collector.insert(Doc {+ self.bucket_collector.insert(SegmentDoc {key: Prehashed(key),id: doc,segment: self.segment_ord,@@ -178,13 +198,13 @@ impl SegmentCollector for TopSegmentCollector {}}-struct BucketCollector {- buckets: PrehashMap<Bucket>,+pub struct BucketCollector<T: Doc> {+ buckets: PrehashMap<Bucket<T>>,heads: MinMaxHeap<BucketHead>,top_n: usize,}-impl BucketCollector {+impl<T: Doc> BucketCollector<T> {pub fn new(top_n: usize) -> Self {assert!(top_n > 0);@@ -195,18 +215,18 @@ impl BucketCollector {}}- pub fn insert(&mut self, doc: Doc) {- if let Some(bucket) = self.buckets.get_mut(&doc.key) {+ pub fn insert(&mut self, doc: T) {+ if let Some(bucket) = self.buckets.get_mut(doc.key()) {bucket.insert(doc);} else {let mut bucket = Bucket::new(self.top_n);bucket.insert(doc.clone());- self.buckets.insert(doc.key.clone(), bucket);+ self.buckets.insert(doc.key().clone(), bucket);self.heads.push(BucketHead {- key: doc.key,- tweaked_score: doc.score,+ key: doc.key().clone(),+ tweaked_score: *doc.score(),});}@@ -263,7 +283,7 @@ impl BucketCollector {bucket_heads}- pub fn into_sorted_vec(mut self, apply_adjust_score: bool) -> Vec<Doc> {+ pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {let mut res = Vec::new();let mut bucket_heads = self.build_heads();@@ -272,8 +292,8 @@ impl BucketCollector {let bucket = self.buckets.get_mut(&head.key).unwrap();if let Some(mut doc) = bucket.pop_best() {- if apply_adjust_score {- doc.score = adjust_score(bucket.num_taken - 1, doc.score);+ if de_rank_similar {+ doc.set_score(adjust_score(bucket.num_taken - 1, *doc.score()));}res.push(doc);@@ -319,13 +339,13 @@ impl Ord for BucketHead {}}-struct Bucket {+struct Bucket<T: Doc> {num_taken: usize,- docs: MinMaxHeap<Doc>,+ docs: MinMaxHeap<T>,top_n: usize,}-impl Bucket {+impl<T: Doc> Bucket<T> {pub fn new(top_n: usize) -> Self {assert!(top_n > 0);@@ -336,7 +356,7 @@ impl Bucket {}}- pub fn pop_best(&mut self) -> Option<Doc> {+ pub fn pop_best(&mut self) -> Option<T> {let res = self.docs.pop_max();self.num_taken += 1;@@ -346,18 +366,16 @@ impl Bucket {pub fn get_best(&self) -> Option<TweakedDoc<'_>> {self.docs.peek_max().map(|doc| TweakedDoc {- tweaked_score: adjust_score(self.num_taken, doc.score),- _doc_id: &doc.id,+ tweaked_score: adjust_score(self.num_taken, *doc.score()),+ _doc_id: doc.id(),})}- pub fn insert(&mut self, doc: Doc) {+ pub fn insert(&mut self, doc: T) {if self.docs.len() >= self.top_n {let mut worst = self.docs.peek_min_mut().unwrap();- worst.id = doc.id;- worst.score = doc.score;- worst.segment = doc.segment;+ worst.update_values_from(&doc);} else {self.docs.push(doc);}@@ -370,33 +388,67 @@ struct TweakedDoc<'a> {}#[derive(Debug, Clone)]-pub struct Doc {+pub struct SegmentDoc {key: Prehashed,id: DocId,segment: SegmentOrdinal,score: f64,}-impl PartialEq for Doc {+impl PartialEq for SegmentDoc {fn eq(&self, other: &Self) -> bool {self.score == other.score}}-impl Eq for Doc {}+impl Eq for SegmentDoc {}-impl PartialOrd for Doc {+impl PartialOrd for SegmentDoc {fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {self.score.partial_cmp(&other.score)}}-impl Ord for Doc {+impl Ord for SegmentDoc {fn cmp(&self, other: &Self) -> std::cmp::Ordering {self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)}}+impl Doc for SegmentDoc {+ fn score(&self) -> &f64 {+ &self.score+ }++ fn key(&self) -> &Prehashed {+ &self.key+ }++ fn id(&self) -> &DocId {+ &self.id+ }++ fn set_score(&mut self, score: f64) {+ self.score = score;+ }++ fn set_key(&mut self, key: Prehashed) {+ self.key = key;+ }++ fn set_id(&mut self, id: DocId) {+ self.id = id;+ }++ fn update_values_from(&mut self, other: &Self) {+ self.set_id(*other.id());+ self.set_key(other.key().clone());+ self.set_score(*other.score());++ self.segment = other.segment;+ }+}+pub(crate) struct TweakedScoreTopCollector<TScoreTweaker> {score_tweaker: TScoreTweaker,collector: TopDocs,@@ -418,7 +470,7 @@ impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>whereTScoreTweaker: ScoreTweaker<f64> + Send + Sync,{- type Fruit = Vec<(f64, DocAddress)>;+ type Fruit = Vec<WebsitePointer>;type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child>;@@ -462,7 +514,7 @@ impl<TSegmentScoreTweaker> SegmentCollectorwhereTSegmentScoreTweaker: 'static + ScoreSegmentTweaker<f64>,{- type Fruit = Vec<Doc>;+ type Fruit = Vec<SegmentDoc>;fn collect(&mut self, doc: DocId, score: Score) {let score = self.segment_scorer.score(doc, score);@@ -482,7 +534,7 @@ mod tests {let mut collector = BucketCollector::new(top_n);for doc in docs {- collector.insert(Doc {+ collector.insert(SegmentDoc {key: Prehashed(doc.0),id: doc.1,score: doc.2,diff --git a/src/entity_index/entity.rs b/src/entity_index/entity.rsindex 81a82053..f143c0f4 100644--- a/src/entity_index/entity.rs+++ b/src/entity_index/entity.rs@@ -37,7 +37,7 @@ pub struct Entity {pub categories: HashSet<String>,}-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct Span {pub text: String,pub links: Vec<Link>,diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rsindex cd23cca2..c8a5b632 100644--- a/src/entity_index/mod.rs+++ b/src/entity_index/mod.rs@@ -22,7 +22,7 @@ use std::{time::Duration,};-use serde::Serialize;+use serde::{Deserialize, Serialize};use tantivy::{collector::TopDocs,query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},@@ -145,7 +145,7 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {]}-#[derive(Debug, Serialize, PartialEq, Eq)]+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]pub struct StoredEntity {pub title: String,pub entity_abstract: String,diff --git a/src/entrypoint/frontend.rs b/src/entrypoint/frontend.rsindex fbe0a63b..a7137f6e 100644--- a/src/entrypoint/frontend.rs+++ b/src/entrypoint/frontend.rs@@ -18,14 +18,8 @@ use anyhow::Result;use crate::frontend::router;-pub async fn run(- index_path: &str,- queries_csv_path: &str,- entity_index_path: Option<String>,- bangs_path: Option<String>,- host: &str,-) -> Result<()> {- let app = router(index_path, queries_csv_path, entity_index_path, bangs_path)?;+pub async fn run(queries_csv_path: &str, host: &str, shards: Vec<Vec<String>>) -> Result<()> {+ let app = router(queries_csv_path, shards)?;let addr = host.parse()?;tracing::info!("listening on {}", addr);axum::Server::bind(&addr)diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 8306c2a1..4619dceb 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -23,7 +23,7 @@ use tracing::{debug, info, trace};use crate::entrypoint::download_all_warc_files;use crate::index::{FrozenIndex, Index};-use crate::mapreduce::{Map, MapReduce, Reduce, Worker};+use crate::mapreduce::{Manager, Map, Reduce, Worker};use crate::ranking::centrality_store::CentralityStore;use crate::ranking::SignalAggregator;use crate::warc::WarcFile;@@ -171,15 +171,15 @@ struct IndexPointer(String);impl Worker for IndexingWorker {}impl Map<IndexingWorker, FrozenIndex> for Job {- fn map(self, worker: &IndexingWorker) -> FrozenIndex {- let index = process_job(&self, worker);+ fn map(&self, worker: &IndexingWorker) -> FrozenIndex {+ let index = process_job(self, worker);index.into()}}impl Map<IndexingWorker, IndexPointer> for Job {- fn map(self, worker: &IndexingWorker) -> IndexPointer {- let index = process_job(&self, worker);+ fn map(&self, worker: &IndexingWorker) -> IndexPointer {+ let index = process_job(self, worker);IndexPointer(index.path)}}@@ -227,54 +227,59 @@ impl Reduce<Index> for Index {impl Indexer {pub fn run_master(config: &IndexingMasterConfig) -> Result<()> {- info!("Running master for index construction");-- let warc_paths = config.warc_source.paths()?;-- let workers: Vec<SocketAddr> = config- .workers- .iter()- .map(|worker| worker.parse().unwrap())- .collect();-- let job_config = match config.warc_source.clone() {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- };-- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(- warc_paths- .into_iter()- .chunks(config.batch_size.unwrap_or(1))- .into_iter()- .map(|warc_paths| Job {- source_config: job_config.clone(),- warc_paths: warc_paths.collect_vec(),- download_images: config.download_images.unwrap_or(true),- base_path: config- .index_base_path- .clone()- .unwrap_or_else(|| "data/index".to_string()),- })- .collect_vec()- .into_iter(),- );+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async {+ info!("Running master for index construction");++ let warc_paths = config.warc_source.paths().unwrap();++ let workers: Vec<SocketAddr> = config+ .workers+ .iter()+ .map(|worker| worker.parse().unwrap())+ .collect();++ let job_config = match config.warc_source.clone() {+ WarcSource::HTTP(config) => JobConfig::Http(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ };- if let Some(limit) = config.limit_warc_files {- warc_paths = Box::new(warc_paths.take(limit));- }+ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(+ warc_paths+ .into_iter()+ .chunks(config.batch_size.unwrap_or(1))+ .into_iter()+ .map(|warc_paths| Job {+ source_config: job_config.clone(),+ warc_paths: warc_paths.collect_vec(),+ download_images: config.download_images.unwrap_or(true),+ base_path: config+ .index_base_path+ .clone()+ .unwrap_or_else(|| "data/index".to_string()),+ })+ .collect_vec()+ .into_iter(),+ );++ if let Some(limit) = config.limit_warc_files {+ warc_paths = Box::new(warc_paths.take(limit));+ }- let mut index = <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<- IndexingWorker,- Job,- FrozenIndex,- Index,- >>::map_reduce(warc_paths, &workers)- .expect("failed to build index");+ let manager = Manager::new(&workers);+ let mut index: Index = manager+ .run::<IndexingWorker, Job, FrozenIndex, Index>(warc_paths)+ .await+ .unwrap();- index- .inverted_index- .merge_into_segments(config.final_num_segments.unwrap_or(20))?;+ index+ .inverted_index+ .merge_into_segments(config.final_num_segments.unwrap_or(20))+ .unwrap();+ });Ok(())}@@ -284,11 +289,20 @@ impl Indexer {centrality_store_path: String,webgraph_path: Option<String>,) -> Result<()> {- IndexingWorker::new(centrality_store_path, webgraph_path).run::<Job, FrozenIndex>(- worker_addr- .parse::<SocketAddr>()- .expect("Could not parse worker address"),- )?;+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async {+ IndexingWorker::new(centrality_store_path, webgraph_path)+ .run::<Job, FrozenIndex>(+ worker_addr+ .parse::<SocketAddr>()+ .expect("Could not parse worker address"),+ )+ .await+ .unwrap();+ });Ok(())}diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex c4da2518..ba8fcfcd 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -17,6 +17,7 @@ mod centrality;mod entity;pub mod frontend;mod indexer;+pub mod search_server;mod webgraph;use std::{fs::File, path::Path};diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rsnew file mode 100644index 00000000..63f90317--- /dev/null+++ b/src/entrypoint/search_server.rs@@ -0,0 +1,131 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use crate::{+ bangs::Bangs,+ entity_index::EntityIndex,+ index::Index,+ inverted_index,+ search_prettifier::{self},+ searcher::{self, LocalSearcher},+ sonic, Result, SearchServerConfig,+};++pub async fn run(config: SearchServerConfig) -> Result<()> {+ let addr: SocketAddr = config.host.parse().unwrap();+ let server = sonic::Server::bind(addr).await.unwrap();+ tracing::info!("listening on {}", addr);++ let entity_index = config+ .entity_index_path+ .map(|path| EntityIndex::open(path).unwrap());+ let bangs = config.bangs_path.map(Bangs::from_path);+ let search_index = Index::open(config.index_path)?;++ let local_searcher = LocalSearcher::new(search_index, entity_index, bangs);++ loop {+ if let Ok(req) = server.accept::<searcher::distributed::Request>().await {+ match &req.body {+ searcher::Request::Search(search) => {+ match local_searcher.search_initial(+ &search.original,+ search.selected_region,+ search.goggle_program.clone(),+ search.skip_pages,+ false,+ ) {+ Ok(response) => {+ req.respond(sonic::Response::Content(response)).await.ok();+ }+ Err(_) => {+ req.respond::<searcher::InitialSearchResult>(sonic::Response::Empty)+ .await+ .ok();+ }+ }+ }+ searcher::Request::RetrieveWebites { websites, query } => {+ match local_searcher.retrieve_websites(websites, query) {+ Ok(response) => {+ req.respond(sonic::Response::Content(response)).await.ok();+ }+ Err(_) => {+ req.respond::<Vec<inverted_index::RetrievedWebpage>>(+ sonic::Response::Empty,+ )+ .await+ .ok();+ }+ }+ }+ searcher::Request::SearchPrettified(search) => {+ match local_searcher.search_initial(+ &search.original,+ search.selected_region,+ search.goggle_program.clone(),+ search.skip_pages,+ false,+ ) {+ Ok(result) => match result {+ searcher::InitialSearchResult::Websites(+ searcher::InitialWebsitesFormatting::Raw(result),+ ) => {+ let res = search_prettifier::initial(result, &local_searcher);++ req.respond(sonic::Response::Content(+ searcher::InitialSearchResult::Websites(+ searcher::InitialWebsitesFormatting::Prettified(res),+ ),+ ))+ .await+ .ok();+ }+ searcher::InitialSearchResult::Bang(_) => todo!(),+ _ => unreachable!(),+ },+ Err(_) => {+ req.respond::<inverted_index::SearchResult>(sonic::Response::Empty)+ .await+ .ok();+ }+ }+ }+ searcher::Request::RetrievePrettifiedWebites { websites, query } => {+ match local_searcher.retrieve_websites(websites, query) {+ Ok(result) => {+ req.respond(sonic::Response::Content(search_prettifier::retrieve(+ result,+ &local_searcher,+ )))+ .await+ .ok();+ }+ Err(_) => {+ req.respond::<Vec<search_prettifier::DisplayedWebpage>>(+ sonic::Response::Empty,+ )+ .await+ .ok();+ }+ }+ }+ }+ }+ }+}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 928aa02b..10eee39b 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -16,7 +16,7 @@use crate::{directory::DirEntry,entrypoint::download_all_warc_files,- mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},+ mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,@@ -97,15 +97,15 @@ fn process_job(job: &Job) -> webgraph::Webgraph {}impl Map<StatelessWorker, FrozenWebgraph> for Job {- fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {- let graph = process_job(&self);+ fn map(&self, _worker: &StatelessWorker) -> FrozenWebgraph {+ let graph = process_job(self);graph.into()}}impl Map<StatelessWorker, GraphPointer> for Job {- fn map(self, _worker: &StatelessWorker) -> GraphPointer {- let graph = process_job(&self);+ fn map(&self, _worker: &StatelessWorker) -> GraphPointer {+ let graph = process_job(self);GraphPointer(graph.path)}}@@ -161,60 +161,74 @@ pub struct Webgraph {}impl Webgraph {pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {- info!("Running master for webgraph construction");-- let warc_paths = config.warc_source.paths()?;-- let workers: Vec<SocketAddr> = config- .workers- .iter()- .map(|worker| worker.parse().unwrap())- .collect();-- let job_config = match config.warc_source.clone() {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- };-- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(- warc_paths- .into_iter()- .chunks(config.batch_size.unwrap_or(1))- .into_iter()- .map(|warc_paths| Job {- config: job_config.clone(),- warc_paths: warc_paths.into_iter().collect(),- graph_base_path: config- .graph_base_path- .clone()- .unwrap_or_else(|| "data/webgraph".to_string()),- })- .collect::<Vec<_>>()- .into_iter(),- );-- if let Some(limit) = config.limit_warc_files {- warc_paths = Box::new(warc_paths.take(limit));- }+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async {+ info!("Running master for webgraph construction");++ let warc_paths = config.warc_source.paths().unwrap();++ let workers: Vec<SocketAddr> = config+ .workers+ .iter()+ .map(|worker| worker.parse().unwrap())+ .collect();++ let job_config = match config.warc_source.clone() {+ WarcSource::HTTP(config) => JobConfig::Http(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ };++ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(+ warc_paths+ .into_iter()+ .chunks(config.batch_size.unwrap_or(1))+ .into_iter()+ .map(|warc_paths| Job {+ config: job_config.clone(),+ warc_paths: warc_paths.into_iter().collect(),+ graph_base_path: config+ .graph_base_path+ .clone()+ .unwrap_or_else(|| "data/webgraph".to_string()),+ })+ .collect::<Vec<_>>()+ .into_iter(),+ );++ if let Some(limit) = config.limit_warc_files {+ warc_paths = Box::new(warc_paths.take(limit));+ }- let _graph: webgraph::Webgraph =- <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<- StatelessWorker,- Job,- webgraph::FrozenWebgraph,- webgraph::Webgraph,- >>::map_reduce(warc_paths, &workers)- .expect("failed to build webgraph");+ let manager = Manager::new(&workers);+ let _graph: webgraph::Webgraph = manager+ .run::<StatelessWorker, Job, webgraph::FrozenWebgraph, webgraph::Webgraph>(+ warc_paths,+ )+ .await+ .unwrap();+ });Ok(())}pub fn run_worker(worker_addr: String) -> Result<()> {- StatelessWorker::default().run::<Job, FrozenWebgraph>(- worker_addr- .parse::<SocketAddr>()- .expect("Could not parse worker address"),- )?;+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async {+ StatelessWorker::default()+ .run::<Job, FrozenWebgraph>(+ worker_addr+ .parse::<SocketAddr>()+ .expect("Could not parse worker address"),+ )+ .await+ .unwrap();+ });Ok(())}diff --git a/src/frontend/api.rs b/src/frontend/api.rsindex 97333707..f935af6d 100644--- a/src/frontend/api.rs+++ b/src/frontend/api.rs@@ -40,7 +40,8 @@ pub async fn search(match state.searcher- .search(query.as_str(), selected_region, None, skip_pages)+ .search_api(query.as_str(), selected_region, None, skip_pages)+ .await{Ok(result) => Json(result),Err(_) => panic!("Search failed"), // TODO: show 500 status to user herediff --git a/src/frontend/entity_image.rs b/src/frontend/entity_image.rsdeleted file mode 100644index 49afd79d..00000000--- a/src/frontend/entity_image.rs+++ /dev/null@@ -1,44 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use axum::body::Body;-use axum::extract::Path;-use axum::http::Response;-use axum::response::IntoResponse;-use axum::Extension;-use reqwest::StatusCode;--use super::State;--#[allow(clippy::unused_async)]-pub async fn route(- Path(entity): Path<String>,- Extension(state): Extension<Arc<State>>,-) -> impl IntoResponse {- let img = state.searcher.entity_image(entity);-- let bytes = match img {- Some(img) => img.as_raw_bytes(),- None => Vec::new(),- };-- Response::builder()- .status(StatusCode::OK)- .body(Body::from(bytes))- .unwrap()-}diff --git a/src/frontend/favicons.rs b/src/frontend/favicons.rsdeleted file mode 100644index 43d4001f..00000000--- a/src/frontend/favicons.rs+++ /dev/null@@ -1,46 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use axum::body::Body;-use axum::extract::Path;-use axum::http::Response;-use axum::response::IntoResponse;-use axum::Extension;-use reqwest::StatusCode;--use crate::webpage::Url;--use super::State;--#[allow(clippy::unused_async)]-pub async fn route(- Path(site): Path<String>,- Extension(state): Extension<Arc<State>>,-) -> impl IntoResponse {- let img = state.searcher.favicon(&Url::from(site));-- let bytes = match img {- Some(img) => img.as_raw_bytes(),- None => include_bytes!("../../frontend/dist/assets/images/globe.png").to_vec(),- };-- Response::builder()- .status(StatusCode::OK)- .body(Body::from(bytes))- .unwrap()-}diff --git a/src/frontend/mod.rs b/src/frontend/mod.rsindex 1a3a6f49..a000196b 100644--- a/src/frontend/mod.rs+++ b/src/frontend/mod.rs@@ -18,8 +18,8 @@ use axum::{body::Body, Extension, Router};use tower_http::compression::CompressionLayer;use crate::{- autosuggest::Autosuggest, bangs::Bangs, entity_index::EntityIndex, index::Index,- searcher::Searcher,+ autosuggest::Autosuggest,+ searcher::{DistributedSearcher, Shard},};use anyhow::Result;use std::sync::Arc;@@ -35,18 +35,15 @@ use axum_extra::routing::SpaRouter;mod about;mod api;mod autosuggest;-mod entity_image;-mod favicons;mod goggles;mod index;-mod primary_image;mod privacy;pub mod search;pub struct HtmlTemplate<T>(T);pub struct State {- pub searcher: Searcher,+ pub searcher: DistributedSearcher,pub autosuggest: Autosuggest,}@@ -76,17 +73,15 @@ pub async fn favicon() -> impl IntoResponse {.unwrap()}-pub fn router(- index_path: &str,- queries_csv_path: &str,- entity_index_path: Option<String>,- bangs_path: Option<String>,-) -> Result<Router> {- let entity_index = entity_index_path.map(|path| EntityIndex::open(path).unwrap());- let bangs = bangs_path.map(Bangs::from_path);- let search_index = Index::open(index_path)?;+pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router> {+ let shards: Vec<_> = shards+ .into_iter()+ .enumerate()+ .map(|(id, replicas)| Shard::new(id as u32, replicas))+ .collect();+let autosuggest = Autosuggest::load_csv(queries_csv_path)?;- let searcher = Searcher::new(search_index, entity_index, bangs);+ let searcher = DistributedSearcher::new(shards);let state = Arc::new(State {searcher,@@ -97,9 +92,6 @@ pub fn router(.route("/", get(index::route)).route("/search", get(search::route)).route("/autosuggest", get(autosuggest::route))- .route("/favicons/:site", get(favicons::route))- .route("/image/:uuid", get(primary_image::route))- .route("/entity/image/:entity", get(entity_image::route)).route("/favicon.ico", get(favicon)).route("/about", get(about::route)).route("/goggles", get(goggles::route))diff --git a/src/frontend/primary_image.rs b/src/frontend/primary_image.rsdeleted file mode 100644index cb19cb6b..00000000--- a/src/frontend/primary_image.rs+++ /dev/null@@ -1,45 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use axum::body::Body;-use axum::extract::Path;-use axum::http::Response;-use axum::response::IntoResponse;-use axum::Extension;-use reqwest::StatusCode;--use super::State;--#[allow(clippy::unused_async)]-pub async fn route(- Path(uuid): Path<String>,- Extension(state): Extension<Arc<State>>,-) -> impl IntoResponse {- let img = state.searcher.primary_image(uuid);-- match img {- Some(img) => Response::builder()- .status(StatusCode::OK)- .body(Body::from(img.as_raw_bytes()))- .unwrap(),- None => Response::builder()- .status(StatusCode::NOT_FOUND)- .body(Body::empty())- .unwrap(),- }-}diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex a257a46d..d326921c 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -14,23 +14,15 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::{collections::HashMap, sync::Arc};+use axum::Extension;-use chrono::{NaiveDate, NaiveDateTime, Utc};-use itertools::{intersperse, Itertools};-use serde::Serialize;use crate::{- entity_index::{entity::Span, StoredEntity},- inverted_index::RetrievedWebpage,- searcher::{SearchResult, Searcher},- webpage::{- region::{Region, ALL_REGIONS},- Url,- },- Error,+ search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},+ searcher::{self, PrettifiedSearchResult},+ webpage::region::{Region, ALL_REGIONS},};-use std::collections::{BTreeMap, HashMap};-use std::sync::Arc;use super::{goggles::{GoggleLink, DEFAULT_GOGGLES},@@ -42,219 +34,6 @@ use axum::{response::{IntoResponse, Redirect},};-pub fn html_escape(s: &str) -> String {- html_escape::decode_html_entities(s)- .chars()- .filter(|c| !matches!(c, '<' | '>' | '&'))- .collect()-}--pub struct DisplayedWebpage {- pub title: String,- pub url: String,- pub favicon_base64: String,- pub domain: String,- pub pretty_url: String,- pub snippet: String,- pub body: String,- pub primary_image_uuid: Option<String>,- pub last_updated: Option<String>,-}--const MAX_PRETTY_URL_LEN: usize = 50;-const MAX_TITLE_LEN: usize = 70;--fn prettify_url(url: &Url) -> String {- let mut pretty_url = url.strip_query().to_string();-- if pretty_url.ends_with('/') {- pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();- }-- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";- pretty_url = Url::from(pretty_url.clone())- .strip_protocol()- .replace('/', " › ");- pretty_url = protocol + &pretty_url;-- if pretty_url.len() > MAX_PRETTY_URL_LEN {- pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();- pretty_url += "...";- }-- pretty_url-}--fn prettify_date(date: NaiveDateTime) -> String {- let current_time = Utc::now().naive_utc();- let diff = current_time.signed_duration_since(date);-- let num_hours = diff.num_hours() + 1;- if num_hours < 24 {- if num_hours <= 1 {- return "1 hour ago".to_string();- }-- return format!("{num_hours} hours ago");- }-- let num_days = diff.num_days();- if num_days < 30 {- if num_days <= 1 {- return "1 day ago".to_string();- }-- return format!("{num_days} days ago");- }-- format!("{}", date.format("%d. %b. %Y"))-}--impl From<RetrievedWebpage> for DisplayedWebpage {- fn from(webpage: RetrievedWebpage) -> Self {- let last_updated = webpage.updated_time.map(prettify_date);-- let url: Url = webpage.url.clone().into();- let domain = url.domain().to_string();- let pretty_url = prettify_url(&url);-- let mut title = html_escape(&webpage.title);-- if title.len() > MAX_TITLE_LEN {- title = title.chars().take(MAX_TITLE_LEN).collect();- title += "...";- }-- let favicon_bytes = webpage- .favicon- .map(|favicon| favicon.as_raw_bytes())- .unwrap_or_else(|| {- include_bytes!("../../frontend/dist/assets/images/globe.png").to_vec()- });-- Self {- title,- url: webpage.url,- pretty_url,- domain,- favicon_base64: base64::encode(favicon_bytes),- snippet: webpage.snippet, // snippet has already been html-escaped.- body: webpage.body,- primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),- last_updated,- }- }-}--#[derive(Serialize)]-pub struct DisplayedEntity {- pub title: String,- pub small_abstract: String,- pub image_base64: Option<String>,- pub related_entities: Vec<DisplayedEntity>,- pub info: Vec<(String, String)>,-}--fn prepare_info(info: BTreeMap<String, Span>, searcher: &Searcher) -> Vec<(String, String)> {- let mut info: Vec<_> = info.into_iter().collect();-- info.sort_by(|(a, _), (b, _)| {- searcher- .attribute_occurrence(b)- .unwrap_or(0)- .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))- });-- info.into_iter()- .map(|(key, value)| {- let mut value = entity_link_to_html(value, 150).replace('*', "•");-- if value.starts_with('•') || value.starts_with("\n•") {- if let Some(first_bullet) = value.find('•') {- value = value.chars().skip(first_bullet + 1).collect();- }- }-- let value = maybe_prettify_entity_date(value);-- (key.replace('_', " "), value)- })- .filter(|(key, _)| {- !matches!(- key.as_str(),- "caption"- | "image size"- | "label"- | "landscape"- | "signature"- | "name"- | "website"- | "logo"- | "image caption"- )- })- .take(5)- .collect()-}--impl DisplayedEntity {- fn from(entity: StoredEntity, searcher: &Searcher) -> Self {- let entity_abstract = Span {- text: entity.entity_abstract,- links: entity.links,- };-- let small_abstract = entity_link_to_html(entity_abstract, 300);-- let image_base64 = entity- .image- .and_then(|image| searcher.entity_image(image))- .map(|image| base64::encode(image.as_raw_bytes()));-- Self {- title: entity.title,- small_abstract,- image_base64,- related_entities: entity- .related_entities- .into_iter()- .map(|entity| DisplayedEntity::from(entity, searcher))- .collect(),- info: prepare_info(entity.info, searcher),- }- }-}--fn maybe_prettify_entity_date(value: String) -> String {- if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {- return format!("{}", date.format("%d/%m/%Y"));- }-- if value.split_whitespace().count() == 6 {- let first_date = NaiveDate::parse_from_str(- itertools::intersperse(value.split_whitespace().take(3), " ")- .collect::<String>()- .as_str(),- "%Y %-m %-d",- );- let second_date = NaiveDate::parse_from_str(- itertools::intersperse(value.split_whitespace().skip(3), " ")- .collect::<String>()- .as_str(),- "%Y %-m %-d",- );-- if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {- // the dates are reversed from the parser, so we need to return second_date before first_date- return format!("{}", second_date.format("%d/%m/%Y"))- + " - "- + format!("{}", first_date.format("%d/%m/%Y")).as_str();- }- }-- value-}-#[derive(Template)]#[template(path = "search/index.html", escape = "none")]struct SearchTemplate {@@ -312,40 +91,15 @@ pub async fn route(match state.searcher- .search(query.as_str(), selected_region, goggle, skip_pages)+ .search_prettified(query.as_str(), selected_region, goggle, skip_pages)+ .await{Ok(result) => match result {- SearchResult::Websites(result) => {- let search_result = result- .webpages- .documents- .into_iter()- .map(|mut webpage| {- webpage.primary_image = webpage.primary_image.and_then(|image| {- if state- .searcher- .primary_image(image.uuid.clone().to_string())- .is_some()- {- Some(image)- } else {- None- }- });-- let url: Url = webpage.url.clone().into();- webpage.favicon = state.searcher.favicon(&url.site().to_string().into());- webpage- })- .map(DisplayedWebpage::from)- .collect();-- let entity = result- .entity- .map(|entity| DisplayedEntity::from(entity, &state.searcher));+ PrettifiedSearchResult::Websites(result) => {+ let entity = result.entity;let spell_correction = result.spell_corrected_query;- let num_matches = thousand_sep_number(result.webpages.num_docs);+ let num_matches = thousand_sep_number(result.num_docs);let search_duration_sec =format!("{:.2}", result.search_duration_ms as f64 / 1000.0);@@ -391,7 +145,7 @@ pub async fn route(};let template = SearchTemplate {- search_result,+ search_result: result.webpages,query,entity,spell_correction,@@ -407,186 +161,11 @@ pub async fn route(HtmlTemplate(template).into_response()}- SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),+ PrettifiedSearchResult::Bang(result) => {+ Redirect::to(&result.redirect_to.full()).into_response()+ }},- Err(Error::EmptyQuery) => Redirect::to("/").into_response(),+ Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),Err(_) => panic!("Search failed"), // TODO: show 500 status to user here}}--fn thousand_sep_number(num: usize) -> String {- let s = num.to_string();- let c = s.chars().rev().chunks(3);- let chunks = c.into_iter().map(|chunk| {- chunk- .into_iter()- .collect::<Vec<char>>()- .into_iter()- .rev()- .collect::<String>()- });-- intersperse(- chunks.collect::<Vec<_>>().into_iter().rev(),- ".".to_string(),- )- .collect()-}--fn entity_link_to_html(span: Span, trunace_to: usize) -> String {- let mut s = span.text;-- let truncated = s.len() > trunace_to;- if truncated {- s = s.chars().take(trunace_to).collect();- }-- let chars = s.chars();- let num_chars = chars.clone().count();-- let mut res = String::new();-- let mut last_link_end = 0;- for link in span.links {- if link.start > num_chars {- break;- }-- res += chars- .clone()- .skip(last_link_end)- .take(link.start - last_link_end)- .collect::<String>()- .as_str();-- let link_text: String = chars- .clone()- .skip(link.start)- .take(link.end - link.start)- .collect();-- res += format!(- "<a href=\"https://en.wikipedia.org/wiki/{}\">",- link.target.replace(' ', "_")- )- .as_str();-- res += link_text.as_str();-- res += "</a>";-- last_link_end = link.end;- }-- res += chars- .clone()- .skip(last_link_end)- .collect::<String>()- .as_str();-- if truncated {- res += "...";- }-- res-}--#[cfg(test)]-mod tests {- use chrono::{DateTime, Utc};-- use crate::entity_index::entity::Link;-- use super::*;-- #[test]- fn prettify_date_in_hours() {- let date = Utc::now().naive_utc();- assert_eq!(prettify_date(date), "1 hour ago".to_string());-- let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();- assert_eq!(prettify_date(date), "2 hours ago".to_string());- }-- #[test]- fn prettify_date_days() {- let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();- assert_eq!(prettify_date(date), "1 day ago".to_string());-- let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();- assert_eq!(prettify_date(date), "2 days ago".to_string());- }-- #[test]- fn prettify_date_rest() {- let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")- .unwrap()- .naive_local();- assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());- }-- #[test]- fn simple_link_to_html() {- assert_eq!(- entity_link_to_html(- Span {- text: "some text with a link".to_string(),- links: vec![Link {- start: 5,- end: 9,- target: "text article".to_string()- }]- },- 10000- ),- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"- .to_string()- );- }-- #[test]- fn truncated_link_to_html() {- assert_eq!(- entity_link_to_html(- Span {- text: "some text".to_string(),- links: vec![Link {- start: 5,- end: 9,- target: "text article".to_string()- }]- },- 7- ),- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()- );- }-- #[test]- fn einstein_date() {- assert_eq!(- maybe_prettify_entity_date("1879 3 14 ".to_string()),- "14/03/1879".to_string()- );- }-- #[test]- fn entity_date_span_prettify() {- assert_eq!(- maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),- "14/03/1879 - 27/05/1999".to_string()- );- }-- #[test]- fn sep_number() {- assert_eq!(thousand_sep_number(0), "0".to_string());- assert_eq!(thousand_sep_number(10), "10".to_string());- assert_eq!(thousand_sep_number(100), "100".to_string());- assert_eq!(thousand_sep_number(1000), "1.000".to_string());- assert_eq!(thousand_sep_number(10_000), "10.000".to_string());- assert_eq!(thousand_sep_number(100_000), "100.000".to_string());- assert_eq!(thousand_sep_number(512_854), "512.854".to_string());- assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());- }-}diff --git a/src/index.rs b/src/index.rsindex 19dd506c..85d8aa83 100644--- a/src/index.rs+++ b/src/index.rs@@ -30,7 +30,7 @@ use uuid::Uuid;use crate::directory::{self, DirEntry};use crate::image_downloader::{ImageDownloadJob, ImageDownloader};use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};-use crate::inverted_index::{InvertedIndex, InvertedIndexSearchResult};+use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};use crate::query::Query;use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};use crate::webpage::region::{Region, RegionCount};@@ -111,13 +111,28 @@ impl Index {Ok(())}- pub fn search<C>(&self, query: &Query, collector: C) -> Result<InvertedIndexSearchResult>+ pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>where- C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,+ C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,{self.inverted_index.search(query, collector)}+ pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>+ where+ C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,+ {+ self.inverted_index.search_initial(query, collector)+ }++ pub fn retrieve_websites(+ &self,+ websites: &[inverted_index::WebsitePointer],+ query: &Query,+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {+ self.inverted_index.retrieve_websites(websites, query)+ }+pub fn merge(mut self, other: Self) -> Self {self.inverted_index.merge(other.inverted_index);diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex b9e29cc1..f9aac627 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -15,15 +15,16 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use chrono::NaiveDateTime;-use serde::Serialize;+use serde::{Deserialize, Serialize};use tantivy::collector::{Collector, Count};use tantivy::directory::MmapDirectory;use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;-use tantivy::{DocAddress, Document, IndexReader, IndexWriter, SegmentMeta};+use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};use crate::image_store::Image;+use crate::prehashed::Prehashed;use crate::query::Query;use crate::schema::{Field, ALL_FIELDS};use crate::snippet;@@ -35,6 +36,43 @@ use std::fs;use std::path::Path;use std::sync::Arc;+#[derive(Debug, Serialize, Deserialize)]+pub struct InitialSearchResult {+ pub num_websites: usize,+ pub top_websites: Vec<WebsitePointer>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct WebsitePointer {+ pub score: f64,+ pub site_hash: Prehashed,+ pub address: DocAddress,+}++#[derive(Debug, Serialize, Deserialize, Clone, Copy)]+pub struct DocAddress {+ pub segment: u32,+ pub doc_id: u32,+}++impl From<tantivy::DocAddress> for DocAddress {+ fn from(address: tantivy::DocAddress) -> Self {+ Self {+ segment: address.segment_ord,+ doc_id: address.doc_id,+ }+ }+}++impl From<DocAddress> for tantivy::DocAddress {+ fn from(address: DocAddress) -> Self {+ Self {+ segment_ord: address.segment,+ doc_id: address.doc_id,+ }+ }+}+struct SegmentMergeCandidate {num_docs: u32,segments: Vec<SegmentMeta>,@@ -118,17 +156,28 @@ impl InvertedIndex {Ok(())}- pub fn search<C>(&self, query: &Query, collector: C) -> Result<InvertedIndexSearchResult>+ pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>where- C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,+ C: Collector<Fruit = Vec<WebsitePointer>>,{let searcher = self.reader.searcher();let (count, docs) = searcher.search(query, &(Count, collector))?;+ Ok(InitialSearchResult {+ num_websites: count,+ top_websites: docs,+ })+ }- let mut webpages: Vec<RetrievedWebpage> = docs- .into_iter()- .map(|(_score, doc_address)| self.retrieve_doc(doc_address, &searcher))+ pub fn retrieve_websites(+ &self,+ websites: &[WebsitePointer],+ query: &Query,+ ) -> Result<Vec<RetrievedWebpage>> {+ let searcher = self.reader.searcher();+ let mut webpages: Vec<RetrievedWebpage> = websites+ .iter()+ .map(|website| self.retrieve_doc(website.address, &searcher)).filter_map(|page| page.ok()).map(|mut doc| {if let Some(image) = doc.primary_image.as_ref() {@@ -159,9 +208,19 @@ impl InvertedIndex {)?;}- Ok(InvertedIndexSearchResult {- num_docs: count,- documents: webpages,+ Ok(webpages)+ }++ pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>+ where+ C: Collector<Fruit = Vec<WebsitePointer>>,+ {+ let initial_result = self.search_initial(query, collector)?;+ let websites = self.retrieve_websites(&initial_result.top_websites, query)?;++ Ok(SearchResult {+ num_docs: initial_result.num_websites,+ documents: websites,})}@@ -222,7 +281,7 @@ impl InvertedIndex {doc_address: DocAddress,searcher: &tantivy::Searcher,) -> Result<RetrievedWebpage> {- let doc = searcher.doc(doc_address)?;+ let doc = searcher.doc(doc_address.into())?;Ok(RetrievedWebpage::from(doc))}@@ -289,12 +348,12 @@ impl InvertedIndex {}#[derive(Debug, Serialize)]-pub struct InvertedIndexSearchResult {+pub struct SearchResult {pub num_docs: usize,pub documents: Vec<RetrievedWebpage>,}-#[derive(Default, Debug, Serialize)]+#[derive(Default, Debug, Serialize, Deserialize)]pub struct RetrievedWebpage {pub title: String,pub url: String,diff --git a/src/lib.rs b/src/lib.rsindex 7e5311be..193bf000 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -48,8 +48,10 @@ mod query;mod ranking;mod schema;mod schema_org;+mod search_prettifier;pub mod searcher;mod snippet;+mod sonic;mod spell;mod tokenizer;mod warc;@@ -134,6 +136,21 @@ pub struct HttpConfig {warc_paths_file: String,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct FrontendConfig {+ pub queries_csv_path: String,+ pub host: String,+ pub search_servers: Vec<Vec<String>>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct SearchServerConfig {+ pub index_path: String,+ pub entity_index_path: Option<String>,+ pub bangs_path: Option<String>,+ pub host: String,+}+#[derive(Error, Debug)]pub enum Error {#[error("Failed to download object from HTTP")]diff --git a/src/main.rs b/src/main.rsindex 643c1009..03865e9a 100644--- a/src/main.rs+++ b/src/main.rs@@ -15,7 +15,8 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;use clap::{Parser, Subcommand};-use cuely::entrypoint::{self, frontend};+use cuely::entrypoint::{self, frontend, search_server};+use cuely::{FrontendConfig, SearchServerConfig};use serde::de::DeserializeOwned;use std::fs;use std::path::Path;@@ -44,13 +45,11 @@ enum Commands {#[clap(subcommand)]options: WebgraphOptions,},+ SearchServer {+ config_path: String,+ },Frontend {- index_path: String,- queries_csv_path: String,- entity_index_path: Option<String>,- bangs_path: Option<String>,- #[clap(default_value = "0.0.0.0:3000")]- host: String,+ config_path: String,},}@@ -132,22 +131,26 @@ fn main() -> Result<()> {entrypoint::Webgraph::run_locally(&config)?;}},- Commands::Frontend {- index_path,- queries_csv_path,- host,- entity_index_path,- bangs_path,- } => tokio::runtime::Builder::new_multi_thread()- .enable_all()- .build()?- .block_on(frontend::run(- &index_path,- &queries_csv_path,- entity_index_path,- bangs_path,- &host,- ))?,+ Commands::Frontend { config_path } => {+ let config: FrontendConfig = load_toml_config(&config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(frontend::run(+ &config.queries_csv_path,+ &config.host,+ config.search_servers,+ ))?+ }+ Commands::SearchServer { config_path } => {+ let config: SearchServerConfig = load_toml_config(&config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(search_server::run(config))?+ }}Ok(())diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 8ebea4c5..ea322173 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -2,39 +2,35 @@ use super::{Error, Result, Worker};use super::{Map, Reduce};use crate::exponential_backoff::ExponentialBackoff;use crate::mapreduce::Task;-use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle};-use rayon::iter::ParallelBridge;-use rayon::prelude::*;+use crate::sonic;+use futures::StreamExt;+use itertools::Itertools;use serde::de::DeserializeOwned;use serde::Serialize;-use std::io::{Read, Write};+use std::net::SocketAddr;use std::net::ToSocketAddrs;-use std::net::{SocketAddr, TcpStream};use std::ops::Deref;use std::sync::atomic::{AtomicU32, Ordering};-use std::sync::{Arc, Mutex};+use std::sync::Arc;use std::time::Duration;+use tokio::sync::Mutex;use tracing::{debug, warn};#[derive(Debug)]struct RemoteWorker {addr: SocketAddr,}-pub const BUF_SIZE: usize = 4096;--pub const END_OF_MESSAGE: [u8; BUF_SIZE] = [42; BUF_SIZE];impl RemoteWorker {fn retry_strategy() -> impl Iterator<Item = Duration> {ExponentialBackoff::from_millis(10).take(5)}- fn connect(&self) -> Result<TcpStream> {- debug!("connecting to {:}", self.addr);+ async fn connect(&self) -> Result<sonic::Connection> {for dur in RemoteWorker::retry_strategy() {- if let Ok(stream) = TcpStream::connect(&self.addr) {+ if let Ok(conn) = sonic::Connection::create(self.addr).await {debug!("connected");- return Ok(stream);+ return Ok(conn);}std::thread::sleep(dur);@@ -43,53 +39,31 @@ impl RemoteWorker {Err(Error::NoResponse)}- fn perform<W, I, O>(&self, job: &I) -> Result<O>+ async fn perform<W, I, O>(&self, job: &I) -> Result<O>whereW: Worker,I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{- let mut stream = self.connect()?;- let serialized_job = bincode::serialize(&Task::Job(job))?;- debug!("sending {:?} bytes", serialized_job.len());- stream.write_all(&serialized_job)?;- stream.write_all(&END_OF_MESSAGE)?;-- let mut buf = [0; BUF_SIZE];- let mut bytes = Vec::new();- loop {- if let Ok(size) = stream.read(&mut buf) {- debug!("read {:?} bytes", size);- if size == 0 && bytes.is_empty() {- return Err(Error::NoResponse);- }- bytes.extend_from_slice(&buf[..size]);-- if bytes.len() >= END_OF_MESSAGE.len()- && bytes[bytes.len() - END_OF_MESSAGE.len()..] == END_OF_MESSAGE- {- break;- }- }+ let conn = self.connect().await?;+ match conn.send(Task::Job(job)).await {+ Ok(sonic::Response::Content(res)) => Ok(res),+ _ => Err(Error::NoResponse),}- bytes = bytes[..bytes.len() - END_OF_MESSAGE.len()].to_vec();- debug!("finished reading {:?} bytes", bytes.len());-- Ok(bincode::deserialize(&bytes)?)}- fn stop<W, I, O>(&self) -> Result<()>+ async fn stop<W, I, O>(&self) -> Result<()>whereW: Worker,I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{debug!("closing worker {:}", self.addr);- let mut stream = self.connect()?;- let serialized_job = bincode::serialize(&Task::<I>::AllFinished)?;- debug!("sending {:?} bytes", serialized_job.len());- stream.write_all(&serialized_job)?;- stream.write_all(&END_OF_MESSAGE)?;+ let conn = self.connect().await?;+ let res: sonic::Response<O> = conn.send(Task::<I>::AllFinished).await?;++ debug_assert!(matches!(res, sonic::Response::Empty));+Ok(())}}@@ -107,8 +81,8 @@ impl<'a> WorkerGuard<'a> {}}- fn success(self) {- self.from_pool.insert(Arc::clone(&self.worker));+ async fn success(self) {+ self.from_pool.insert(Arc::clone(&self.worker)).await;}}@@ -158,12 +132,12 @@ impl WorkerPool {self.running_workers.fetch_sub(1, Ordering::SeqCst);}- fn insert(&self, worker: Arc<RemoteWorker>) {- self.ready_workers.lock().unwrap().push(worker);+ async fn insert(&self, worker: Arc<RemoteWorker>) {+ self.ready_workers.lock().await.push(worker);}- fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {- let mut ready_workers = self.ready_workers.lock().unwrap();+ async fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {+ let mut ready_workers = self.ready_workers.lock().await;if ready_workers.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {return Err(Error::NoAvailableWorker);}@@ -176,7 +150,7 @@ impl WorkerPool {}}- fn stop_workers<W, I, O>(&self)+ async fn stop_workers<W, I, O>(&self)whereW: Worker,I: Map<W, O> + Send,@@ -184,7 +158,7 @@ impl WorkerPool {{let mut failing_workers = Vec::new();for worker in &self.all_workers {- if worker.stop::<W, I, O>().is_err() {+ if worker.stop::<W, I, O>().await.is_err() {failing_workers.push(worker);}}@@ -196,6 +170,10 @@ impl WorkerPool {);}}++ fn size(&self) -> usize {+ self.all_workers.len()+ }}pub struct Manager {@@ -212,17 +190,17 @@ impl Manager {}}- fn try_map<W, I, O>(&self, job: &I) -> Result<O>+ async fn try_map<W, I, O>(&self, job: &I) -> Result<O>whereW: Worker,I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{loop {- match self.pool.get_worker()? {+ match self.pool.get_worker().await? {Some(worker) => {- let res = worker.perform(job)?;- worker.success();+ let res = worker.perform(job).await?;+ worker.success().await;return Ok(res);}@@ -233,14 +211,14 @@ impl Manager {/// Execute job on one of the remote machines. If the remote machine fails for some reason,/// the job should be allocated to another machine.- pub fn map<W, I, O>(&self, job: I) -> O+ pub async fn map<W, I, O>(&self, job: I) -> OwhereW: Worker,I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{loop {- match self.try_map(&job) {+ match self.try_map(&job).await {Ok(res) => return res,Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),Err(err) => {@@ -262,82 +240,40 @@ impl Manager {}}- fn reduce_end<O>(acc: Option<O>, elem: O) -> O- where- O: Reduce<O> + Send,- {- match acc {- Some(acc) => acc.reduce(elem),- None => elem,- }- }-#[allow(clippy::trait_duplication_in_bounds)]- fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>+ async fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>whereW: Worker,I: Map<W, O1> + Send,O1: Serialize + DeserializeOwned + Send,O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{- let acc: Arc<Mutex<Option<O2>>> = Arc::new(Mutex::new(None));-- let size = jobs.size_hint();-- match size.1 {- Some(size) => {- let pb = ProgressBar::new(size as u64);- pb.set_style(- ProgressStyle::default_bar()- .template(- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",- )- .progress_chars("#>-"),- );- jobs.par_bridge()- .map(|job| self.map::<W, I, O1>(job))- .progress_with(pb)- .fold(- || None,- |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),- )- .for_each(|res| {- if let Some(res) = res {- let mut lock = acc.lock().unwrap();- *lock = Some(Manager::reduce_end(lock.take(), res));- }- });- }- None => {- jobs.par_bridge()- .map(|job| self.map::<W, I, O1>(job))- .fold(- || None,- |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),- )- .for_each(|res| {- if let Some(res) = res {- let mut lock = acc.lock().unwrap();- *lock = Some(Manager::reduce_end(lock.take(), res));- }- });+ let mut acc = None;++ for chunk in jobs.chunks(self.pool.size()).into_iter() {+ let results = futures::stream::iter(chunk.map(|job| self.map::<W, I, O1>(job)))+ .buffer_unordered(self.pool.size())+ .collect::<Vec<_>>()+ .await;++ for elem in results {+ acc = Some(Self::reduce(acc, elem));}}- let x = acc.lock().unwrap().take();- x+ acc}#[allow(clippy::trait_duplication_in_bounds)]- pub fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>+ pub async fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>whereW: Worker,I: Map<W, O1> + Send,O1: Serialize + DeserializeOwned + Send,O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{- let result = self.get_results(jobs);- self.pool.stop_workers::<W, I, O1>();+ let result = self.get_results(jobs).await;+ self.pool.stop_workers::<W, I, O1>().await;result}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex 570a62e9..c1f1f0f0 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use serde::{de::DeserializeOwned, Deserialize, Serialize};-use std::net::SocketAddr;mod manager;mod worker;@@ -25,15 +24,14 @@ use thiserror::Error;pub use worker::StatelessWorker;pub use worker::Worker;+use crate::sonic;+pub(crate) type Result<T> = std::result::Result<T, Error>;#[derive(Error, Debug)]pub enum Error {- #[error("I/O error")]- IO(#[from] std::io::Error),-- #[error("error while serializing/deserializing to/from bytes")]- Serialization(#[from] bincode::Error),+ #[error("network error")]+ Sonic(#[from] sonic::Error),#[error("could not get a working worker")]NoAvailableWorker,@@ -48,7 +46,7 @@ whereT: Serialize + DeserializeOwned + Send,W: Worker,{- fn map(self, worker: &W) -> T;+ fn map(&self, worker: &W) -> T;}pub trait Reduce<T> {@@ -56,32 +54,6 @@ pub trait Reduce<T> {fn reduce(self, element: T) -> Self;}-#[allow(clippy::trait_duplication_in_bounds)]-pub trait MapReduce<W, I, O1, O2>-where- Self: Sized + Iterator<Item = I> + Send,- W: Worker,- I: Map<W, O1>,- O1: Serialize + DeserializeOwned + Send,- O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,-{- fn map_reduce(self, workers: &[SocketAddr]) -> Option<O2> {- let manager = Manager::new(workers);- manager.run(self)- }-}--#[allow(clippy::trait_duplication_in_bounds)]-impl<W, I, O1, O2, T> MapReduce<W, I, O1, O2> for T-where- W: Worker,- T: Iterator<Item = I> + Sized + Send,- I: Map<W, O1>,- O1: Serialize + DeserializeOwned + Send,- O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,-{-}-#[derive(Serialize, Deserialize, Debug)]enum Task<T> {Job(T),diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rsindex f3c0e641..464057d5 100644--- a/src/mapreduce/worker.rs+++ b/src/mapreduce/worker.rs@@ -1,167 +1,60 @@-use std::{- io::{Read, Write},- net::{SocketAddr, TcpListener},-};--use crate::mapreduce::manager::{BUF_SIZE, END_OF_MESSAGE};--use super::{Error, Map, Result, Task};+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use crate::sonic;++use super::{Map, Result, Task};+use async_trait::async_trait;use serde::{de::DeserializeOwned, Serialize};use tracing::{debug, info};#[derive(Default)]pub struct StatelessWorker {}-pub enum State {- Continue,- Finished,-}-+#[async_trait]pub trait Worker {- fn run_stream<W, I, O, S>(&self, mut stream: S) -> Result<State>+ async fn run<I, O>(&self, addr: SocketAddr) -> Result<()>whereSelf: Sized,- I: Map<Self, O>,+ I: Map<Self, O> + Send + Sync,O: Serialize + DeserializeOwned + Send,- S: Read + Write + Unpin,{- let mut buf = [0; BUF_SIZE];- let mut bytes = Vec::new();+ let server = sonic::Server::bind(addr).await?;+ info!("worker listening on: {:}", addr);+loop {- if let Ok(size) = stream.read(&mut buf) {- debug!("read {:?} bytes", size);- if size == 0 && bytes.is_empty() {- return Err(Error::NoResponse);+ let req: sonic::Request<Task<I>> = server.accept().await?;+ debug!("received request");+ match &req.body {+ Task::Job(job) => {+ debug!("request is a job");+ let res = job.map(self);+ req.respond(sonic::Response::Content(res)).await?;}-- bytes.extend_from_slice(&buf[..size]);-- if bytes.len() >= END_OF_MESSAGE.len()- && bytes[bytes.len() - END_OF_MESSAGE.len()..] == END_OF_MESSAGE- {+ Task::AllFinished => {+ req.respond::<Task<I>>(sonic::Response::Empty).await?;break;}}}- bytes = bytes[..bytes.len() - END_OF_MESSAGE.len()].to_vec();-- match bincode::deserialize::<Task<I>>(&bytes)? {- Task::Job(job) => {- debug!("received job");- let res = job.map(self);- let bytes = bincode::serialize(&res)?;- debug!("serialized result into {} bytes", bytes.len());- stream.write_all(&bytes)?;- stream.write_all(&END_OF_MESSAGE)?;- }- Task::AllFinished => {- debug!("shutting down");- return Ok(State::Finished);- }- };-- Ok(State::Continue)- }-- fn run<I, O>(&self, addr: SocketAddr) -> Result<()>- where- Self: Sized,- I: Map<Self, O>,- O: Serialize + DeserializeOwned + Send,- {- let listener = TcpListener::bind(addr)?;- info!("worker listening on: {:}", addr);-- loop {- let (socket, _) = listener.accept()?;- debug!("received connection");- match self.run_stream::<Self, I, O, _>(socket)? {- State::Finished => break,- State::Continue => {}- }- }-Ok(())}}impl Worker for StatelessWorker {}--#[cfg(test)]-mod tests {- use serde::Deserialize;-- use super::*;-- struct MockTcpStream {- contents: Vec<u8>,- result: Vec<u8>,- num_read: usize,- }-- impl MockTcpStream {- fn new(contents: Vec<u8>) -> Self {- Self {- contents,- num_read: 0,- result: Vec::new(),- }- }- }-- impl Read for MockTcpStream {- fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {- if self.num_read == 0 {- buf.write_all(&self.contents[..]).unwrap();- } else if self.num_read == 1 {- buf.write_all(&END_OF_MESSAGE).unwrap();- }-- self.contents = Vec::new();- self.num_read += 1;- Ok(buf.len())- }- }-- impl Write for MockTcpStream {- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {- self.result.extend(Vec::from(buf));- Ok(buf.len())- }-- fn flush(&mut self) -> std::io::Result<()> {- Ok(())- }- }-- #[derive(Serialize, Deserialize, Debug)]- struct MockJob {- contents: Vec<usize>,- }-- #[derive(Serialize, Deserialize, Debug)]- struct Count(usize);-- impl Map<StatelessWorker, Count> for MockJob {- fn map(self, _worker: &StatelessWorker) -> Count {- Count(self.contents.into_iter().filter(|d| *d == 0).count())- }- }-- #[test]- fn execute() {- let contents = vec![1, 2, 0, 1, 0, 1, 0];- let job = bincode::serialize(&Task::Job(MockJob { contents })).unwrap();-- let mut stream = MockTcpStream::new(job);- StatelessWorker::default()- .run_stream::<StatelessWorker, MockJob, _, _>(&mut stream)- .expect("worker failed");-- let result_bytes = &stream.result[..stream.result.len() - END_OF_MESSAGE.len()];- let res: Count = bincode::deserialize(result_bytes).unwrap();-- assert_eq!(res.0, 3);- }-}diff --git a/src/prehashed.rs b/src/prehashed.rsindex 6d5dd8ec..3165d756 100644--- a/src/prehashed.rs+++ b/src/prehashed.rs@@ -17,6 +17,8 @@ use std::{hash::{BuildHasher, Hash, Hasher},};+use serde::{Deserialize, Serialize};+struct PrehashBuilder {}impl BuildHasher for PrehashBuilder {@@ -45,7 +47,7 @@ impl Hasher for Prehasher {}}-#[derive(Debug, Eq, Clone)]+#[derive(Debug, Eq, Clone, Serialize, Deserialize)]pub struct Prehashed(pub u128);impl Hash for Prehashed {diff --git a/src/ranking/goggles/ast.rs b/src/ranking/goggles/ast.rsindex 1c4dfb0c..cfabaeff 100644--- a/src/ranking/goggles/ast.rs+++ b/src/ranking/goggles/ast.rs@@ -112,7 +112,7 @@ pub enum RawAction {pub fn parse(goggle: &str) -> CrateResult<RawGoggle> {let newlines = Regex::new(r"[\n]+").unwrap();let clean = newlines.replace_all(goggle.trim(), "\n").to_string();- let clean = clean.trim().replace('\n', ";").replace('\r', ";");+ let clean = clean.trim().replace(['\n', '\r'], ";");match PARSER.parse(clean.as_str()) {Ok(blocks) => Ok(RawGoggle::from(blocks)),diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 5b23a169..7fe3f21f 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -294,7 +294,7 @@ mod tests {use crate::{index::Index,schema::create_schema,- searcher::Searcher,+ searcher::LocalSearcher,webpage::{Html, Webpage},};@@ -358,7 +358,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let res = searcher.search("website", None, None, None)@@ -480,7 +480,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let _ = searcher.search(@@ -606,7 +606,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let res = searcher.search(diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex c0c37cc2..5151301b 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -27,6 +27,7 @@ use tantivy::collector::Collector;use crate::{collector::{MaxDocsConsidered, TopDocs},+ inverted_index,searcher::NUM_RESULTS_PER_PAGE,webpage::region::{Region, RegionCount},};@@ -39,6 +40,7 @@ pub struct Ranker {max_docs: Option<MaxDocsConsidered>,offset: Option<usize>,aggregator: SignalAggregator,+ de_rank_similar: bool,}impl Ranker {@@ -49,6 +51,7 @@ impl Ranker {offset: None,aggregator,max_docs: None,+ de_rank_similar: true,}}@@ -70,7 +73,11 @@ impl Ranker {self}- pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {+ pub fn de_rank_similar(&mut self, de_rank_similar: bool) {+ self.de_rank_similar = de_rank_similar;+ }++ pub fn collector(&self) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {let score_tweaker = InitialScoreTweaker::new(Arc::clone(&self.region_count),self.selected_region,@@ -79,6 +86,10 @@ impl Ranker {let mut collector = TopDocs::with_limit(NUM_RESULTS_PER_PAGE);+ if self.de_rank_similar {+ collector = collector.and_de_rank_similar()+ }+if let Some(offset) = self.offset {collector = collector.and_offset(offset);}@@ -95,7 +106,7 @@ impl Ranker {mod tests {use crate::{index::Index,- searcher::Searcher,+ searcher::LocalSearcher,webpage::{Html, Link, Webpage},};@@ -158,7 +169,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("example", None, None, None).expect("Search failed")@@ -226,7 +237,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("example", None, None, None).expect("Search failed")@@ -316,7 +327,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("dr dk", None, None, None).expect("Search failed")@@ -385,7 +396,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("title", None, None, None).expect("Search failed")@@ -459,7 +470,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("test", None, None, None).expect("Search failed")@@ -526,7 +537,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("test", None, None, None).expect("Search failed")@@ -616,7 +627,7 @@ mod tests {index.commit().unwrap();- let searcher = Searcher::new(index, None, None);+ let searcher = LocalSearcher::new(index, None, None);let res = searcher.search(@@ -752,7 +763,7 @@ mod tests {}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::new(index, None, None);+ let searcher = LocalSearcher::new(index, None, None);let result = searcher.search("termA termB", None, None, None)@@ -823,7 +834,7 @@ mod tests {}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::new(index, None, None);+ let searcher = LocalSearcher::new(index, None, None);let result = searcher.search("test", None, None, None)diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex f0644179..28c2d069 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -341,13 +341,7 @@ impl SignalAggregator {Signal::PageCentrality => {(webpage.page_centrality * (CENTRALITY_SCALING as f64)) as u64}- Signal::IsHomepage => {- if webpage.html.url().is_homepage() {- 1- } else {- 0- }- }+ Signal::IsHomepage => webpage.html.url().is_homepage().into(),Signal::FetchTimeMs => webpage.fetch_time_ms,Signal::UpdateTimestamp => webpage.htmldiff --git a/src/search_prettifier.rs b/src/search_prettifier.rsnew file mode 100644index 00000000..1fd4c96a--- /dev/null+++ b/src/search_prettifier.rs@@ -0,0 +1,462 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::collections::BTreeMap;++use chrono::{NaiveDate, NaiveDateTime, Utc};+use itertools::{intersperse, Itertools};+use serde::{Deserialize, Serialize};++use crate::{+ entity_index::{entity::Span, StoredEntity},+ inverted_index::{self, RetrievedWebpage},+ searcher::{self, LocalSearcher},+ webpage::Url,+};++pub fn initial(+ result: searcher::local::InitialWebsiteResult,+ local_searcher: &LocalSearcher,+) -> InitialWebsiteResult {+ let entity = result+ .entity+ .map(|entity| DisplayedEntity::from(entity, local_searcher));++ InitialWebsiteResult {+ spell_corrected_query: result.spell_corrected_query,+ websites: result.websites,+ entity,+ }+}++pub fn retrieve(result: Vec<RetrievedWebpage>, searcher: &LocalSearcher) -> Vec<DisplayedWebpage> {+ result+ .into_iter()+ .map(|mut webpage| {+ webpage.primary_image = webpage.primary_image.and_then(|image| {+ if searcher+ .primary_image(image.uuid.clone().to_string())+ .is_some()+ {+ Some(image)+ } else {+ None+ }+ });++ let url: Url = webpage.url.clone().into();+ webpage.favicon = searcher.favicon(&url.site().to_string().into());+ webpage+ })+ .map(DisplayedWebpage::from)+ .collect()+}++#[derive(Debug, Serialize, Deserialize)]+pub struct InitialWebsiteResult {+ pub spell_corrected_query: Option<String>,+ pub websites: inverted_index::InitialSearchResult,+ pub entity: Option<DisplayedEntity>,+}++pub fn html_escape(s: &str) -> String {+ html_escape::decode_html_entities(s)+ .chars()+ .filter(|c| !matches!(c, '<' | '>' | '&'))+ .collect()+}++#[derive(Debug, Serialize, Deserialize)]+pub struct DisplayedWebpage {+ pub title: String,+ pub url: String,+ pub favicon_base64: String,+ pub domain: String,+ pub pretty_url: String,+ pub snippet: String,+ pub body: String,+ pub primary_image_uuid: Option<String>,+ pub last_updated: Option<String>,+}++const MAX_PRETTY_URL_LEN: usize = 50;+const MAX_TITLE_LEN: usize = 70;++fn prettify_url(url: &Url) -> String {+ let mut pretty_url = url.strip_query().to_string();++ if pretty_url.ends_with('/') {+ pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();+ }++ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";+ pretty_url = Url::from(pretty_url.clone())+ .strip_protocol()+ .replace('/', " › ");+ pretty_url = protocol + &pretty_url;++ if pretty_url.len() > MAX_PRETTY_URL_LEN {+ pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();+ pretty_url += "...";+ }++ pretty_url+}++fn prettify_date(date: NaiveDateTime) -> String {+ let current_time = Utc::now().naive_utc();+ let diff = current_time.signed_duration_since(date);++ let num_hours = diff.num_hours() + 1;+ if num_hours < 24 {+ if num_hours <= 1 {+ return "1 hour ago".to_string();+ }++ return format!("{num_hours} hours ago");+ }++ let num_days = diff.num_days();+ if num_days < 30 {+ if num_days <= 1 {+ return "1 day ago".to_string();+ }++ return format!("{num_days} days ago");+ }++ format!("{}", date.format("%d. %b. %Y"))+}++impl From<RetrievedWebpage> for DisplayedWebpage {+ fn from(webpage: RetrievedWebpage) -> Self {+ let last_updated = webpage.updated_time.map(prettify_date);++ let url: Url = webpage.url.clone().into();+ let domain = url.domain().to_string();+ let pretty_url = prettify_url(&url);++ let mut title = html_escape(&webpage.title);++ if title.len() > MAX_TITLE_LEN {+ title = title.chars().take(MAX_TITLE_LEN).collect();+ title += "...";+ }++ let favicon_bytes = webpage+ .favicon+ .map(|favicon| favicon.as_raw_bytes())+ .unwrap_or_else(|| include_bytes!("../frontend/dist/assets/images/globe.png").to_vec());++ Self {+ title,+ url: webpage.url,+ pretty_url,+ domain,+ favicon_base64: base64::encode(favicon_bytes),+ snippet: webpage.snippet, // snippet has already been html-escaped.+ body: webpage.body,+ primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),+ last_updated,+ }+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct DisplayedEntity {+ pub title: String,+ pub small_abstract: String,+ pub image_base64: Option<String>,+ pub related_entities: Vec<DisplayedEntity>,+ pub info: Vec<(String, String)>,+}++fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {+ let mut info: Vec<_> = info.into_iter().collect();++ info.sort_by(|(a, _), (b, _)| {+ searcher+ .attribute_occurrence(b)+ .unwrap_or(0)+ .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))+ });++ info.into_iter()+ .map(|(key, value)| {+ let mut value = entity_link_to_html(value, 150).replace('*', "•");++ if value.starts_with('•') || value.starts_with("\n•") {+ if let Some(first_bullet) = value.find('•') {+ value = value.chars().skip(first_bullet + 1).collect();+ }+ }++ let value = maybe_prettify_entity_date(value);++ (key.replace('_', " "), value)+ })+ .filter(|(key, _)| {+ !matches!(+ key.as_str(),+ "caption"+ | "image size"+ | "label"+ | "landscape"+ | "signature"+ | "name"+ | "website"+ | "logo"+ | "image caption"+ )+ })+ .take(5)+ .collect()+}++impl DisplayedEntity {+ pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {+ let entity_abstract = Span {+ text: entity.entity_abstract,+ links: entity.links,+ };++ let small_abstract = entity_link_to_html(entity_abstract, 300);++ let image_base64 = entity+ .image+ .and_then(|image| searcher.entity_image(image))+ .map(|image| base64::encode(image.as_raw_bytes()));++ Self {+ title: entity.title,+ small_abstract,+ image_base64,+ related_entities: entity+ .related_entities+ .into_iter()+ .map(|entity| DisplayedEntity::from(entity, searcher))+ .collect(),+ info: prepare_info(entity.info, searcher),+ }+ }+}++fn maybe_prettify_entity_date(value: String) -> String {+ if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {+ return format!("{}", date.format("%d/%m/%Y"));+ }++ if value.split_whitespace().count() == 6 {+ let first_date = NaiveDate::parse_from_str(+ itertools::intersperse(value.split_whitespace().take(3), " ")+ .collect::<String>()+ .as_str(),+ "%Y %-m %-d",+ );+ let second_date = NaiveDate::parse_from_str(+ itertools::intersperse(value.split_whitespace().skip(3), " ")+ .collect::<String>()+ .as_str(),+ "%Y %-m %-d",+ );++ if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {+ // the dates are reversed from the parser, so we need to return second_date before first_date+ return format!("{}", second_date.format("%d/%m/%Y"))+ + " - "+ + format!("{}", first_date.format("%d/%m/%Y")).as_str();+ }+ }++ value+}++pub fn thousand_sep_number(num: usize) -> String {+ let s = num.to_string();+ let c = s.chars().rev().chunks(3);+ let chunks = c.into_iter().map(|chunk| {+ chunk+ .into_iter()+ .collect::<Vec<char>>()+ .into_iter()+ .rev()+ .collect::<String>()+ });++ intersperse(+ chunks.collect::<Vec<_>>().into_iter().rev(),+ ".".to_string(),+ )+ .collect()+}++fn entity_link_to_html(span: Span, trunace_to: usize) -> String {+ let mut s = span.text;++ let truncated = s.len() > trunace_to;+ if truncated {+ s = s.chars().take(trunace_to).collect();+ }++ let chars = s.chars();+ let num_chars = chars.clone().count();++ let mut res = String::new();++ let mut last_link_end = 0;+ for link in span.links {+ if link.start > num_chars {+ break;+ }++ res += chars+ .clone()+ .skip(last_link_end)+ .take(link.start - last_link_end)+ .collect::<String>()+ .as_str();++ let link_text: String = chars+ .clone()+ .skip(link.start)+ .take(link.end - link.start)+ .collect();++ res += format!(+ "<a href=\"https://en.wikipedia.org/wiki/{}\">",+ link.target.replace(' ', "_")+ )+ .as_str();++ res += link_text.as_str();++ res += "</a>";++ last_link_end = link.end;+ }++ res += chars+ .clone()+ .skip(last_link_end)+ .collect::<String>()+ .as_str();++ if truncated {+ res += "...";+ }++ res+}++#[cfg(test)]+mod tests {+ use chrono::{DateTime, Utc};++ use crate::entity_index::entity::Link;++ use super::*;++ #[test]+ fn prettify_date_in_hours() {+ let date = Utc::now().naive_utc();+ assert_eq!(prettify_date(date), "1 hour ago".to_string());++ let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();+ assert_eq!(prettify_date(date), "2 hours ago".to_string());+ }++ #[test]+ fn prettify_date_days() {+ let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();+ assert_eq!(prettify_date(date), "1 day ago".to_string());++ let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();+ assert_eq!(prettify_date(date), "2 days ago".to_string());+ }++ #[test]+ fn prettify_date_rest() {+ let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")+ .unwrap()+ .naive_local();+ assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());+ }++ #[test]+ fn simple_link_to_html() {+ assert_eq!(+ entity_link_to_html(+ Span {+ text: "some text with a link".to_string(),+ links: vec![Link {+ start: 5,+ end: 9,+ target: "text article".to_string()+ }]+ },+ 10000+ ),+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"+ .to_string()+ );+ }++ #[test]+ fn truncated_link_to_html() {+ assert_eq!(+ entity_link_to_html(+ Span {+ text: "some text".to_string(),+ links: vec![Link {+ start: 5,+ end: 9,+ target: "text article".to_string()+ }]+ },+ 7+ ),+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()+ );+ }++ #[test]+ fn einstein_date() {+ assert_eq!(+ maybe_prettify_entity_date("1879 3 14 ".to_string()),+ "14/03/1879".to_string()+ );+ }++ #[test]+ fn entity_date_span_prettify() {+ assert_eq!(+ maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),+ "14/03/1879 - 27/05/1999".to_string()+ );+ }++ #[test]+ fn sep_number() {+ assert_eq!(thousand_sep_number(0), "0".to_string());+ assert_eq!(thousand_sep_number(10), "10".to_string());+ assert_eq!(thousand_sep_number(100), "100".to_string());+ assert_eq!(thousand_sep_number(1000), "1.000".to_string());+ assert_eq!(thousand_sep_number(10_000), "10.000".to_string());+ assert_eq!(thousand_sep_number(100_000), "100.000".to_string());+ assert_eq!(thousand_sep_number(512_854), "512.854".to_string());+ assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());+ }+}diff --git a/src/searcher/distributed.rs b/src/searcher/distributed.rsnew file mode 100644index 00000000..f9e17a72--- /dev/null+++ b/src/searcher/distributed.rs@@ -0,0 +1,612 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ collector::{self, BucketCollector},+ exponential_backoff::ExponentialBackoff,+ inverted_index::{self, RetrievedWebpage},+ search_prettifier::DisplayedWebpage,+ searcher::{+ InitialWebsitesFormatting, PrettifiedWebsitesResult, SearchResult, WebsitesResult,+ NUM_RESULTS_PER_PAGE,+ },+};++use std::{cmp::Ordering, net::SocketAddr, time::Instant};++use futures::stream::FuturesUnordered;+use futures::StreamExt;+use serde::{Deserialize, Serialize};+use thiserror::Error;++use crate::{sonic, webpage::region::Region};++use super::{InitialSearchResult, PrettifiedSearchResult};++type Result<T> = std::result::Result<T, Error>;++struct RemoteSearcher {+ addr: SocketAddr,+}++#[derive(Error, Debug)]+pub enum Error {+ #[error("Failed to get search result")]+ SearchFailed,++ #[error("Query cannot be empty")]+ EmptyQuery,+}++impl RemoteSearcher {+ async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResult> {+ for timeout in ExponentialBackoff::from_millis(30).take(5) {+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await+ {+ if let Ok(sonic::Response::Content(body)) =+ connection.send(Request::Search(query.clone())).await+ {+ return Ok(body);+ }+ }+ }++ Err(Error::SearchFailed)+ }++ async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResult> {+ for timeout in ExponentialBackoff::from_millis(30).take(5) {+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await+ {+ if let Ok(sonic::Response::Content(body)) = connection+ .send(Request::SearchPrettified(query.clone()))+ .await+ {+ return Ok(body);+ }+ }+ }++ Err(Error::SearchFailed)+ }++ async fn retrieve_websites(+ &self,+ pointers: &[inverted_index::WebsitePointer],+ original_query: &str,+ ) -> Result<Vec<RetrievedWebpage>> {+ for timeout in ExponentialBackoff::from_millis(30).take(5) {+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await+ {+ if let Ok(sonic::Response::Content(body)) = connection+ .send(Request::RetrieveWebites {+ websites: pointers.to_vec(),+ query: original_query.to_string(),+ })+ .await+ {+ return Ok(body);+ }+ }+ }++ Err(Error::SearchFailed)+ }++ async fn retrieve_websites_prettified(+ &self,+ pointers: &[inverted_index::WebsitePointer],+ original_query: &str,+ ) -> Result<Vec<DisplayedWebpage>> {+ for timeout in ExponentialBackoff::from_millis(30).take(5) {+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await+ {+ if let Ok(sonic::Response::Content(body)) = connection+ .send(Request::RetrievePrettifiedWebites {+ websites: pointers.to_vec(),+ query: original_query.to_string(),+ })+ .await+ {+ return Ok(body);+ }+ }+ }++ Err(Error::SearchFailed)+ }+}++#[derive(Clone, PartialEq, Eq)]+struct ShardId(u32);++pub struct Shard {+ id: ShardId,+ replicas: Vec<RemoteSearcher>,+}++impl Shard {+ pub fn new(id: u32, replicas: Vec<String>) -> Self {+ let mut parsed_replicas = Vec::new();++ for replica in replicas {+ parsed_replicas.push(RemoteSearcher {+ addr: replica.parse().unwrap(),+ });+ }++ Self {+ id: ShardId(id),+ replicas: parsed_replicas,+ }+ }++ async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {+ match self+ .replicas+ .iter()+ .map(|remote| remote.search(query))+ .collect::<FuturesUnordered<_>>()+ .next()+ .await+ {+ Some(result) => Ok(InitialSearchResultShard {+ local_result: result?,+ shard: self.id.clone(),+ }),+ None => Err(Error::SearchFailed),+ }+ }++ async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {+ match self+ .replicas+ .iter()+ .map(|remote| remote.search_prettified(query))+ .collect::<FuturesUnordered<_>>()+ .next()+ .await+ {+ Some(result) => Ok(InitialSearchResultShard {+ local_result: result?,+ shard: self.id.clone(),+ }),+ None => Err(Error::SearchFailed),+ }+ }++ async fn retrieve_websites(+ &self,+ pointers: &[inverted_index::WebsitePointer],+ original_query: &str,+ ) -> Result<Vec<RetrievedWebpage>> {+ match self+ .replicas+ .iter()+ .map(|remote| remote.retrieve_websites(pointers, original_query))+ .collect::<FuturesUnordered<_>>()+ .next()+ .await+ {+ Some(Ok(websites)) => Ok(websites),+ _ => Err(Error::SearchFailed),+ }+ }++ async fn retrieve_websites_prettified(+ &self,+ pointers: &[inverted_index::WebsitePointer],+ original_query: &str,+ ) -> Result<Vec<DisplayedWebpage>> {+ match self+ .replicas+ .iter()+ .map(|remote| remote.retrieve_websites_prettified(pointers, original_query))+ .collect::<FuturesUnordered<_>>()+ .next()+ .await+ {+ Some(Ok(websites)) => Ok(websites),+ _ => Err(Error::SearchFailed),+ }+ }+}++struct InitialSearchResultShard {+ local_result: InitialSearchResult,+ shard: ShardId,+}++#[derive(Serialize, Deserialize, Clone)]+pub struct SearchQuery {+ pub original: String,+ pub selected_region: Option<Region>,+ pub goggle_program: Option<String>,+ pub skip_pages: Option<usize>,+}++#[derive(Serialize, Deserialize)]+pub enum Request {+ Search(SearchQuery),+ SearchPrettified(SearchQuery),+ RetrieveWebites {+ websites: Vec<inverted_index::WebsitePointer>,+ query: String,+ },+ RetrievePrettifiedWebites {+ websites: Vec<inverted_index::WebsitePointer>,+ query: String,+ },+}++pub struct DistributedSearcher {+ shards: Vec<Shard>,+}++#[derive(Clone)]+struct ScoredWebsitePointer {+ local_pointer: inverted_index::WebsitePointer,+ shard: ShardId,+}++impl collector::Doc for ScoredWebsitePointer {+ fn score(&self) -> &f64 {+ &self.local_pointer.score+ }++ fn key(&self) -> &crate::prehashed::Prehashed {+ &self.local_pointer.site_hash+ }++ fn id(&self) -> &tantivy::DocId {+ &self.local_pointer.address.doc_id+ }++ fn set_score(&mut self, score: f64) {+ self.local_pointer.score = score;+ }++ fn set_key(&mut self, key: crate::prehashed::Prehashed) {+ self.local_pointer.site_hash = key;+ }++ fn set_id(&mut self, id: tantivy::DocId) {+ self.local_pointer.address.doc_id = id;+ }++ fn update_values_from(&mut self, other: &Self) {+ self.set_score(*other.score());+ self.set_key(other.key().clone());+ self.set_id(*other.id());++ self.shard = other.shard.clone();+ }+}++impl PartialOrd for ScoredWebsitePointer {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.local_pointer+ .score+ .partial_cmp(&other.local_pointer.score)+ }+}++impl Ord for ScoredWebsitePointer {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(Ordering::Equal)+ }+}++impl PartialEq for ScoredWebsitePointer {+ fn eq(&self, other: &Self) -> bool {+ self.local_pointer.score == other.local_pointer.score+ }+}++impl Eq for ScoredWebsitePointer {}++impl DistributedSearcher {+ pub fn new(shards: Vec<Shard>) -> Self {+ Self { shards }+ }++ pub async fn search_api(+ &self,+ query: &str,+ selected_region: Option<Region>,+ goggle_program: Option<String>,+ skip_pages: Option<usize>,+ ) -> Result<SearchResult> {+ let start = Instant::now();++ if query.is_empty() {+ return Err(Error::EmptyQuery);+ }++ let query = SearchQuery {+ original: query.to_string(),+ selected_region,+ goggle_program,+ skip_pages,+ };++ // search shards+ let initial_results = self+ .shards+ .iter()+ .map(|shard| shard.search(&query))+ .collect::<FuturesUnordered<_>>()+ .collect::<Vec<_>>()+ .await+ .into_iter()+ .filter_map(|result| result.ok())+ .collect::<Vec<_>>();++ // check if any result has a bang hit+ if let Some(result) = initial_results+ .iter()+ .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))+ {+ if let InitialSearchResult::Bang(bang) = &result.local_result {+ return Ok(SearchResult::Bang(bang.clone()));+ }+ }++ let spell_corrected_query = initial_results.first().and_then(|result| {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =+ &result.local_result+ {+ result.spell_corrected_query.clone()+ } else {+ None+ }+ });++ let entity = initial_results.first().and_then(|result| {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =+ &result.local_result+ {+ result.entity.clone()+ } else {+ None+ }+ });++ let num_docs = initial_results+ .iter()+ .map(|result| {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =+ &result.local_result+ {+ result.websites.num_websites+ } else {+ 0+ }+ })+ .sum();++ // combine results+ let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);+ let mut collector = BucketCollector::new(top_n);++ for result in initial_results {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(local_result)) =+ result.local_result+ {+ for website in local_result.websites.top_websites {+ let pointer = ScoredWebsitePointer {+ local_pointer: website,+ shard: result.shard.clone(),+ };++ collector.insert(pointer);+ }+ }+ }++ let top_websites = collector+ .into_sorted_vec(true)+ .into_iter()+ .skip(skip_pages.unwrap_or(0))+ .take(NUM_RESULTS_PER_PAGE)+ .collect::<Vec<_>>();++ // retrieve webpages+ let mut retrieved_webpages = Vec::new();++ for _ in 0..top_websites.len() {+ retrieved_webpages.push(None);+ }++ for shard in &self.shards {+ let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites+ .iter()+ .enumerate()+ .filter(|(_, pointer)| pointer.shard == shard.id)+ .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))+ .unzip();++ if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {+ for (index, website) in indexes.into_iter().zip(websites.into_iter()) {+ retrieved_webpages[index] = Some(website);+ }+ }+ }++ let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();++ debug_assert_eq!(retrieved_webpages.len(), top_websites.len());++ if retrieved_webpages.is_empty() && !top_websites.is_empty() {+ return Err(Error::SearchFailed);+ }++ Ok(SearchResult::Websites(WebsitesResult {+ spell_corrected_query,+ webpages: inverted_index::SearchResult {+ num_docs,+ documents: retrieved_webpages,+ },+ entity,+ search_duration_ms: start.elapsed().as_millis(),+ }))+ }++ pub async fn search_prettified(+ &self,+ query: &str,+ selected_region: Option<Region>,+ goggle_program: Option<String>,+ skip_pages: Option<usize>,+ ) -> Result<PrettifiedSearchResult> {+ let start = Instant::now();++ if query.is_empty() {+ return Err(Error::EmptyQuery);+ }++ let query = SearchQuery {+ original: query.to_string(),+ selected_region,+ goggle_program,+ skip_pages,+ };++ // search shards+ let initial_results = self+ .shards+ .iter()+ .map(|shard| shard.search_prettified(&query))+ .collect::<FuturesUnordered<_>>()+ .collect::<Vec<_>>()+ .await+ .into_iter()+ .filter_map(|result| result.ok())+ .collect::<Vec<_>>();++ // check if any result has a bang hit+ if let Some(result) = initial_results+ .iter()+ .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))+ {+ if let InitialSearchResult::Bang(bang) = &result.local_result {+ return Ok(PrettifiedSearchResult::Bang(bang.clone()));+ }+ }++ let spell_corrected_query = initial_results.first().and_then(|result| {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =+ &result.local_result+ {+ result.spell_corrected_query.clone()+ } else {+ None+ }+ });++ let entity = initial_results.first().and_then(|result| {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =+ &result.local_result+ {+ result.entity.clone()+ } else {+ None+ }+ });++ let num_docs = initial_results+ .iter()+ .map(|result| {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(+ result,+ )) = &result.local_result+ {+ result.websites.num_websites+ } else {+ 0+ }+ })+ .sum();++ // combine results+ let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);+ let mut collector = BucketCollector::new(top_n);++ for result in initial_results {+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(+ local_result,+ )) = result.local_result+ {+ for website in local_result.websites.top_websites {+ let pointer = ScoredWebsitePointer {+ local_pointer: website,+ shard: result.shard.clone(),+ };++ collector.insert(pointer);+ }+ }+ }++ let top_websites = collector+ .into_sorted_vec(true)+ .into_iter()+ .skip(skip_pages.unwrap_or(0))+ .take(NUM_RESULTS_PER_PAGE)+ .collect::<Vec<_>>();++ // retrieve webpages+ let mut retrieved_webpages = Vec::new();++ for _ in 0..top_websites.len() {+ retrieved_webpages.push(None);+ }++ for shard in &self.shards {+ let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites+ .iter()+ .enumerate()+ .filter(|(_, pointer)| pointer.shard == shard.id)+ .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))+ .unzip();++ if let Ok(websites) = shard+ .retrieve_websites_prettified(&pointers, &query.original)+ .await+ {+ for (index, website) in indexes.into_iter().zip(websites.into_iter()) {+ retrieved_webpages[index] = Some(website);+ }+ }+ }++ let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();++ debug_assert_eq!(retrieved_webpages.len(), top_websites.len());++ if retrieved_webpages.is_empty() && !top_websites.is_empty() {+ return Err(Error::SearchFailed);+ }++ Ok(PrettifiedSearchResult::Websites(PrettifiedWebsitesResult {+ spell_corrected_query,+ num_docs,+ webpages: retrieved_webpages,+ entity,+ search_duration_ms: start.elapsed().as_millis(),+ }))+ }+}diff --git a/src/searcher.rs b/src/searcher/local.rssimilarity index 68%rename from src/searcher.rsrename to src/searcher/local.rsindex 58b9dd20..f49ed41e 100644--- a/src/searcher.rs+++ b/src/searcher/local.rs@@ -17,67 +17,61 @@use std::str::FromStr;use std::time::Instant;-use serde::Serialize;+use serde::{Deserialize, Serialize};use uuid::Uuid;-use crate::bangs::{BangHit, Bangs};+use crate::bangs::Bangs;use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;-use crate::inverted_index::InvertedIndexSearchResult;use crate::query::Query;use crate::ranking::goggles;use crate::ranking::{Ranker, SignalAggregator};use crate::webpage::region::Region;use crate::webpage::Url;-use crate::{Error, Result};+use crate::{inverted_index, Error, Result};-pub const NUM_RESULTS_PER_PAGE: usize = 20;+use super::{+ InitialSearchResult, InitialWebsitesFormatting, SearchResult, WebsitesResult,+ NUM_RESULTS_PER_PAGE,+};-#[derive(Debug, Serialize)]-pub struct WebsitesResult {- pub spell_corrected_query: Option<String>,- pub webpages: InvertedIndexSearchResult,- pub entity: Option<StoredEntity>,- pub search_duration_ms: u128,-}--#[derive(Debug, Serialize)]-pub enum SearchResult {- Websites(WebsitesResult),- Bang(BangHit),-}--pub struct Searcher {+pub struct LocalSearcher {index: Index,entity_index: Option<EntityIndex>,bangs: Option<Bangs>,}-impl From<Index> for Searcher {+impl From<Index> for LocalSearcher {fn from(index: Index) -> Self {Self::new(index, None, None)}}-impl Searcher {+#[derive(Debug, Serialize, Deserialize)]+pub struct InitialWebsiteResult {+ pub spell_corrected_query: Option<String>,+ pub websites: inverted_index::InitialSearchResult,+ pub entity: Option<StoredEntity>,+}++impl LocalSearcher {pub fn new(index: Index, entity_index: Option<EntityIndex>, bangs: Option<Bangs>) -> Self {- Searcher {+ LocalSearcher {index,entity_index,bangs,}}- pub fn search(+ pub fn search_initial(&self,query: &str,selected_region: Option<Region>,goggle_program: Option<String>,skip_pages: Option<usize>,- ) -> Result<SearchResult> {- let start = Instant::now();-+ de_rank_similar: bool,+ ) -> Result<InitialSearchResult> {let raw_query = query.to_string();let goggle = goggle_program.and_then(|program| goggles::parse(&program).ok());@@ -94,16 +88,17 @@ impl Searcher {if query.is_empty() {return Err(Error::EmptyQuery);}- if let Some(goggle) = &goggle {- query.set_goggle(goggle, &self.index.schema());- }if let Some(bangs) = self.bangs.as_ref() {if let Some(bang) = bangs.get(&query) {- return Ok(SearchResult::Bang(bang));+ return Ok(InitialSearchResult::Bang(bang));}}+ if let Some(goggle) = &goggle {+ query.set_goggle(goggle, &self.index.schema());+ }+let mut ranker = Ranker::new(self.index.region_count.clone(),goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),@@ -120,8 +115,9 @@ impl Searcher {}ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());+ ranker.de_rank_similar(de_rank_similar);- let webpages = self.index.search(&query, ranker.collector())?;+ let webpages = self.index.search_initial(&query, ranker.collector())?;let correction = self.index.spell_correction(&query.simple_terms());let entity = self@@ -129,14 +125,64 @@ impl Searcher {.as_ref().and_then(|index| index.search(&raw_query));- let search_duration_ms = start.elapsed().as_millis();+ Ok(InitialSearchResult::Websites(+ InitialWebsitesFormatting::Raw(InitialWebsiteResult {+ spell_corrected_query: correction,+ websites: webpages,+ entity,+ }),+ ))+ }++ pub fn retrieve_websites(+ &self,+ websites: &[inverted_index::WebsitePointer],+ query: &str,+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {+ let query = Query::parse(+ query,+ self.index.schema(),+ self.index.tokenizers(),+ &SignalAggregator::default(),+ )?;++ if query.is_empty() {+ return Err(Error::EmptyQuery);+ }++ self.index.retrieve_websites(websites, &query)+ }- Ok(SearchResult::Websites(WebsitesResult {- webpages,- entity,- spell_corrected_query: correction,- search_duration_ms,- }))+ pub fn search(+ &self,+ query: &str,+ selected_region: Option<Region>,+ goggle_program: Option<String>,+ skip_pages: Option<usize>,+ ) -> Result<SearchResult> {+ let start = Instant::now();++ let initial_result =+ self.search_initial(query, selected_region, goggle_program, skip_pages, true)?;++ match initial_result {+ InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(search_result)) => {+ let retrieved_sites =+ self.retrieve_websites(&search_result.websites.top_websites, query)?;++ Ok(SearchResult::Websites(WebsitesResult {+ spell_corrected_query: search_result.spell_corrected_query,+ webpages: inverted_index::SearchResult {+ num_docs: search_result.websites.num_websites,+ documents: retrieved_sites,+ },+ entity: search_result.entity,+ search_duration_ms: start.elapsed().as_millis(),+ }))+ }+ InitialSearchResult::Bang(bang) => Ok(SearchResult::Bang(bang)),+ _ => unreachable!(),+ }}pub fn favicon(&self, site: &Url) -> Option<Image> {@@ -214,7 +260,7 @@ mod tests {index.commit().unwrap();- let searcher = Searcher::new(index, None, None);+ let searcher = LocalSearcher::new(index, None, None);for p in 0..NUM_PAGES {let urls: Vec<_> = searcherdiff --git a/src/searcher/mod.rs b/src/searcher/mod.rsnew file mode 100644index 00000000..6f9c3f04--- /dev/null+++ b/src/searcher/mod.rs@@ -0,0 +1,72 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod distributed;+pub mod local;++pub use distributed::*;+pub use local::*;+use serde::{Deserialize, Serialize};++use crate::{+ bangs::BangHit,+ entity_index::StoredEntity,+ inverted_index,+ search_prettifier::{self, DisplayedEntity, DisplayedWebpage},+};++pub const NUM_RESULTS_PER_PAGE: usize = 20;++#[derive(Debug, Serialize)]+pub struct WebsitesResult {+ pub spell_corrected_query: Option<String>,+ pub webpages: inverted_index::SearchResult,+ pub entity: Option<StoredEntity>,+ pub search_duration_ms: u128,+}++#[derive(Debug, Serialize)]+pub enum SearchResult {+ Websites(WebsitesResult),+ Bang(BangHit),+}++#[derive(Debug, Serialize)]+pub enum PrettifiedSearchResult {+ Websites(PrettifiedWebsitesResult),+ Bang(BangHit),+}++#[derive(Debug, Serialize)]+pub struct PrettifiedWebsitesResult {+ pub spell_corrected_query: Option<String>,+ pub webpages: Vec<DisplayedWebpage>,+ pub num_docs: usize,+ pub entity: Option<DisplayedEntity>,+ pub search_duration_ms: u128,+}++#[derive(Debug, Serialize, Deserialize)]+pub enum InitialSearchResult {+ Websites(InitialWebsitesFormatting),+ Bang(BangHit),+}++#[derive(Debug, Serialize, Deserialize)]+pub enum InitialWebsitesFormatting {+ Raw(local::InitialWebsiteResult),+ Prettified(search_prettifier::InitialWebsiteResult),+}diff --git a/src/snippet.rs b/src/snippet.rsindex 209fc730..264ca9c4 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::frontend::search::html_escape;+use crate::search_prettifier::html_escape;use crate::tokenizer::Stemmed;use crate::webpage::region::Region;use std::cmp::Ordering;@@ -320,7 +320,7 @@ pub fn generate(#[cfg(test)]mod tests {use super::*;- use crate::{index::Index, searcher::Searcher, webpage::Webpage};+ use crate::{index::Index, searcher::LocalSearcher, webpage::Webpage};use maplit::btreemap;use tantivy::tokenizer::SimpleTokenizer;@@ -360,7 +360,7 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("rust language", None, None, None)@@ -397,7 +397,7 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("describe", None, None, None)@@ -589,7 +589,7 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = Searcher::from(index);+ let searcher = LocalSearcher::from(index);let result = searcher.search("paradigms", None, None, None)diff --git a/src/sonic.rs b/src/sonic.rsnew file mode 100644index 00000000..3f87e570--- /dev/null+++ b/src/sonic.rs@@ -0,0 +1,179 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::time::Duration;++use serde::{de::DeserializeOwned, Deserialize, Serialize};+use thiserror::Error;+use tokio::{+ io::{AsyncReadExt, AsyncWriteExt},+ net::{TcpListener, TcpStream, ToSocketAddrs},+};++type Result<T> = std::result::Result<T, Error>;++#[derive(Error, Debug)]+pub enum Error {+ #[error("Got an IO error")]+ IO(#[from] std::io::Error),++ #[error("Error while serializing/deserializing to/from bytes")]+ Serialization(#[from] bincode::Error),++ #[error("Failed to connect to peer: connection timeout")]+ ConnectionTimeout,+}++#[derive(Serialize, Deserialize)]+pub enum Response<T: Serialize> {+ Empty,+ Content(T),+}++#[repr(C)]+struct Header {+ body_size: usize,+}++pub struct Request<T> {+ stream: TcpStream,+ pub body: T,+}++unsafe fn any_as_u8_slice<T: Sized>(p: &T) -> &[u8] {+ std::slice::from_raw_parts((p as *const T) as *const u8, std::mem::size_of::<T>())+}++impl<T> Request<T> {+ pub async fn respond<R: Serialize>(mut self, response: Response<R>) -> Result<()> {+ let bytes = bincode::serialize(&response)?;+ let header = Header {+ body_size: bytes.len(),+ };++ self.stream+ .write_all(unsafe { any_as_u8_slice(&header) })+ .await?;+ self.stream.write_all(&bytes).await?;+ self.stream.flush().await?;++ self.stream.shutdown().await?;++ Ok(())+ }+}++pub struct Server {+ listener: TcpListener,+}++impl Server {+ pub async fn bind(addr: impl ToSocketAddrs) -> Result<Self> {+ let listener = TcpListener::bind(addr).await?;+ Ok(Self { listener })+ }++ pub async fn accept<T>(&self) -> Result<Request<T>>+ where+ T: Serialize + DeserializeOwned,+ {+ let (mut stream, client) = self.listener.accept().await?;+ tracing::debug!("accepted connection from: {}", &client);++ let mut header_buf = vec![0; std::mem::size_of::<Header>()];+ stream.read_exact(&mut header_buf).await?;+ let header: Header = unsafe { std::ptr::read(header_buf.as_ptr() as *const _) };++ let mut buf = vec![0; header.body_size];++ stream.read_exact(&mut buf).await?;+ tracing::debug!("received bytes: {:?}", &buf);++ let body = bincode::deserialize(&buf)?;++ Ok(Request { stream, body })+ }+}++pub struct Connection {+ stream: TcpStream,+}++impl Connection {+ pub async fn create(server: impl ToSocketAddrs) -> Result<Self> {+ Self::create_with_timeout(server, Duration::from_secs(30)).await+ }++ pub async fn create_with_timeout(+ server: impl ToSocketAddrs,+ timeout: Duration,+ ) -> Result<Self> {+ match tokio::time::timeout(timeout, TcpStream::connect(server)).await {+ Ok(stream) => {+ let stream = stream?;+ Ok(Connection { stream })+ }+ Err(_) => Err(Error::ConnectionTimeout),+ }+ }++ pub async fn send_without_timeout<T: Serialize, R: DeserializeOwned + Serialize>(+ mut self,+ request: T,+ ) -> Result<Response<R>> {+ let bytes = bincode::serialize(&request)?;++ let header = Header {+ body_size: bytes.len(),+ };++ self.stream+ .write_all(unsafe { any_as_u8_slice(&header) })+ .await?;+ self.stream.write_all(&bytes).await?;+ self.stream.flush().await?;++ let mut header_buf = vec![0; std::mem::size_of::<Header>()];+ self.stream.read_exact(&mut header_buf).await?;+ let header: Header = unsafe { std::ptr::read(header_buf.as_ptr() as *const _) };++ let mut buf = vec![0; header.body_size];+ self.stream.read_exact(&mut buf).await?;++ self.stream.shutdown().await?;++ Ok(bincode::deserialize(&buf)?)+ }++ pub async fn send<T: Serialize, R: DeserializeOwned + Serialize>(+ self,+ request: T,+ ) -> Result<Response<R>> {+ self.send_with_timeout(request, Duration::from_secs(30))+ .await+ }++ pub async fn send_with_timeout<T: Serialize, R: DeserializeOwned + Serialize>(+ self,+ request: T,+ timeout: Duration,+ ) -> Result<Response<R>> {+ match tokio::time::timeout(timeout, self.send_without_timeout(request)).await {+ Ok(res) => res,+ Err(_) => Err(Error::ConnectionTimeout),+ }+ }+}diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rsindex 4459ffee..5cbf8cce 100644--- a/src/spell/dictionary.rs+++ b/src/spell/dictionary.rs@@ -297,11 +297,10 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {let searcher = searcher.unwrap();let mut matches = self.perform_search(&searcher);- let dist_metric = LevenshteinDistance::new(edit_distance);while let Some((correction, freq)) = matches.next() {let correction = std::str::from_utf8(correction).unwrap().to_owned();- let dist = dist_metric.compare(term, &correction);+ let dist = LevenshteinDistance::compare(term, &correction);let prob = freq as f64 / self.total_freq as f64;diff --git a/src/spell/distance.rs b/src/spell/distance.rsindex 515a0ae4..5317f439 100644--- a/src/spell/distance.rs+++ b/src/spell/distance.rs@@ -39,11 +39,11 @@ impl LevenshteinDistance {////// The runtime complexity is `O(m*n)`, where `m` and `n` are the/// strings' lengths.- pub fn compare(&self, a: &str, b: &str) -> usize {+ pub fn compare(a: &str, b: &str) -> usize {let len_a = a.chars().count();let len_b = b.chars().count();if len_a < len_b {- return self.compare(b, a);+ return Self::compare(b, a);}// handle special case of 0 lengthif len_a == 0 {diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 352b3380..089eceee 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -627,7 +627,7 @@ impl Html {}}Field::IsHomepage => {- doc.add_u64(tantivy_field, if self.url().is_homepage() { 1 } else { 0 });+ doc.add_u64(tantivy_field, self.url().is_homepage().into());}Field::LastUpdated => doc.add_u64(tantivy_field,diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex 650a7785..bdb1e191 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -16,10 +16,10 @@use std::{fmt::Display, time::Duration};-use serde::Serialize;+use serde::{Deserialize, Serialize};use tracing::debug;-#[derive(Debug, Serialize, PartialEq, Eq, Clone, Hash)]+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]pub struct Url(String);impl Display for Url {@@ -71,7 +71,7 @@ impl Url {pub fn domain(&self) -> &str {let site = self.site();- let num_punctuations: usize = site.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();+ let num_punctuations: usize = site.chars().map(|c| (c == '.') as usize).sum();if num_punctuations > 1 {let domain_index = site.rfind('.').unwrap();let mut start_index = site[..domain_index].rfind('.').unwrap() + 1;