Skip to content

web search done right

Ftr/distributed search (#59)

Mikkel Denker3cc7c84a3299
diff --git a/Cargo.lock b/Cargo.lock
index 5a97aca8..5d383a45 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -127,9 +127,9 @@ dependencies = [
[[package]]
name = "async-trait"
-version = "0.1.56"
+version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716"
+checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f"
dependencies = [
"proc-macro2",
"quote",
@@ -674,6 +674,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"askama",
+ "async-trait",
"axum",
"axum-extra",
"base64",
diff --git a/Cargo.toml b/Cargo.toml
index 386b2918..fa4d1402 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -57,6 +57,7 @@ serde_urlencoded = "0.7.1"
rust-stemmers = "1.2.0"
min-max-heap = "1.3.0"
base64 = "0.13.0"
+async-trait = "0.1.57"
[dev-dependencies]
criterion = "0.3.6"
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rs
index 85ae6c87..9258cdc5 100644
--- a/benches/search-preindexed-goggle.rs
+++ b/benches/search-preindexed-goggle.rs
@@ -1,5 +1,5 @@
use criterion::{criterion_group, criterion_main, Criterion};
-use cuely::{index::Index, searcher::Searcher};
+use cuely::{index::Index, searcher::LocalSearcher};
const INDEX_PATH: &str = "data/index";
@@ -21,7 +21,7 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
- let searcher = Searcher::new(index, None, None);
+ let searcher = LocalSearcher::new(index, None, None);
let goggle = include_str!("../testcases/goggles/hacker_news.goggle");
// for _ in 0..10 {
diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rs
index 0033eca2..bbcdee67 100644
--- a/benches/search-preindexed.rs
+++ b/benches/search-preindexed.rs
@@ -1,5 +1,5 @@
use criterion::{criterion_group, criterion_main, Criterion};
-use cuely::{index::Index, searcher::Searcher};
+use cuely::{index::Index, searcher::LocalSearcher};
const INDEX_PATH: &str = "data/index";
@@ -16,7 +16,7 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
- let searcher = Searcher::new(index, None, None);
+ let searcher = LocalSearcher::new(index, None, None);
// for _ in 0..100 {
bench!("the", searcher, c);
diff --git a/configs/frontend.toml b/configs/frontend.toml
new file mode 100644
index 00000000..11006110
--- /dev/null
+++ b/configs/frontend.toml
@@ -0,0 +1,3 @@
+queries_csv_path = "data/queries_us.csv"
+host = "0.0.0.0:3000"
+search_servers = [["0.0.0.0:3001"]]
\ No newline at end of file
diff --git a/configs/search_server.toml b/configs/search_server.toml
new file mode 100644
index 00000000..089bf194
--- /dev/null
+++ b/configs/search_server.toml
@@ -0,0 +1,4 @@
+host = "0.0.0.0:3001"
+index_path = "data/index"
+entity_index_path = "data/entity"
+bangs_path = "data/bangs.json"
\ No newline at end of file
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
index f8206863..e13fa4a5 100644
--- a/examples/mapreduce_manager.rs
+++ b/examples/mapreduce_manager.rs
@@ -1,6 +1,6 @@
use std::net::SocketAddr;
-use cuely::mapreduce::{Map, MapReduce, Reduce, StatelessWorker};
+use cuely::mapreduce::{Manager, Map, Reduce, StatelessWorker};
use serde::{Deserialize, Serialize};
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -14,7 +14,7 @@ struct Job {
struct Count(usize);
impl Map<StatelessWorker, Count> for Job {
- fn map(self, _worker: &StatelessWorker) -> Count {
+ fn map(&self, _worker: &StatelessWorker) -> Count {
std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(1)
}
@@ -26,7 +26,8 @@ impl Reduce<Count> for Count {
}
}
-fn main() {
+#[tokio::main]
+async fn main() {
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG)
.finish();
@@ -41,9 +42,8 @@ fn main() {
Job { id: 5 },
];
- let res: Count = jobs
- .into_iter()
- .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])
- .unwrap();
+ let manager = Manager::new(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()]);
+ let res: Count = manager.run(jobs.into_iter()).await.unwrap();
+
println!("{:?}", res.0);
}
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
index fdca5e9c..9d136537 100644
--- a/examples/mapreduce_worker.rs
+++ b/examples/mapreduce_worker.rs
@@ -14,7 +14,7 @@ struct Job {
struct Count(usize);
impl Map<StatelessWorker, Count> for Job {
- fn map(self, _worker: &StatelessWorker) -> Count {
+ fn map(&self, _worker: &StatelessWorker) -> Count {
std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(1)
}
@@ -26,7 +26,8 @@ impl Reduce<Count> for Count {
}
}
-fn main() {
+#[tokio::main]
+async fn main() {
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG)
.finish();
@@ -36,5 +37,6 @@ fn main() {
StatelessWorker::default()
.run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())
+ .await
.expect("failed to run worker");
}
diff --git a/justfile b/justfile
index 1c52da5d..2d323990 100644
--- a/justfile
+++ b/justfile
@@ -21,7 +21,7 @@
# time ((cd frontend; npm run build) && cargo build)
# cargo run -- frontend data/index data/queries_us.csv data/entity data/bangs.json
cd frontend; npm run build
- cargo run -- frontend data/index data/queries_us.csv data/entity data/bangs.json
+ bash scripts/run_frontend.sh
@frontend:
cd frontend; npm install
diff --git a/nohup.out b/nohup.out
new file mode 100644
index 00000000..0da8ba7d
--- /dev/null
+++ b/nohup.out
@@ -0,0 +1,20 @@
+ Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)
+ Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)
+ Finished dev [unoptimized + debuginfo] target(s) in 4.40s
+ Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`
+2022-09-28T09:43:26.587009Z  INFO cuely::entrypoint::frontend: listening on 0.0.0.0:3000
+ Blocking waiting for file lock on package cache
+ Blocking waiting for file lock on package cache
+ Blocking waiting for file lock on package cache
+ Blocking waiting for file lock on build directory
+ Finished dev [unoptimized + debuginfo] target(s) in 4.31s
+ Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`
+thread 'main' panicked at 'unable to open rocks db: Error { message: "IO error: While lock file: data/entity/attribute_occurrences/LOCK: Resource temporarily unavailable" }', src/kv/rocksdb_store.rs:39:41
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+ Blocking waiting for file lock on package cache
+ Blocking waiting for file lock on package cache
+ Blocking waiting for file lock on package cache
+ Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)
+ Finished dev [unoptimized + debuginfo] target(s) in 4.22s
+ Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`
+2022-09-28T09:44:05.809083Z  INFO cuely::entrypoint::frontend: listening on 0.0.0.0:3000
diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.sh
new file mode 100644
index 00000000..f7c69417
--- /dev/null
+++ b/scripts/run_frontend.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/bash
+trap 'kill $(jobs -p)' EXIT
+cargo run -- search-server configs/search_server.toml &
+cargo run -- frontend configs/frontend.toml
diff --git a/src/bangs.rs b/src/bangs.rs
index 21a86c22..18a41903 100644
--- a/src/bangs.rs
+++ b/src/bangs.rs
@@ -50,7 +50,7 @@ pub struct Bang {
url: String,
}
-#[derive(Debug, Serialize, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
pub struct BangHit {
bang: Bang,
pub redirect_to: Url,
diff --git a/src/collector.rs b/src/collector.rs
index 6df5752c..833c4969 100644
--- a/src/collector.rs
+++ b/src/collector.rs
@@ -18,10 +18,11 @@ use min_max_heap::MinMaxHeap;
use tantivy::{
collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},
fastfield::MultiValuedFastFieldReader,
- DocAddress, DocId, Score, SegmentOrdinal, SegmentReader,
+ DocId, Score, SegmentOrdinal, SegmentReader,
};
use crate::{
+ inverted_index::{DocAddress, WebsitePointer},
prehashed::{combine_u64s, PrehashMap, Prehashed},
schema::Field,
};
@@ -37,10 +38,23 @@ pub struct MaxDocsConsidered {
pub segments: usize,
}
+pub trait Doc: PartialEq + Eq + PartialOrd + Ord + Clone {
+ fn score(&self) -> &f64;
+ fn key(&self) -> &Prehashed;
+ fn id(&self) -> &DocId;
+
+ fn set_score(&mut self, score: f64);
+ fn set_key(&mut self, key: Prehashed);
+ fn set_id(&mut self, id: DocId);
+
+ fn update_values_from(&mut self, other: &Self);
+}
+
pub struct TopDocs {
top_n: usize,
offset: usize,
max_docs: Option<MaxDocsConsidered>,
+ de_rank_similar: bool,
}
impl TopDocs {
@@ -49,6 +63,7 @@ impl TopDocs {
top_n,
offset: 0,
max_docs: None,
+ de_rank_similar: false,
}
}
@@ -63,10 +78,16 @@ impl TopDocs {
self
}
+ pub fn and_de_rank_similar(mut self) -> Self {
+ self.de_rank_similar = true;
+
+ self
+ }
+
pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(
self,
score_tweaker: TScoreTweaker,
- ) -> impl Collector<Fruit = Vec<(f64, DocAddress)>>
+ ) -> impl Collector<Fruit = Vec<WebsitePointer>>
where
TScoreSegmentTweaker: ScoreSegmentTweaker<f64> + 'static,
TScoreTweaker: ScoreTweaker<f64, Child = TScoreSegmentTweaker> + Send + Sync,
@@ -76,7 +97,7 @@ impl TopDocs {
}
impl Collector for TopDocs {
- type Fruit = Vec<(f64, DocAddress)>;
+ type Fruit = Vec<WebsitePointer>;
type Child = TopSegmentCollector;
@@ -123,17 +144,16 @@ impl Collector for TopDocs {
}
Ok(collector
- .into_sorted_vec(true)
+ .into_sorted_vec(self.de_rank_similar)
.into_iter()
.skip(self.offset)
- .map(|doc| {
- (
- doc.score,
- DocAddress {
- segment_ord: doc.segment,
- doc_id: doc.id,
- },
- )
+ .map(|doc| WebsitePointer {
+ score: doc.score,
+ site_hash: doc.key,
+ address: DocAddress {
+ segment: doc.segment,
+ doc_id: doc.id,
+ },
})
.collect())
}
@@ -144,11 +164,11 @@ pub struct TopSegmentCollector {
max_docs: Option<usize>,
num_docs_taken: usize,
segment_ord: SegmentOrdinal,
- bucket_collector: BucketCollector,
+ bucket_collector: BucketCollector<SegmentDoc>,
}
impl SegmentCollector for TopSegmentCollector {
- type Fruit = Vec<Doc>;
+ type Fruit = Vec<SegmentDoc>;
fn collect(&mut self, doc: DocId, score: Score) {
if let Some(max_docs) = &self.max_docs {
@@ -165,7 +185,7 @@ impl SegmentCollector for TopSegmentCollector {
let keys = [keys[0], keys[1]];
let key = combine_u64s(keys);
- self.bucket_collector.insert(Doc {
+ self.bucket_collector.insert(SegmentDoc {
key: Prehashed(key),
id: doc,
segment: self.segment_ord,
@@ -178,13 +198,13 @@ impl SegmentCollector for TopSegmentCollector {
}
}
-struct BucketCollector {
- buckets: PrehashMap<Bucket>,
+pub struct BucketCollector<T: Doc> {
+ buckets: PrehashMap<Bucket<T>>,
heads: MinMaxHeap<BucketHead>,
top_n: usize,
}
-impl BucketCollector {
+impl<T: Doc> BucketCollector<T> {
pub fn new(top_n: usize) -> Self {
assert!(top_n > 0);
@@ -195,18 +215,18 @@ impl BucketCollector {
}
}
- pub fn insert(&mut self, doc: Doc) {
- if let Some(bucket) = self.buckets.get_mut(&doc.key) {
+ pub fn insert(&mut self, doc: T) {
+ if let Some(bucket) = self.buckets.get_mut(doc.key()) {
bucket.insert(doc);
} else {
let mut bucket = Bucket::new(self.top_n);
bucket.insert(doc.clone());
- self.buckets.insert(doc.key.clone(), bucket);
+ self.buckets.insert(doc.key().clone(), bucket);
self.heads.push(BucketHead {
- key: doc.key,
- tweaked_score: doc.score,
+ key: doc.key().clone(),
+ tweaked_score: *doc.score(),
});
}
@@ -263,7 +283,7 @@ impl BucketCollector {
bucket_heads
}
- pub fn into_sorted_vec(mut self, apply_adjust_score: bool) -> Vec<Doc> {
+ pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {
let mut res = Vec::new();
let mut bucket_heads = self.build_heads();
@@ -272,8 +292,8 @@ impl BucketCollector {
let bucket = self.buckets.get_mut(&head.key).unwrap();
if let Some(mut doc) = bucket.pop_best() {
- if apply_adjust_score {
- doc.score = adjust_score(bucket.num_taken - 1, doc.score);
+ if de_rank_similar {
+ doc.set_score(adjust_score(bucket.num_taken - 1, *doc.score()));
}
res.push(doc);
@@ -319,13 +339,13 @@ impl Ord for BucketHead {
}
}
-struct Bucket {
+struct Bucket<T: Doc> {
num_taken: usize,
- docs: MinMaxHeap<Doc>,
+ docs: MinMaxHeap<T>,
top_n: usize,
}
-impl Bucket {
+impl<T: Doc> Bucket<T> {
pub fn new(top_n: usize) -> Self {
assert!(top_n > 0);
@@ -336,7 +356,7 @@ impl Bucket {
}
}
- pub fn pop_best(&mut self) -> Option<Doc> {
+ pub fn pop_best(&mut self) -> Option<T> {
let res = self.docs.pop_max();
self.num_taken += 1;
@@ -346,18 +366,16 @@ impl Bucket {
pub fn get_best(&self) -> Option<TweakedDoc<'_>> {
self.docs.peek_max().map(|doc| TweakedDoc {
- tweaked_score: adjust_score(self.num_taken, doc.score),
- _doc_id: &doc.id,
+ tweaked_score: adjust_score(self.num_taken, *doc.score()),
+ _doc_id: doc.id(),
})
}
- pub fn insert(&mut self, doc: Doc) {
+ pub fn insert(&mut self, doc: T) {
if self.docs.len() >= self.top_n {
let mut worst = self.docs.peek_min_mut().unwrap();
- worst.id = doc.id;
- worst.score = doc.score;
- worst.segment = doc.segment;
+ worst.update_values_from(&doc);
} else {
self.docs.push(doc);
}
@@ -370,33 +388,67 @@ struct TweakedDoc<'a> {
}
#[derive(Debug, Clone)]
-pub struct Doc {
+pub struct SegmentDoc {
key: Prehashed,
id: DocId,
segment: SegmentOrdinal,
score: f64,
}
-impl PartialEq for Doc {
+impl PartialEq for SegmentDoc {
fn eq(&self, other: &Self) -> bool {
self.score == other.score
}
}
-impl Eq for Doc {}
+impl Eq for SegmentDoc {}
-impl PartialOrd for Doc {
+impl PartialOrd for SegmentDoc {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.score.partial_cmp(&other.score)
}
}
-impl Ord for Doc {
+impl Ord for SegmentDoc {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
}
}
+impl Doc for SegmentDoc {
+ fn score(&self) -> &f64 {
+ &self.score
+ }
+
+ fn key(&self) -> &Prehashed {
+ &self.key
+ }
+
+ fn id(&self) -> &DocId {
+ &self.id
+ }
+
+ fn set_score(&mut self, score: f64) {
+ self.score = score;
+ }
+
+ fn set_key(&mut self, key: Prehashed) {
+ self.key = key;
+ }
+
+ fn set_id(&mut self, id: DocId) {
+ self.id = id;
+ }
+
+ fn update_values_from(&mut self, other: &Self) {
+ self.set_id(*other.id());
+ self.set_key(other.key().clone());
+ self.set_score(*other.score());
+
+ self.segment = other.segment;
+ }
+}
+
pub(crate) struct TweakedScoreTopCollector<TScoreTweaker> {
score_tweaker: TScoreTweaker,
collector: TopDocs,
@@ -418,7 +470,7 @@ impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>
where
TScoreTweaker: ScoreTweaker<f64> + Send + Sync,
{
- type Fruit = Vec<(f64, DocAddress)>;
+ type Fruit = Vec<WebsitePointer>;
type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child>;
@@ -462,7 +514,7 @@ impl<TSegmentScoreTweaker> SegmentCollector
where
TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<f64>,
{
- type Fruit = Vec<Doc>;
+ type Fruit = Vec<SegmentDoc>;
fn collect(&mut self, doc: DocId, score: Score) {
let score = self.segment_scorer.score(doc, score);
@@ -482,7 +534,7 @@ mod tests {
let mut collector = BucketCollector::new(top_n);
for doc in docs {
- collector.insert(Doc {
+ collector.insert(SegmentDoc {
key: Prehashed(doc.0),
id: doc.1,
score: doc.2,
diff --git a/src/entity_index/entity.rs b/src/entity_index/entity.rs
index 81a82053..f143c0f4 100644
--- a/src/entity_index/entity.rs
+++ b/src/entity_index/entity.rs
@@ -37,7 +37,7 @@ pub struct Entity {
pub categories: HashSet<String>,
}
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Span {
pub text: String,
pub links: Vec<Link>,
diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rs
index cd23cca2..c8a5b632 100644
--- a/src/entity_index/mod.rs
+++ b/src/entity_index/mod.rs
@@ -22,7 +22,7 @@ use std::{
time::Duration,
};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use tantivy::{
collector::TopDocs,
query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},
@@ -145,7 +145,7 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {
]
}
-#[derive(Debug, Serialize, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct StoredEntity {
pub title: String,
pub entity_abstract: String,
diff --git a/src/entrypoint/frontend.rs b/src/entrypoint/frontend.rs
index fbe0a63b..a7137f6e 100644
--- a/src/entrypoint/frontend.rs
+++ b/src/entrypoint/frontend.rs
@@ -18,14 +18,8 @@ use anyhow::Result;
use crate::frontend::router;
-pub async fn run(
- index_path: &str,
- queries_csv_path: &str,
- entity_index_path: Option<String>,
- bangs_path: Option<String>,
- host: &str,
-) -> Result<()> {
- let app = router(index_path, queries_csv_path, entity_index_path, bangs_path)?;
+pub async fn run(queries_csv_path: &str, host: &str, shards: Vec<Vec<String>>) -> Result<()> {
+ let app = router(queries_csv_path, shards)?;
let addr = host.parse()?;
tracing::info!("listening on {}", addr);
axum::Server::bind(&addr)
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 8306c2a1..4619dceb 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -23,7 +23,7 @@ use tracing::{debug, info, trace};
use crate::entrypoint::download_all_warc_files;
use crate::index::{FrozenIndex, Index};
-use crate::mapreduce::{Map, MapReduce, Reduce, Worker};
+use crate::mapreduce::{Manager, Map, Reduce, Worker};
use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::SignalAggregator;
use crate::warc::WarcFile;
@@ -171,15 +171,15 @@ struct IndexPointer(String);
impl Worker for IndexingWorker {}
impl Map<IndexingWorker, FrozenIndex> for Job {
- fn map(self, worker: &IndexingWorker) -> FrozenIndex {
- let index = process_job(&self, worker);
+ fn map(&self, worker: &IndexingWorker) -> FrozenIndex {
+ let index = process_job(self, worker);
index.into()
}
}
impl Map<IndexingWorker, IndexPointer> for Job {
- fn map(self, worker: &IndexingWorker) -> IndexPointer {
- let index = process_job(&self, worker);
+ fn map(&self, worker: &IndexingWorker) -> IndexPointer {
+ let index = process_job(self, worker);
IndexPointer(index.path)
}
}
@@ -227,54 +227,59 @@ impl Reduce<Index> for Index {
impl Indexer {
pub fn run_master(config: &IndexingMasterConfig) -> Result<()> {
- info!("Running master for index construction");
-
- let warc_paths = config.warc_source.paths()?;
-
- let workers: Vec<SocketAddr> = config
- .workers
- .iter()
- .map(|worker| worker.parse().unwrap())
- .collect();
-
- let job_config = match config.warc_source.clone() {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- };
-
- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
- warc_paths
- .into_iter()
- .chunks(config.batch_size.unwrap_or(1))
- .into_iter()
- .map(|warc_paths| Job {
- source_config: job_config.clone(),
- warc_paths: warc_paths.collect_vec(),
- download_images: config.download_images.unwrap_or(true),
- base_path: config
- .index_base_path
- .clone()
- .unwrap_or_else(|| "data/index".to_string()),
- })
- .collect_vec()
- .into_iter(),
- );
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ info!("Running master for index construction");
+
+ let warc_paths = config.warc_source.paths().unwrap();
+
+ let workers: Vec<SocketAddr> = config
+ .workers
+ .iter()
+ .map(|worker| worker.parse().unwrap())
+ .collect();
+
+ let job_config = match config.warc_source.clone() {
+ WarcSource::HTTP(config) => JobConfig::Http(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ };
- if let Some(limit) = config.limit_warc_files {
- warc_paths = Box::new(warc_paths.take(limit));
- }
+ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
+ warc_paths
+ .into_iter()
+ .chunks(config.batch_size.unwrap_or(1))
+ .into_iter()
+ .map(|warc_paths| Job {
+ source_config: job_config.clone(),
+ warc_paths: warc_paths.collect_vec(),
+ download_images: config.download_images.unwrap_or(true),
+ base_path: config
+ .index_base_path
+ .clone()
+ .unwrap_or_else(|| "data/index".to_string()),
+ })
+ .collect_vec()
+ .into_iter(),
+ );
+
+ if let Some(limit) = config.limit_warc_files {
+ warc_paths = Box::new(warc_paths.take(limit));
+ }
- let mut index = <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<
- IndexingWorker,
- Job,
- FrozenIndex,
- Index,
- >>::map_reduce(warc_paths, &workers)
- .expect("failed to build index");
+ let manager = Manager::new(&workers);
+ let mut index: Index = manager
+ .run::<IndexingWorker, Job, FrozenIndex, Index>(warc_paths)
+ .await
+ .unwrap();
- index
- .inverted_index
- .merge_into_segments(config.final_num_segments.unwrap_or(20))?;
+ index
+ .inverted_index
+ .merge_into_segments(config.final_num_segments.unwrap_or(20))
+ .unwrap();
+ });
Ok(())
}
@@ -284,11 +289,20 @@ impl Indexer {
centrality_store_path: String,
webgraph_path: Option<String>,
) -> Result<()> {
- IndexingWorker::new(centrality_store_path, webgraph_path).run::<Job, FrozenIndex>(
- worker_addr
- .parse::<SocketAddr>()
- .expect("Could not parse worker address"),
- )?;
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ IndexingWorker::new(centrality_store_path, webgraph_path)
+ .run::<Job, FrozenIndex>(
+ worker_addr
+ .parse::<SocketAddr>()
+ .expect("Could not parse worker address"),
+ )
+ .await
+ .unwrap();
+ });
Ok(())
}
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index c4da2518..ba8fcfcd 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -17,6 +17,7 @@ mod centrality;
mod entity;
pub mod frontend;
mod indexer;
+pub mod search_server;
mod webgraph;
use std::{fs::File, path::Path};
diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rs
new file mode 100644
index 00000000..63f90317
--- /dev/null
+++ b/src/entrypoint/search_server.rs
@@ -0,0 +1,131 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::SocketAddr;
+
+use crate::{
+ bangs::Bangs,
+ entity_index::EntityIndex,
+ index::Index,
+ inverted_index,
+ search_prettifier::{self},
+ searcher::{self, LocalSearcher},
+ sonic, Result, SearchServerConfig,
+};
+
+pub async fn run(config: SearchServerConfig) -> Result<()> {
+ let addr: SocketAddr = config.host.parse().unwrap();
+ let server = sonic::Server::bind(addr).await.unwrap();
+ tracing::info!("listening on {}", addr);
+
+ let entity_index = config
+ .entity_index_path
+ .map(|path| EntityIndex::open(path).unwrap());
+ let bangs = config.bangs_path.map(Bangs::from_path);
+ let search_index = Index::open(config.index_path)?;
+
+ let local_searcher = LocalSearcher::new(search_index, entity_index, bangs);
+
+ loop {
+ if let Ok(req) = server.accept::<searcher::distributed::Request>().await {
+ match &req.body {
+ searcher::Request::Search(search) => {
+ match local_searcher.search_initial(
+ &search.original,
+ search.selected_region,
+ search.goggle_program.clone(),
+ search.skip_pages,
+ false,
+ ) {
+ Ok(response) => {
+ req.respond(sonic::Response::Content(response)).await.ok();
+ }
+ Err(_) => {
+ req.respond::<searcher::InitialSearchResult>(sonic::Response::Empty)
+ .await
+ .ok();
+ }
+ }
+ }
+ searcher::Request::RetrieveWebites { websites, query } => {
+ match local_searcher.retrieve_websites(websites, query) {
+ Ok(response) => {
+ req.respond(sonic::Response::Content(response)).await.ok();
+ }
+ Err(_) => {
+ req.respond::<Vec<inverted_index::RetrievedWebpage>>(
+ sonic::Response::Empty,
+ )
+ .await
+ .ok();
+ }
+ }
+ }
+ searcher::Request::SearchPrettified(search) => {
+ match local_searcher.search_initial(
+ &search.original,
+ search.selected_region,
+ search.goggle_program.clone(),
+ search.skip_pages,
+ false,
+ ) {
+ Ok(result) => match result {
+ searcher::InitialSearchResult::Websites(
+ searcher::InitialWebsitesFormatting::Raw(result),
+ ) => {
+ let res = search_prettifier::initial(result, &local_searcher);
+
+ req.respond(sonic::Response::Content(
+ searcher::InitialSearchResult::Websites(
+ searcher::InitialWebsitesFormatting::Prettified(res),
+ ),
+ ))
+ .await
+ .ok();
+ }
+ searcher::InitialSearchResult::Bang(_) => todo!(),
+ _ => unreachable!(),
+ },
+ Err(_) => {
+ req.respond::<inverted_index::SearchResult>(sonic::Response::Empty)
+ .await
+ .ok();
+ }
+ }
+ }
+ searcher::Request::RetrievePrettifiedWebites { websites, query } => {
+ match local_searcher.retrieve_websites(websites, query) {
+ Ok(result) => {
+ req.respond(sonic::Response::Content(search_prettifier::retrieve(
+ result,
+ &local_searcher,
+ )))
+ .await
+ .ok();
+ }
+ Err(_) => {
+ req.respond::<Vec<search_prettifier::DisplayedWebpage>>(
+ sonic::Response::Empty,
+ )
+ .await
+ .ok();
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 928aa02b..10eee39b 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -16,7 +16,7 @@
use crate::{
directory::DirEntry,
entrypoint::download_all_warc_files,
- mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},
+ mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
@@ -97,15 +97,15 @@ fn process_job(job: &Job) -> webgraph::Webgraph {
}
impl Map<StatelessWorker, FrozenWebgraph> for Job {
- fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {
- let graph = process_job(&self);
+ fn map(&self, _worker: &StatelessWorker) -> FrozenWebgraph {
+ let graph = process_job(self);
graph.into()
}
}
impl Map<StatelessWorker, GraphPointer> for Job {
- fn map(self, _worker: &StatelessWorker) -> GraphPointer {
- let graph = process_job(&self);
+ fn map(&self, _worker: &StatelessWorker) -> GraphPointer {
+ let graph = process_job(self);
GraphPointer(graph.path)
}
}
@@ -161,60 +161,74 @@ pub struct Webgraph {}
impl Webgraph {
pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
- info!("Running master for webgraph construction");
-
- let warc_paths = config.warc_source.paths()?;
-
- let workers: Vec<SocketAddr> = config
- .workers
- .iter()
- .map(|worker| worker.parse().unwrap())
- .collect();
-
- let job_config = match config.warc_source.clone() {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- };
-
- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
- warc_paths
- .into_iter()
- .chunks(config.batch_size.unwrap_or(1))
- .into_iter()
- .map(|warc_paths| Job {
- config: job_config.clone(),
- warc_paths: warc_paths.into_iter().collect(),
- graph_base_path: config
- .graph_base_path
- .clone()
- .unwrap_or_else(|| "data/webgraph".to_string()),
- })
- .collect::<Vec<_>>()
- .into_iter(),
- );
-
- if let Some(limit) = config.limit_warc_files {
- warc_paths = Box::new(warc_paths.take(limit));
- }
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ info!("Running master for webgraph construction");
+
+ let warc_paths = config.warc_source.paths().unwrap();
+
+ let workers: Vec<SocketAddr> = config
+ .workers
+ .iter()
+ .map(|worker| worker.parse().unwrap())
+ .collect();
+
+ let job_config = match config.warc_source.clone() {
+ WarcSource::HTTP(config) => JobConfig::Http(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ };
+
+ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
+ warc_paths
+ .into_iter()
+ .chunks(config.batch_size.unwrap_or(1))
+ .into_iter()
+ .map(|warc_paths| Job {
+ config: job_config.clone(),
+ warc_paths: warc_paths.into_iter().collect(),
+ graph_base_path: config
+ .graph_base_path
+ .clone()
+ .unwrap_or_else(|| "data/webgraph".to_string()),
+ })
+ .collect::<Vec<_>>()
+ .into_iter(),
+ );
+
+ if let Some(limit) = config.limit_warc_files {
+ warc_paths = Box::new(warc_paths.take(limit));
+ }
- let _graph: webgraph::Webgraph =
- <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<
- StatelessWorker,
- Job,
- webgraph::FrozenWebgraph,
- webgraph::Webgraph,
- >>::map_reduce(warc_paths, &workers)
- .expect("failed to build webgraph");
+ let manager = Manager::new(&workers);
+ let _graph: webgraph::Webgraph = manager
+ .run::<StatelessWorker, Job, webgraph::FrozenWebgraph, webgraph::Webgraph>(
+ warc_paths,
+ )
+ .await
+ .unwrap();
+ });
Ok(())
}
pub fn run_worker(worker_addr: String) -> Result<()> {
- StatelessWorker::default().run::<Job, FrozenWebgraph>(
- worker_addr
- .parse::<SocketAddr>()
- .expect("Could not parse worker address"),
- )?;
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async {
+ StatelessWorker::default()
+ .run::<Job, FrozenWebgraph>(
+ worker_addr
+ .parse::<SocketAddr>()
+ .expect("Could not parse worker address"),
+ )
+ .await
+ .unwrap();
+ });
Ok(())
}
diff --git a/src/frontend/api.rs b/src/frontend/api.rs
index 97333707..f935af6d 100644
--- a/src/frontend/api.rs
+++ b/src/frontend/api.rs
@@ -40,7 +40,8 @@ pub async fn search(
match state
.searcher
- .search(query.as_str(), selected_region, None, skip_pages)
+ .search_api(query.as_str(), selected_region, None, skip_pages)
+ .await
{
Ok(result) => Json(result),
Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
diff --git a/src/frontend/entity_image.rs b/src/frontend/entity_image.rs
deleted file mode 100644
index 49afd79d..00000000
--- a/src/frontend/entity_image.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::sync::Arc;
-
-use axum::body::Body;
-use axum::extract::Path;
-use axum::http::Response;
-use axum::response::IntoResponse;
-use axum::Extension;
-use reqwest::StatusCode;
-
-use super::State;
-
-#[allow(clippy::unused_async)]
-pub async fn route(
- Path(entity): Path<String>,
- Extension(state): Extension<Arc<State>>,
-) -> impl IntoResponse {
- let img = state.searcher.entity_image(entity);
-
- let bytes = match img {
- Some(img) => img.as_raw_bytes(),
- None => Vec::new(),
- };
-
- Response::builder()
- .status(StatusCode::OK)
- .body(Body::from(bytes))
- .unwrap()
-}
diff --git a/src/frontend/favicons.rs b/src/frontend/favicons.rs
deleted file mode 100644
index 43d4001f..00000000
--- a/src/frontend/favicons.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::sync::Arc;
-
-use axum::body::Body;
-use axum::extract::Path;
-use axum::http::Response;
-use axum::response::IntoResponse;
-use axum::Extension;
-use reqwest::StatusCode;
-
-use crate::webpage::Url;
-
-use super::State;
-
-#[allow(clippy::unused_async)]
-pub async fn route(
- Path(site): Path<String>,
- Extension(state): Extension<Arc<State>>,
-) -> impl IntoResponse {
- let img = state.searcher.favicon(&Url::from(site));
-
- let bytes = match img {
- Some(img) => img.as_raw_bytes(),
- None => include_bytes!("../../frontend/dist/assets/images/globe.png").to_vec(),
- };
-
- Response::builder()
- .status(StatusCode::OK)
- .body(Body::from(bytes))
- .unwrap()
-}
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index 1a3a6f49..a000196b 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -18,8 +18,8 @@ use axum::{body::Body, Extension, Router};
use tower_http::compression::CompressionLayer;
use crate::{
- autosuggest::Autosuggest, bangs::Bangs, entity_index::EntityIndex, index::Index,
- searcher::Searcher,
+ autosuggest::Autosuggest,
+ searcher::{DistributedSearcher, Shard},
};
use anyhow::Result;
use std::sync::Arc;
@@ -35,18 +35,15 @@ use axum_extra::routing::SpaRouter;
mod about;
mod api;
mod autosuggest;
-mod entity_image;
-mod favicons;
mod goggles;
mod index;
-mod primary_image;
mod privacy;
pub mod search;
pub struct HtmlTemplate<T>(T);
pub struct State {
- pub searcher: Searcher,
+ pub searcher: DistributedSearcher,
pub autosuggest: Autosuggest,
}
@@ -76,17 +73,15 @@ pub async fn favicon() -> impl IntoResponse {
.unwrap()
}
-pub fn router(
- index_path: &str,
- queries_csv_path: &str,
- entity_index_path: Option<String>,
- bangs_path: Option<String>,
-) -> Result<Router> {
- let entity_index = entity_index_path.map(|path| EntityIndex::open(path).unwrap());
- let bangs = bangs_path.map(Bangs::from_path);
- let search_index = Index::open(index_path)?;
+pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router> {
+ let shards: Vec<_> = shards
+ .into_iter()
+ .enumerate()
+ .map(|(id, replicas)| Shard::new(id as u32, replicas))
+ .collect();
+
let autosuggest = Autosuggest::load_csv(queries_csv_path)?;
- let searcher = Searcher::new(search_index, entity_index, bangs);
+ let searcher = DistributedSearcher::new(shards);
let state = Arc::new(State {
searcher,
@@ -97,9 +92,6 @@ pub fn router(
.route("/", get(index::route))
.route("/search", get(search::route))
.route("/autosuggest", get(autosuggest::route))
- .route("/favicons/:site", get(favicons::route))
- .route("/image/:uuid", get(primary_image::route))
- .route("/entity/image/:entity", get(entity_image::route))
.route("/favicon.ico", get(favicon))
.route("/about", get(about::route))
.route("/goggles", get(goggles::route))
diff --git a/src/frontend/primary_image.rs b/src/frontend/primary_image.rs
deleted file mode 100644
index cb19cb6b..00000000
--- a/src/frontend/primary_image.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::sync::Arc;
-
-use axum::body::Body;
-use axum::extract::Path;
-use axum::http::Response;
-use axum::response::IntoResponse;
-use axum::Extension;
-use reqwest::StatusCode;
-
-use super::State;
-
-#[allow(clippy::unused_async)]
-pub async fn route(
- Path(uuid): Path<String>,
- Extension(state): Extension<Arc<State>>,
-) -> impl IntoResponse {
- let img = state.searcher.primary_image(uuid);
-
- match img {
- Some(img) => Response::builder()
- .status(StatusCode::OK)
- .body(Body::from(img.as_raw_bytes()))
- .unwrap(),
- None => Response::builder()
- .status(StatusCode::NOT_FOUND)
- .body(Body::empty())
- .unwrap(),
- }
-}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index a257a46d..d326921c 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -14,23 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::{collections::HashMap, sync::Arc};
+
use axum::Extension;
-use chrono::{NaiveDate, NaiveDateTime, Utc};
-use itertools::{intersperse, Itertools};
-use serde::Serialize;
use crate::{
- entity_index::{entity::Span, StoredEntity},
- inverted_index::RetrievedWebpage,
- searcher::{SearchResult, Searcher},
- webpage::{
- region::{Region, ALL_REGIONS},
- Url,
- },
- Error,
+ search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},
+ searcher::{self, PrettifiedSearchResult},
+ webpage::region::{Region, ALL_REGIONS},
};
-use std::collections::{BTreeMap, HashMap};
-use std::sync::Arc;
use super::{
goggles::{GoggleLink, DEFAULT_GOGGLES},
@@ -42,219 +34,6 @@ use axum::{
response::{IntoResponse, Redirect},
};
-pub fn html_escape(s: &str) -> String {
- html_escape::decode_html_entities(s)
- .chars()
- .filter(|c| !matches!(c, '<' | '>' | '&'))
- .collect()
-}
-
-pub struct DisplayedWebpage {
- pub title: String,
- pub url: String,
- pub favicon_base64: String,
- pub domain: String,
- pub pretty_url: String,
- pub snippet: String,
- pub body: String,
- pub primary_image_uuid: Option<String>,
- pub last_updated: Option<String>,
-}
-
-const MAX_PRETTY_URL_LEN: usize = 50;
-const MAX_TITLE_LEN: usize = 70;
-
-fn prettify_url(url: &Url) -> String {
- let mut pretty_url = url.strip_query().to_string();
-
- if pretty_url.ends_with('/') {
- pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
- }
-
- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
- pretty_url = Url::from(pretty_url.clone())
- .strip_protocol()
- .replace('/', " › ");
- pretty_url = protocol + &pretty_url;
-
- if pretty_url.len() > MAX_PRETTY_URL_LEN {
- pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();
- pretty_url += "...";
- }
-
- pretty_url
-}
-
-fn prettify_date(date: NaiveDateTime) -> String {
- let current_time = Utc::now().naive_utc();
- let diff = current_time.signed_duration_since(date);
-
- let num_hours = diff.num_hours() + 1;
- if num_hours < 24 {
- if num_hours <= 1 {
- return "1 hour ago".to_string();
- }
-
- return format!("{num_hours} hours ago");
- }
-
- let num_days = diff.num_days();
- if num_days < 30 {
- if num_days <= 1 {
- return "1 day ago".to_string();
- }
-
- return format!("{num_days} days ago");
- }
-
- format!("{}", date.format("%d. %b. %Y"))
-}
-
-impl From<RetrievedWebpage> for DisplayedWebpage {
- fn from(webpage: RetrievedWebpage) -> Self {
- let last_updated = webpage.updated_time.map(prettify_date);
-
- let url: Url = webpage.url.clone().into();
- let domain = url.domain().to_string();
- let pretty_url = prettify_url(&url);
-
- let mut title = html_escape(&webpage.title);
-
- if title.len() > MAX_TITLE_LEN {
- title = title.chars().take(MAX_TITLE_LEN).collect();
- title += "...";
- }
-
- let favicon_bytes = webpage
- .favicon
- .map(|favicon| favicon.as_raw_bytes())
- .unwrap_or_else(|| {
- include_bytes!("../../frontend/dist/assets/images/globe.png").to_vec()
- });
-
- Self {
- title,
- url: webpage.url,
- pretty_url,
- domain,
- favicon_base64: base64::encode(favicon_bytes),
- snippet: webpage.snippet, // snippet has already been html-escaped.
- body: webpage.body,
- primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
- last_updated,
- }
- }
-}
-
-#[derive(Serialize)]
-pub struct DisplayedEntity {
- pub title: String,
- pub small_abstract: String,
- pub image_base64: Option<String>,
- pub related_entities: Vec<DisplayedEntity>,
- pub info: Vec<(String, String)>,
-}
-
-fn prepare_info(info: BTreeMap<String, Span>, searcher: &Searcher) -> Vec<(String, String)> {
- let mut info: Vec<_> = info.into_iter().collect();
-
- info.sort_by(|(a, _), (b, _)| {
- searcher
- .attribute_occurrence(b)
- .unwrap_or(0)
- .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))
- });
-
- info.into_iter()
- .map(|(key, value)| {
- let mut value = entity_link_to_html(value, 150).replace('*', "•");
-
- if value.starts_with('•') || value.starts_with("\n•") {
- if let Some(first_bullet) = value.find('•') {
- value = value.chars().skip(first_bullet + 1).collect();
- }
- }
-
- let value = maybe_prettify_entity_date(value);
-
- (key.replace('_', " "), value)
- })
- .filter(|(key, _)| {
- !matches!(
- key.as_str(),
- "caption"
- | "image size"
- | "label"
- | "landscape"
- | "signature"
- | "name"
- | "website"
- | "logo"
- | "image caption"
- )
- })
- .take(5)
- .collect()
-}
-
-impl DisplayedEntity {
- fn from(entity: StoredEntity, searcher: &Searcher) -> Self {
- let entity_abstract = Span {
- text: entity.entity_abstract,
- links: entity.links,
- };
-
- let small_abstract = entity_link_to_html(entity_abstract, 300);
-
- let image_base64 = entity
- .image
- .and_then(|image| searcher.entity_image(image))
- .map(|image| base64::encode(image.as_raw_bytes()));
-
- Self {
- title: entity.title,
- small_abstract,
- image_base64,
- related_entities: entity
- .related_entities
- .into_iter()
- .map(|entity| DisplayedEntity::from(entity, searcher))
- .collect(),
- info: prepare_info(entity.info, searcher),
- }
- }
-}
-
-fn maybe_prettify_entity_date(value: String) -> String {
- if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {
- return format!("{}", date.format("%d/%m/%Y"));
- }
-
- if value.split_whitespace().count() == 6 {
- let first_date = NaiveDate::parse_from_str(
- itertools::intersperse(value.split_whitespace().take(3), " ")
- .collect::<String>()
- .as_str(),
- "%Y %-m %-d",
- );
- let second_date = NaiveDate::parse_from_str(
- itertools::intersperse(value.split_whitespace().skip(3), " ")
- .collect::<String>()
- .as_str(),
- "%Y %-m %-d",
- );
-
- if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {
- // the dates are reversed from the parser, so we need to return second_date before first_date
- return format!("{}", second_date.format("%d/%m/%Y"))
- + " - "
- + format!("{}", first_date.format("%d/%m/%Y")).as_str();
- }
- }
-
- value
-}
-
#[derive(Template)]
#[template(path = "search/index.html", escape = "none")]
struct SearchTemplate {
@@ -312,40 +91,15 @@ pub async fn route(
match state
.searcher
- .search(query.as_str(), selected_region, goggle, skip_pages)
+ .search_prettified(query.as_str(), selected_region, goggle, skip_pages)
+ .await
{
Ok(result) => match result {
- SearchResult::Websites(result) => {
- let search_result = result
- .webpages
- .documents
- .into_iter()
- .map(|mut webpage| {
- webpage.primary_image = webpage.primary_image.and_then(|image| {
- if state
- .searcher
- .primary_image(image.uuid.clone().to_string())
- .is_some()
- {
- Some(image)
- } else {
- None
- }
- });
-
- let url: Url = webpage.url.clone().into();
- webpage.favicon = state.searcher.favicon(&url.site().to_string().into());
- webpage
- })
- .map(DisplayedWebpage::from)
- .collect();
-
- let entity = result
- .entity
- .map(|entity| DisplayedEntity::from(entity, &state.searcher));
+ PrettifiedSearchResult::Websites(result) => {
+ let entity = result.entity;
let spell_correction = result.spell_corrected_query;
- let num_matches = thousand_sep_number(result.webpages.num_docs);
+ let num_matches = thousand_sep_number(result.num_docs);
let search_duration_sec =
format!("{:.2}", result.search_duration_ms as f64 / 1000.0);
@@ -391,7 +145,7 @@ pub async fn route(
};
let template = SearchTemplate {
- search_result,
+ search_result: result.webpages,
query,
entity,
spell_correction,
@@ -407,186 +161,11 @@ pub async fn route(
HtmlTemplate(template).into_response()
}
- SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),
+ PrettifiedSearchResult::Bang(result) => {
+ Redirect::to(&result.redirect_to.full()).into_response()
+ }
},
- Err(Error::EmptyQuery) => Redirect::to("/").into_response(),
+ Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),
Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
}
}
-
-fn thousand_sep_number(num: usize) -> String {
- let s = num.to_string();
- let c = s.chars().rev().chunks(3);
- let chunks = c.into_iter().map(|chunk| {
- chunk
- .into_iter()
- .collect::<Vec<char>>()
- .into_iter()
- .rev()
- .collect::<String>()
- });
-
- intersperse(
- chunks.collect::<Vec<_>>().into_iter().rev(),
- ".".to_string(),
- )
- .collect()
-}
-
-fn entity_link_to_html(span: Span, trunace_to: usize) -> String {
- let mut s = span.text;
-
- let truncated = s.len() > trunace_to;
- if truncated {
- s = s.chars().take(trunace_to).collect();
- }
-
- let chars = s.chars();
- let num_chars = chars.clone().count();
-
- let mut res = String::new();
-
- let mut last_link_end = 0;
- for link in span.links {
- if link.start > num_chars {
- break;
- }
-
- res += chars
- .clone()
- .skip(last_link_end)
- .take(link.start - last_link_end)
- .collect::<String>()
- .as_str();
-
- let link_text: String = chars
- .clone()
- .skip(link.start)
- .take(link.end - link.start)
- .collect();
-
- res += format!(
- "<a href=\"https://en.wikipedia.org/wiki/{}\">",
- link.target.replace(' ', "_")
- )
- .as_str();
-
- res += link_text.as_str();
-
- res += "</a>";
-
- last_link_end = link.end;
- }
-
- res += chars
- .clone()
- .skip(last_link_end)
- .collect::<String>()
- .as_str();
-
- if truncated {
- res += "...";
- }
-
- res
-}
-
-#[cfg(test)]
-mod tests {
- use chrono::{DateTime, Utc};
-
- use crate::entity_index::entity::Link;
-
- use super::*;
-
- #[test]
- fn prettify_date_in_hours() {
- let date = Utc::now().naive_utc();
- assert_eq!(prettify_date(date), "1 hour ago".to_string());
-
- let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();
- assert_eq!(prettify_date(date), "2 hours ago".to_string());
- }
-
- #[test]
- fn prettify_date_days() {
- let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();
- assert_eq!(prettify_date(date), "1 day ago".to_string());
-
- let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();
- assert_eq!(prettify_date(date), "2 days ago".to_string());
- }
-
- #[test]
- fn prettify_date_rest() {
- let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
- .unwrap()
- .naive_local();
- assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());
- }
-
- #[test]
- fn simple_link_to_html() {
- assert_eq!(
- entity_link_to_html(
- Span {
- text: "some text with a link".to_string(),
- links: vec![Link {
- start: 5,
- end: 9,
- target: "text article".to_string()
- }]
- },
- 10000
- ),
- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
- .to_string()
- );
- }
-
- #[test]
- fn truncated_link_to_html() {
- assert_eq!(
- entity_link_to_html(
- Span {
- text: "some text".to_string(),
- links: vec![Link {
- start: 5,
- end: 9,
- target: "text article".to_string()
- }]
- },
- 7
- ),
- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
- );
- }
-
- #[test]
- fn einstein_date() {
- assert_eq!(
- maybe_prettify_entity_date("1879 3 14 ".to_string()),
- "14/03/1879".to_string()
- );
- }
-
- #[test]
- fn entity_date_span_prettify() {
- assert_eq!(
- maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),
- "14/03/1879 - 27/05/1999".to_string()
- );
- }
-
- #[test]
- fn sep_number() {
- assert_eq!(thousand_sep_number(0), "0".to_string());
- assert_eq!(thousand_sep_number(10), "10".to_string());
- assert_eq!(thousand_sep_number(100), "100".to_string());
- assert_eq!(thousand_sep_number(1000), "1.000".to_string());
- assert_eq!(thousand_sep_number(10_000), "10.000".to_string());
- assert_eq!(thousand_sep_number(100_000), "100.000".to_string());
- assert_eq!(thousand_sep_number(512_854), "512.854".to_string());
- assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());
- }
-}
diff --git a/src/index.rs b/src/index.rs
index 19dd506c..85d8aa83 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -30,7 +30,7 @@ use uuid::Uuid;
use crate::directory::{self, DirEntry};
use crate::image_downloader::{ImageDownloadJob, ImageDownloader};
use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
-use crate::inverted_index::{InvertedIndex, InvertedIndexSearchResult};
+use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};
use crate::query::Query;
use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};
use crate::webpage::region::{Region, RegionCount};
@@ -111,13 +111,28 @@ impl Index {
Ok(())
}
- pub fn search<C>(&self, query: &Query, collector: C) -> Result<InvertedIndexSearchResult>
+ pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>
where
- C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,
+ C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
{
self.inverted_index.search(query, collector)
}
+ pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>
+ where
+ C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
+ {
+ self.inverted_index.search_initial(query, collector)
+ }
+
+ pub fn retrieve_websites(
+ &self,
+ websites: &[inverted_index::WebsitePointer],
+ query: &Query,
+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {
+ self.inverted_index.retrieve_websites(websites, query)
+ }
+
pub fn merge(mut self, other: Self) -> Self {
self.inverted_index.merge(other.inverted_index);
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index b9e29cc1..f9aac627 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -15,15 +15,16 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use chrono::NaiveDateTime;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use tantivy::collector::{Collector, Count};
use tantivy::directory::MmapDirectory;
use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
-use tantivy::{DocAddress, Document, IndexReader, IndexWriter, SegmentMeta};
+use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
use crate::image_store::Image;
+use crate::prehashed::Prehashed;
use crate::query::Query;
use crate::schema::{Field, ALL_FIELDS};
use crate::snippet;
@@ -35,6 +36,43 @@ use std::fs;
use std::path::Path;
use std::sync::Arc;
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InitialSearchResult {
+ pub num_websites: usize,
+ pub top_websites: Vec<WebsitePointer>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WebsitePointer {
+ pub score: f64,
+ pub site_hash: Prehashed,
+ pub address: DocAddress,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, Copy)]
+pub struct DocAddress {
+ pub segment: u32,
+ pub doc_id: u32,
+}
+
+impl From<tantivy::DocAddress> for DocAddress {
+ fn from(address: tantivy::DocAddress) -> Self {
+ Self {
+ segment: address.segment_ord,
+ doc_id: address.doc_id,
+ }
+ }
+}
+
+impl From<DocAddress> for tantivy::DocAddress {
+ fn from(address: DocAddress) -> Self {
+ Self {
+ segment_ord: address.segment,
+ doc_id: address.doc_id,
+ }
+ }
+}
+
struct SegmentMergeCandidate {
num_docs: u32,
segments: Vec<SegmentMeta>,
@@ -118,17 +156,28 @@ impl InvertedIndex {
Ok(())
}
- pub fn search<C>(&self, query: &Query, collector: C) -> Result<InvertedIndexSearchResult>
+ pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>
where
- C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,
+ C: Collector<Fruit = Vec<WebsitePointer>>,
{
let searcher = self.reader.searcher();
let (count, docs) = searcher.search(query, &(Count, collector))?;
+ Ok(InitialSearchResult {
+ num_websites: count,
+ top_websites: docs,
+ })
+ }
- let mut webpages: Vec<RetrievedWebpage> = docs
- .into_iter()
- .map(|(_score, doc_address)| self.retrieve_doc(doc_address, &searcher))
+ pub fn retrieve_websites(
+ &self,
+ websites: &[WebsitePointer],
+ query: &Query,
+ ) -> Result<Vec<RetrievedWebpage>> {
+ let searcher = self.reader.searcher();
+ let mut webpages: Vec<RetrievedWebpage> = websites
+ .iter()
+ .map(|website| self.retrieve_doc(website.address, &searcher))
.filter_map(|page| page.ok())
.map(|mut doc| {
if let Some(image) = doc.primary_image.as_ref() {
@@ -159,9 +208,19 @@ impl InvertedIndex {
)?;
}
- Ok(InvertedIndexSearchResult {
- num_docs: count,
- documents: webpages,
+ Ok(webpages)
+ }
+
+ pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>
+ where
+ C: Collector<Fruit = Vec<WebsitePointer>>,
+ {
+ let initial_result = self.search_initial(query, collector)?;
+ let websites = self.retrieve_websites(&initial_result.top_websites, query)?;
+
+ Ok(SearchResult {
+ num_docs: initial_result.num_websites,
+ documents: websites,
})
}
@@ -222,7 +281,7 @@ impl InvertedIndex {
doc_address: DocAddress,
searcher: &tantivy::Searcher,
) -> Result<RetrievedWebpage> {
- let doc = searcher.doc(doc_address)?;
+ let doc = searcher.doc(doc_address.into())?;
Ok(RetrievedWebpage::from(doc))
}
@@ -289,12 +348,12 @@ impl InvertedIndex {
}
#[derive(Debug, Serialize)]
-pub struct InvertedIndexSearchResult {
+pub struct SearchResult {
pub num_docs: usize,
pub documents: Vec<RetrievedWebpage>,
}
-#[derive(Default, Debug, Serialize)]
+#[derive(Default, Debug, Serialize, Deserialize)]
pub struct RetrievedWebpage {
pub title: String,
pub url: String,
diff --git a/src/lib.rs b/src/lib.rs
index 7e5311be..193bf000 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -48,8 +48,10 @@ mod query;
mod ranking;
mod schema;
mod schema_org;
+mod search_prettifier;
pub mod searcher;
mod snippet;
+mod sonic;
mod spell;
mod tokenizer;
mod warc;
@@ -134,6 +136,21 @@ pub struct HttpConfig {
warc_paths_file: String,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct FrontendConfig {
+ pub queries_csv_path: String,
+ pub host: String,
+ pub search_servers: Vec<Vec<String>>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct SearchServerConfig {
+ pub index_path: String,
+ pub entity_index_path: Option<String>,
+ pub bangs_path: Option<String>,
+ pub host: String,
+}
+
#[derive(Error, Debug)]
pub enum Error {
#[error("Failed to download object from HTTP")]
diff --git a/src/main.rs b/src/main.rs
index 643c1009..03865e9a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,7 +15,8 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
use clap::{Parser, Subcommand};
-use cuely::entrypoint::{self, frontend};
+use cuely::entrypoint::{self, frontend, search_server};
+use cuely::{FrontendConfig, SearchServerConfig};
use serde::de::DeserializeOwned;
use std::fs;
use std::path::Path;
@@ -44,13 +45,11 @@ enum Commands {
#[clap(subcommand)]
options: WebgraphOptions,
},
+ SearchServer {
+ config_path: String,
+ },
Frontend {
- index_path: String,
- queries_csv_path: String,
- entity_index_path: Option<String>,
- bangs_path: Option<String>,
- #[clap(default_value = "0.0.0.0:3000")]
- host: String,
+ config_path: String,
},
}
@@ -132,22 +131,26 @@ fn main() -> Result<()> {
entrypoint::Webgraph::run_locally(&config)?;
}
},
- Commands::Frontend {
- index_path,
- queries_csv_path,
- host,
- entity_index_path,
- bangs_path,
- } => tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()?
- .block_on(frontend::run(
- &index_path,
- &queries_csv_path,
- entity_index_path,
- bangs_path,
- &host,
- ))?,
+ Commands::Frontend { config_path } => {
+ let config: FrontendConfig = load_toml_config(&config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(frontend::run(
+ &config.queries_csv_path,
+ &config.host,
+ config.search_servers,
+ ))?
+ }
+ Commands::SearchServer { config_path } => {
+ let config: SearchServerConfig = load_toml_config(&config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(search_server::run(config))?
+ }
}
Ok(())
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 8ebea4c5..ea322173 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -2,39 +2,35 @@ use super::{Error, Result, Worker};
use super::{Map, Reduce};
use crate::exponential_backoff::ExponentialBackoff;
use crate::mapreduce::Task;
-use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle};
-use rayon::iter::ParallelBridge;
-use rayon::prelude::*;
+use crate::sonic;
+use futures::StreamExt;
+use itertools::Itertools;
use serde::de::DeserializeOwned;
use serde::Serialize;
-use std::io::{Read, Write};
+use std::net::SocketAddr;
use std::net::ToSocketAddrs;
-use std::net::{SocketAddr, TcpStream};
use std::ops::Deref;
use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
use std::time::Duration;
+use tokio::sync::Mutex;
use tracing::{debug, warn};
#[derive(Debug)]
struct RemoteWorker {
addr: SocketAddr,
}
-pub const BUF_SIZE: usize = 4096;
-
-pub const END_OF_MESSAGE: [u8; BUF_SIZE] = [42; BUF_SIZE];
impl RemoteWorker {
fn retry_strategy() -> impl Iterator<Item = Duration> {
ExponentialBackoff::from_millis(10).take(5)
}
- fn connect(&self) -> Result<TcpStream> {
- debug!("connecting to {:}", self.addr);
+ async fn connect(&self) -> Result<sonic::Connection> {
for dur in RemoteWorker::retry_strategy() {
- if let Ok(stream) = TcpStream::connect(&self.addr) {
+ if let Ok(conn) = sonic::Connection::create(self.addr).await {
debug!("connected");
- return Ok(stream);
+ return Ok(conn);
}
std::thread::sleep(dur);
@@ -43,53 +39,31 @@ impl RemoteWorker {
Err(Error::NoResponse)
}
- fn perform<W, I, O>(&self, job: &I) -> Result<O>
+ async fn perform<W, I, O>(&self, job: &I) -> Result<O>
where
W: Worker,
I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
- let mut stream = self.connect()?;
- let serialized_job = bincode::serialize(&Task::Job(job))?;
- debug!("sending {:?} bytes", serialized_job.len());
- stream.write_all(&serialized_job)?;
- stream.write_all(&END_OF_MESSAGE)?;
-
- let mut buf = [0; BUF_SIZE];
- let mut bytes = Vec::new();
- loop {
- if let Ok(size) = stream.read(&mut buf) {
- debug!("read {:?} bytes", size);
- if size == 0 && bytes.is_empty() {
- return Err(Error::NoResponse);
- }
- bytes.extend_from_slice(&buf[..size]);
-
- if bytes.len() >= END_OF_MESSAGE.len()
- && bytes[bytes.len() - END_OF_MESSAGE.len()..] == END_OF_MESSAGE
- {
- break;
- }
- }
+ let conn = self.connect().await?;
+ match conn.send(Task::Job(job)).await {
+ Ok(sonic::Response::Content(res)) => Ok(res),
+ _ => Err(Error::NoResponse),
}
- bytes = bytes[..bytes.len() - END_OF_MESSAGE.len()].to_vec();
- debug!("finished reading {:?} bytes", bytes.len());
-
- Ok(bincode::deserialize(&bytes)?)
}
- fn stop<W, I, O>(&self) -> Result<()>
+ async fn stop<W, I, O>(&self) -> Result<()>
where
W: Worker,
I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
debug!("closing worker {:}", self.addr);
- let mut stream = self.connect()?;
- let serialized_job = bincode::serialize(&Task::<I>::AllFinished)?;
- debug!("sending {:?} bytes", serialized_job.len());
- stream.write_all(&serialized_job)?;
- stream.write_all(&END_OF_MESSAGE)?;
+ let conn = self.connect().await?;
+ let res: sonic::Response<O> = conn.send(Task::<I>::AllFinished).await?;
+
+ debug_assert!(matches!(res, sonic::Response::Empty));
+
Ok(())
}
}
@@ -107,8 +81,8 @@ impl<'a> WorkerGuard<'a> {
}
}
- fn success(self) {
- self.from_pool.insert(Arc::clone(&self.worker));
+ async fn success(self) {
+ self.from_pool.insert(Arc::clone(&self.worker)).await;
}
}
@@ -158,12 +132,12 @@ impl WorkerPool {
self.running_workers.fetch_sub(1, Ordering::SeqCst);
}
- fn insert(&self, worker: Arc<RemoteWorker>) {
- self.ready_workers.lock().unwrap().push(worker);
+ async fn insert(&self, worker: Arc<RemoteWorker>) {
+ self.ready_workers.lock().await.push(worker);
}
- fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {
- let mut ready_workers = self.ready_workers.lock().unwrap();
+ async fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {
+ let mut ready_workers = self.ready_workers.lock().await;
if ready_workers.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {
return Err(Error::NoAvailableWorker);
}
@@ -176,7 +150,7 @@ impl WorkerPool {
}
}
- fn stop_workers<W, I, O>(&self)
+ async fn stop_workers<W, I, O>(&self)
where
W: Worker,
I: Map<W, O> + Send,
@@ -184,7 +158,7 @@ impl WorkerPool {
{
let mut failing_workers = Vec::new();
for worker in &self.all_workers {
- if worker.stop::<W, I, O>().is_err() {
+ if worker.stop::<W, I, O>().await.is_err() {
failing_workers.push(worker);
}
}
@@ -196,6 +170,10 @@ impl WorkerPool {
);
}
}
+
+ fn size(&self) -> usize {
+ self.all_workers.len()
+ }
}
pub struct Manager {
@@ -212,17 +190,17 @@ impl Manager {
}
}
- fn try_map<W, I, O>(&self, job: &I) -> Result<O>
+ async fn try_map<W, I, O>(&self, job: &I) -> Result<O>
where
W: Worker,
I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
loop {
- match self.pool.get_worker()? {
+ match self.pool.get_worker().await? {
Some(worker) => {
- let res = worker.perform(job)?;
- worker.success();
+ let res = worker.perform(job).await?;
+ worker.success().await;
return Ok(res);
}
@@ -233,14 +211,14 @@ impl Manager {
/// Execute job on one of the remote machines. If the remote machine fails for some reason,
/// the job should be allocated to another machine.
- pub fn map<W, I, O>(&self, job: I) -> O
+ pub async fn map<W, I, O>(&self, job: I) -> O
where
W: Worker,
I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
loop {
- match self.try_map(&job) {
+ match self.try_map(&job).await {
Ok(res) => return res,
Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),
Err(err) => {
@@ -262,82 +240,40 @@ impl Manager {
}
}
- fn reduce_end<O>(acc: Option<O>, elem: O) -> O
- where
- O: Reduce<O> + Send,
- {
- match acc {
- Some(acc) => acc.reduce(elem),
- None => elem,
- }
- }
-
#[allow(clippy::trait_duplication_in_bounds)]
- fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
+ async fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
W: Worker,
I: Map<W, O1> + Send,
O1: Serialize + DeserializeOwned + Send,
O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
- let acc: Arc<Mutex<Option<O2>>> = Arc::new(Mutex::new(None));
-
- let size = jobs.size_hint();
-
- match size.1 {
- Some(size) => {
- let pb = ProgressBar::new(size as u64);
- pb.set_style(
- ProgressStyle::default_bar()
- .template(
- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",
- )
- .progress_chars("#>-"),
- );
- jobs.par_bridge()
- .map(|job| self.map::<W, I, O1>(job))
- .progress_with(pb)
- .fold(
- || None,
- |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),
- )
- .for_each(|res| {
- if let Some(res) = res {
- let mut lock = acc.lock().unwrap();
- *lock = Some(Manager::reduce_end(lock.take(), res));
- }
- });
- }
- None => {
- jobs.par_bridge()
- .map(|job| self.map::<W, I, O1>(job))
- .fold(
- || None,
- |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),
- )
- .for_each(|res| {
- if let Some(res) = res {
- let mut lock = acc.lock().unwrap();
- *lock = Some(Manager::reduce_end(lock.take(), res));
- }
- });
+ let mut acc = None;
+
+ for chunk in jobs.chunks(self.pool.size()).into_iter() {
+ let results = futures::stream::iter(chunk.map(|job| self.map::<W, I, O1>(job)))
+ .buffer_unordered(self.pool.size())
+ .collect::<Vec<_>>()
+ .await;
+
+ for elem in results {
+ acc = Some(Self::reduce(acc, elem));
}
}
- let x = acc.lock().unwrap().take();
- x
+ acc
}
#[allow(clippy::trait_duplication_in_bounds)]
- pub fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
+ pub async fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
W: Worker,
I: Map<W, O1> + Send,
O1: Serialize + DeserializeOwned + Send,
O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
- let result = self.get_results(jobs);
- self.pool.stop_workers::<W, I, O1>();
+ let result = self.get_results(jobs).await;
+ self.pool.stop_workers::<W, I, O1>().await;
result
}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index 570a62e9..c1f1f0f0 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use serde::{de::DeserializeOwned, Deserialize, Serialize};
-use std::net::SocketAddr;
mod manager;
mod worker;
@@ -25,15 +24,14 @@ use thiserror::Error;
pub use worker::StatelessWorker;
pub use worker::Worker;
+use crate::sonic;
+
pub(crate) type Result<T> = std::result::Result<T, Error>;
#[derive(Error, Debug)]
pub enum Error {
- #[error("I/O error")]
- IO(#[from] std::io::Error),
-
- #[error("error while serializing/deserializing to/from bytes")]
- Serialization(#[from] bincode::Error),
+ #[error("network error")]
+ Sonic(#[from] sonic::Error),
#[error("could not get a working worker")]
NoAvailableWorker,
@@ -48,7 +46,7 @@ where
T: Serialize + DeserializeOwned + Send,
W: Worker,
{
- fn map(self, worker: &W) -> T;
+ fn map(&self, worker: &W) -> T;
}
pub trait Reduce<T> {
@@ -56,32 +54,6 @@ pub trait Reduce<T> {
fn reduce(self, element: T) -> Self;
}
-#[allow(clippy::trait_duplication_in_bounds)]
-pub trait MapReduce<W, I, O1, O2>
-where
- Self: Sized + Iterator<Item = I> + Send,
- W: Worker,
- I: Map<W, O1>,
- O1: Serialize + DeserializeOwned + Send,
- O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
-{
- fn map_reduce(self, workers: &[SocketAddr]) -> Option<O2> {
- let manager = Manager::new(workers);
- manager.run(self)
- }
-}
-
-#[allow(clippy::trait_duplication_in_bounds)]
-impl<W, I, O1, O2, T> MapReduce<W, I, O1, O2> for T
-where
- W: Worker,
- T: Iterator<Item = I> + Sized + Send,
- I: Map<W, O1>,
- O1: Serialize + DeserializeOwned + Send,
- O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
-{
-}
-
#[derive(Serialize, Deserialize, Debug)]
enum Task<T> {
Job(T),
diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rs
index f3c0e641..464057d5 100644
--- a/src/mapreduce/worker.rs
+++ b/src/mapreduce/worker.rs
@@ -1,167 +1,60 @@
-use std::{
- io::{Read, Write},
- net::{SocketAddr, TcpListener},
-};
-
-use crate::mapreduce::manager::{BUF_SIZE, END_OF_MESSAGE};
-
-use super::{Error, Map, Result, Task};
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::SocketAddr;
+
+use crate::sonic;
+
+use super::{Map, Result, Task};
+use async_trait::async_trait;
use serde::{de::DeserializeOwned, Serialize};
use tracing::{debug, info};
#[derive(Default)]
pub struct StatelessWorker {}
-pub enum State {
- Continue,
- Finished,
-}
-
+#[async_trait]
pub trait Worker {
- fn run_stream<W, I, O, S>(&self, mut stream: S) -> Result<State>
+ async fn run<I, O>(&self, addr: SocketAddr) -> Result<()>
where
Self: Sized,
- I: Map<Self, O>,
+ I: Map<Self, O> + Send + Sync,
O: Serialize + DeserializeOwned + Send,
- S: Read + Write + Unpin,
{
- let mut buf = [0; BUF_SIZE];
- let mut bytes = Vec::new();
+ let server = sonic::Server::bind(addr).await?;
+ info!("worker listening on: {:}", addr);
+
loop {
- if let Ok(size) = stream.read(&mut buf) {
- debug!("read {:?} bytes", size);
- if size == 0 && bytes.is_empty() {
- return Err(Error::NoResponse);
+ let req: sonic::Request<Task<I>> = server.accept().await?;
+ debug!("received request");
+ match &req.body {
+ Task::Job(job) => {
+ debug!("request is a job");
+ let res = job.map(self);
+ req.respond(sonic::Response::Content(res)).await?;
}
-
- bytes.extend_from_slice(&buf[..size]);
-
- if bytes.len() >= END_OF_MESSAGE.len()
- && bytes[bytes.len() - END_OF_MESSAGE.len()..] == END_OF_MESSAGE
- {
+ Task::AllFinished => {
+ req.respond::<Task<I>>(sonic::Response::Empty).await?;
break;
}
}
}
- bytes = bytes[..bytes.len() - END_OF_MESSAGE.len()].to_vec();
-
- match bincode::deserialize::<Task<I>>(&bytes)? {
- Task::Job(job) => {
- debug!("received job");
- let res = job.map(self);
- let bytes = bincode::serialize(&res)?;
- debug!("serialized result into {} bytes", bytes.len());
- stream.write_all(&bytes)?;
- stream.write_all(&END_OF_MESSAGE)?;
- }
- Task::AllFinished => {
- debug!("shutting down");
- return Ok(State::Finished);
- }
- };
-
- Ok(State::Continue)
- }
-
- fn run<I, O>(&self, addr: SocketAddr) -> Result<()>
- where
- Self: Sized,
- I: Map<Self, O>,
- O: Serialize + DeserializeOwned + Send,
- {
- let listener = TcpListener::bind(addr)?;
- info!("worker listening on: {:}", addr);
-
- loop {
- let (socket, _) = listener.accept()?;
- debug!("received connection");
- match self.run_stream::<Self, I, O, _>(socket)? {
- State::Finished => break,
- State::Continue => {}
- }
- }
-
Ok(())
}
}
impl Worker for StatelessWorker {}
-
-#[cfg(test)]
-mod tests {
- use serde::Deserialize;
-
- use super::*;
-
- struct MockTcpStream {
- contents: Vec<u8>,
- result: Vec<u8>,
- num_read: usize,
- }
-
- impl MockTcpStream {
- fn new(contents: Vec<u8>) -> Self {
- Self {
- contents,
- num_read: 0,
- result: Vec::new(),
- }
- }
- }
-
- impl Read for MockTcpStream {
- fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
- if self.num_read == 0 {
- buf.write_all(&self.contents[..]).unwrap();
- } else if self.num_read == 1 {
- buf.write_all(&END_OF_MESSAGE).unwrap();
- }
-
- self.contents = Vec::new();
- self.num_read += 1;
- Ok(buf.len())
- }
- }
-
- impl Write for MockTcpStream {
- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
- self.result.extend(Vec::from(buf));
- Ok(buf.len())
- }
-
- fn flush(&mut self) -> std::io::Result<()> {
- Ok(())
- }
- }
-
- #[derive(Serialize, Deserialize, Debug)]
- struct MockJob {
- contents: Vec<usize>,
- }
-
- #[derive(Serialize, Deserialize, Debug)]
- struct Count(usize);
-
- impl Map<StatelessWorker, Count> for MockJob {
- fn map(self, _worker: &StatelessWorker) -> Count {
- Count(self.contents.into_iter().filter(|d| *d == 0).count())
- }
- }
-
- #[test]
- fn execute() {
- let contents = vec![1, 2, 0, 1, 0, 1, 0];
- let job = bincode::serialize(&Task::Job(MockJob { contents })).unwrap();
-
- let mut stream = MockTcpStream::new(job);
- StatelessWorker::default()
- .run_stream::<StatelessWorker, MockJob, _, _>(&mut stream)
- .expect("worker failed");
-
- let result_bytes = &stream.result[..stream.result.len() - END_OF_MESSAGE.len()];
- let res: Count = bincode::deserialize(result_bytes).unwrap();
-
- assert_eq!(res.0, 3);
- }
-}
diff --git a/src/prehashed.rs b/src/prehashed.rs
index 6d5dd8ec..3165d756 100644
--- a/src/prehashed.rs
+++ b/src/prehashed.rs
@@ -17,6 +17,8 @@ use std::{
hash::{BuildHasher, Hash, Hasher},
};
+use serde::{Deserialize, Serialize};
+
struct PrehashBuilder {}
impl BuildHasher for PrehashBuilder {
@@ -45,7 +47,7 @@ impl Hasher for Prehasher {
}
}
-#[derive(Debug, Eq, Clone)]
+#[derive(Debug, Eq, Clone, Serialize, Deserialize)]
pub struct Prehashed(pub u128);
impl Hash for Prehashed {
diff --git a/src/ranking/goggles/ast.rs b/src/ranking/goggles/ast.rs
index 1c4dfb0c..cfabaeff 100644
--- a/src/ranking/goggles/ast.rs
+++ b/src/ranking/goggles/ast.rs
@@ -112,7 +112,7 @@ pub enum RawAction {
pub fn parse(goggle: &str) -> CrateResult<RawGoggle> {
let newlines = Regex::new(r"[\n]+").unwrap();
let clean = newlines.replace_all(goggle.trim(), "\n").to_string();
- let clean = clean.trim().replace('\n', ";").replace('\r', ";");
+ let clean = clean.trim().replace(['\n', '\r'], ";");
match PARSER.parse(clean.as_str()) {
Ok(blocks) => Ok(RawGoggle::from(blocks)),
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 5b23a169..7fe3f21f 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -294,7 +294,7 @@ mod tests {
use crate::{
index::Index,
schema::create_schema,
- searcher::Searcher,
+ searcher::LocalSearcher,
webpage::{Html, Webpage},
};
@@ -358,7 +358,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let res = searcher
.search("website", None, None, None)
@@ -480,7 +480,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let _ = searcher
.search(
@@ -606,7 +606,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let res = searcher
.search(
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index c0c37cc2..5151301b 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -27,6 +27,7 @@ use tantivy::collector::Collector;
use crate::{
collector::{MaxDocsConsidered, TopDocs},
+ inverted_index,
searcher::NUM_RESULTS_PER_PAGE,
webpage::region::{Region, RegionCount},
};
@@ -39,6 +40,7 @@ pub struct Ranker {
max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
aggregator: SignalAggregator,
+ de_rank_similar: bool,
}
impl Ranker {
@@ -49,6 +51,7 @@ impl Ranker {
offset: None,
aggregator,
max_docs: None,
+ de_rank_similar: true,
}
}
@@ -70,7 +73,11 @@ impl Ranker {
self
}
- pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
+ pub fn de_rank_similar(&mut self, de_rank_similar: bool) {
+ self.de_rank_similar = de_rank_similar;
+ }
+
+ pub fn collector(&self) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {
let score_tweaker = InitialScoreTweaker::new(
Arc::clone(&self.region_count),
self.selected_region,
@@ -79,6 +86,10 @@ impl Ranker {
let mut collector = TopDocs::with_limit(NUM_RESULTS_PER_PAGE);
+ if self.de_rank_similar {
+ collector = collector.and_de_rank_similar()
+ }
+
if let Some(offset) = self.offset {
collector = collector.and_offset(offset);
}
@@ -95,7 +106,7 @@ impl Ranker {
mod tests {
use crate::{
index::Index,
- searcher::Searcher,
+ searcher::LocalSearcher,
webpage::{Html, Link, Webpage},
};
@@ -158,7 +169,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("example", None, None, None)
.expect("Search failed")
@@ -226,7 +237,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("example", None, None, None)
.expect("Search failed")
@@ -316,7 +327,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("dr dk", None, None, None)
.expect("Search failed")
@@ -385,7 +396,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("title", None, None, None)
.expect("Search failed")
@@ -459,7 +470,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("test", None, None, None)
.expect("Search failed")
@@ -526,7 +537,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("test", None, None, None)
.expect("Search failed")
@@ -616,7 +627,7 @@ mod tests {
index.commit().unwrap();
- let searcher = Searcher::new(index, None, None);
+ let searcher = LocalSearcher::new(index, None, None);
let res = searcher
.search(
@@ -752,7 +763,7 @@ mod tests {
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::new(index, None, None);
+ let searcher = LocalSearcher::new(index, None, None);
let result = searcher
.search("termA termB", None, None, None)
@@ -823,7 +834,7 @@ mod tests {
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::new(index, None, None);
+ let searcher = LocalSearcher::new(index, None, None);
let result = searcher
.search("test", None, None, None)
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index f0644179..28c2d069 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -341,13 +341,7 @@ impl SignalAggregator {
Signal::PageCentrality => {
(webpage.page_centrality * (CENTRALITY_SCALING as f64)) as u64
}
- Signal::IsHomepage => {
- if webpage.html.url().is_homepage() {
- 1
- } else {
- 0
- }
- }
+ Signal::IsHomepage => webpage.html.url().is_homepage().into(),
Signal::FetchTimeMs => webpage.fetch_time_ms,
Signal::UpdateTimestamp => webpage
.html
diff --git a/src/search_prettifier.rs b/src/search_prettifier.rs
new file mode 100644
index 00000000..1fd4c96a
--- /dev/null
+++ b/src/search_prettifier.rs
@@ -0,0 +1,462 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::BTreeMap;
+
+use chrono::{NaiveDate, NaiveDateTime, Utc};
+use itertools::{intersperse, Itertools};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ entity_index::{entity::Span, StoredEntity},
+ inverted_index::{self, RetrievedWebpage},
+ searcher::{self, LocalSearcher},
+ webpage::Url,
+};
+
+pub fn initial(
+ result: searcher::local::InitialWebsiteResult,
+ local_searcher: &LocalSearcher,
+) -> InitialWebsiteResult {
+ let entity = result
+ .entity
+ .map(|entity| DisplayedEntity::from(entity, local_searcher));
+
+ InitialWebsiteResult {
+ spell_corrected_query: result.spell_corrected_query,
+ websites: result.websites,
+ entity,
+ }
+}
+
+pub fn retrieve(result: Vec<RetrievedWebpage>, searcher: &LocalSearcher) -> Vec<DisplayedWebpage> {
+ result
+ .into_iter()
+ .map(|mut webpage| {
+ webpage.primary_image = webpage.primary_image.and_then(|image| {
+ if searcher
+ .primary_image(image.uuid.clone().to_string())
+ .is_some()
+ {
+ Some(image)
+ } else {
+ None
+ }
+ });
+
+ let url: Url = webpage.url.clone().into();
+ webpage.favicon = searcher.favicon(&url.site().to_string().into());
+ webpage
+ })
+ .map(DisplayedWebpage::from)
+ .collect()
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InitialWebsiteResult {
+ pub spell_corrected_query: Option<String>,
+ pub websites: inverted_index::InitialSearchResult,
+ pub entity: Option<DisplayedEntity>,
+}
+
+pub fn html_escape(s: &str) -> String {
+ html_escape::decode_html_entities(s)
+ .chars()
+ .filter(|c| !matches!(c, '<' | '>' | '&'))
+ .collect()
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct DisplayedWebpage {
+ pub title: String,
+ pub url: String,
+ pub favicon_base64: String,
+ pub domain: String,
+ pub pretty_url: String,
+ pub snippet: String,
+ pub body: String,
+ pub primary_image_uuid: Option<String>,
+ pub last_updated: Option<String>,
+}
+
+const MAX_PRETTY_URL_LEN: usize = 50;
+const MAX_TITLE_LEN: usize = 70;
+
+fn prettify_url(url: &Url) -> String {
+ let mut pretty_url = url.strip_query().to_string();
+
+ if pretty_url.ends_with('/') {
+ pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
+ }
+
+ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
+ pretty_url = Url::from(pretty_url.clone())
+ .strip_protocol()
+ .replace('/', " › ");
+ pretty_url = protocol + &pretty_url;
+
+ if pretty_url.len() > MAX_PRETTY_URL_LEN {
+ pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();
+ pretty_url += "...";
+ }
+
+ pretty_url
+}
+
+fn prettify_date(date: NaiveDateTime) -> String {
+ let current_time = Utc::now().naive_utc();
+ let diff = current_time.signed_duration_since(date);
+
+ let num_hours = diff.num_hours() + 1;
+ if num_hours < 24 {
+ if num_hours <= 1 {
+ return "1 hour ago".to_string();
+ }
+
+ return format!("{num_hours} hours ago");
+ }
+
+ let num_days = diff.num_days();
+ if num_days < 30 {
+ if num_days <= 1 {
+ return "1 day ago".to_string();
+ }
+
+ return format!("{num_days} days ago");
+ }
+
+ format!("{}", date.format("%d. %b. %Y"))
+}
+
+impl From<RetrievedWebpage> for DisplayedWebpage {
+ fn from(webpage: RetrievedWebpage) -> Self {
+ let last_updated = webpage.updated_time.map(prettify_date);
+
+ let url: Url = webpage.url.clone().into();
+ let domain = url.domain().to_string();
+ let pretty_url = prettify_url(&url);
+
+ let mut title = html_escape(&webpage.title);
+
+ if title.len() > MAX_TITLE_LEN {
+ title = title.chars().take(MAX_TITLE_LEN).collect();
+ title += "...";
+ }
+
+ let favicon_bytes = webpage
+ .favicon
+ .map(|favicon| favicon.as_raw_bytes())
+ .unwrap_or_else(|| include_bytes!("../frontend/dist/assets/images/globe.png").to_vec());
+
+ Self {
+ title,
+ url: webpage.url,
+ pretty_url,
+ domain,
+ favicon_base64: base64::encode(favicon_bytes),
+ snippet: webpage.snippet, // snippet has already been html-escaped.
+ body: webpage.body,
+ primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
+ last_updated,
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct DisplayedEntity {
+ pub title: String,
+ pub small_abstract: String,
+ pub image_base64: Option<String>,
+ pub related_entities: Vec<DisplayedEntity>,
+ pub info: Vec<(String, String)>,
+}
+
+fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {
+ let mut info: Vec<_> = info.into_iter().collect();
+
+ info.sort_by(|(a, _), (b, _)| {
+ searcher
+ .attribute_occurrence(b)
+ .unwrap_or(0)
+ .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))
+ });
+
+ info.into_iter()
+ .map(|(key, value)| {
+ let mut value = entity_link_to_html(value, 150).replace('*', "•");
+
+ if value.starts_with('•') || value.starts_with("\n•") {
+ if let Some(first_bullet) = value.find('•') {
+ value = value.chars().skip(first_bullet + 1).collect();
+ }
+ }
+
+ let value = maybe_prettify_entity_date(value);
+
+ (key.replace('_', " "), value)
+ })
+ .filter(|(key, _)| {
+ !matches!(
+ key.as_str(),
+ "caption"
+ | "image size"
+ | "label"
+ | "landscape"
+ | "signature"
+ | "name"
+ | "website"
+ | "logo"
+ | "image caption"
+ )
+ })
+ .take(5)
+ .collect()
+}
+
+impl DisplayedEntity {
+ pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {
+ let entity_abstract = Span {
+ text: entity.entity_abstract,
+ links: entity.links,
+ };
+
+ let small_abstract = entity_link_to_html(entity_abstract, 300);
+
+ let image_base64 = entity
+ .image
+ .and_then(|image| searcher.entity_image(image))
+ .map(|image| base64::encode(image.as_raw_bytes()));
+
+ Self {
+ title: entity.title,
+ small_abstract,
+ image_base64,
+ related_entities: entity
+ .related_entities
+ .into_iter()
+ .map(|entity| DisplayedEntity::from(entity, searcher))
+ .collect(),
+ info: prepare_info(entity.info, searcher),
+ }
+ }
+}
+
+fn maybe_prettify_entity_date(value: String) -> String {
+ if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {
+ return format!("{}", date.format("%d/%m/%Y"));
+ }
+
+ if value.split_whitespace().count() == 6 {
+ let first_date = NaiveDate::parse_from_str(
+ itertools::intersperse(value.split_whitespace().take(3), " ")
+ .collect::<String>()
+ .as_str(),
+ "%Y %-m %-d",
+ );
+ let second_date = NaiveDate::parse_from_str(
+ itertools::intersperse(value.split_whitespace().skip(3), " ")
+ .collect::<String>()
+ .as_str(),
+ "%Y %-m %-d",
+ );
+
+ if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {
+ // the dates are reversed from the parser, so we need to return second_date before first_date
+ return format!("{}", second_date.format("%d/%m/%Y"))
+ + " - "
+ + format!("{}", first_date.format("%d/%m/%Y")).as_str();
+ }
+ }
+
+ value
+}
+
+pub fn thousand_sep_number(num: usize) -> String {
+ let s = num.to_string();
+ let c = s.chars().rev().chunks(3);
+ let chunks = c.into_iter().map(|chunk| {
+ chunk
+ .into_iter()
+ .collect::<Vec<char>>()
+ .into_iter()
+ .rev()
+ .collect::<String>()
+ });
+
+ intersperse(
+ chunks.collect::<Vec<_>>().into_iter().rev(),
+ ".".to_string(),
+ )
+ .collect()
+}
+
+fn entity_link_to_html(span: Span, trunace_to: usize) -> String {
+ let mut s = span.text;
+
+ let truncated = s.len() > trunace_to;
+ if truncated {
+ s = s.chars().take(trunace_to).collect();
+ }
+
+ let chars = s.chars();
+ let num_chars = chars.clone().count();
+
+ let mut res = String::new();
+
+ let mut last_link_end = 0;
+ for link in span.links {
+ if link.start > num_chars {
+ break;
+ }
+
+ res += chars
+ .clone()
+ .skip(last_link_end)
+ .take(link.start - last_link_end)
+ .collect::<String>()
+ .as_str();
+
+ let link_text: String = chars
+ .clone()
+ .skip(link.start)
+ .take(link.end - link.start)
+ .collect();
+
+ res += format!(
+ "<a href=\"https://en.wikipedia.org/wiki/{}\">",
+ link.target.replace(' ', "_")
+ )
+ .as_str();
+
+ res += link_text.as_str();
+
+ res += "</a>";
+
+ last_link_end = link.end;
+ }
+
+ res += chars
+ .clone()
+ .skip(last_link_end)
+ .collect::<String>()
+ .as_str();
+
+ if truncated {
+ res += "...";
+ }
+
+ res
+}
+
+#[cfg(test)]
+mod tests {
+ use chrono::{DateTime, Utc};
+
+ use crate::entity_index::entity::Link;
+
+ use super::*;
+
+ #[test]
+ fn prettify_date_in_hours() {
+ let date = Utc::now().naive_utc();
+ assert_eq!(prettify_date(date), "1 hour ago".to_string());
+
+ let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();
+ assert_eq!(prettify_date(date), "2 hours ago".to_string());
+ }
+
+ #[test]
+ fn prettify_date_days() {
+ let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();
+ assert_eq!(prettify_date(date), "1 day ago".to_string());
+
+ let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();
+ assert_eq!(prettify_date(date), "2 days ago".to_string());
+ }
+
+ #[test]
+ fn prettify_date_rest() {
+ let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
+ .unwrap()
+ .naive_local();
+ assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());
+ }
+
+ #[test]
+ fn simple_link_to_html() {
+ assert_eq!(
+ entity_link_to_html(
+ Span {
+ text: "some text with a link".to_string(),
+ links: vec![Link {
+ start: 5,
+ end: 9,
+ target: "text article".to_string()
+ }]
+ },
+ 10000
+ ),
+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
+ .to_string()
+ );
+ }
+
+ #[test]
+ fn truncated_link_to_html() {
+ assert_eq!(
+ entity_link_to_html(
+ Span {
+ text: "some text".to_string(),
+ links: vec![Link {
+ start: 5,
+ end: 9,
+ target: "text article".to_string()
+ }]
+ },
+ 7
+ ),
+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
+ );
+ }
+
+ #[test]
+ fn einstein_date() {
+ assert_eq!(
+ maybe_prettify_entity_date("1879 3 14 ".to_string()),
+ "14/03/1879".to_string()
+ );
+ }
+
+ #[test]
+ fn entity_date_span_prettify() {
+ assert_eq!(
+ maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),
+ "14/03/1879 - 27/05/1999".to_string()
+ );
+ }
+
+ #[test]
+ fn sep_number() {
+ assert_eq!(thousand_sep_number(0), "0".to_string());
+ assert_eq!(thousand_sep_number(10), "10".to_string());
+ assert_eq!(thousand_sep_number(100), "100".to_string());
+ assert_eq!(thousand_sep_number(1000), "1.000".to_string());
+ assert_eq!(thousand_sep_number(10_000), "10.000".to_string());
+ assert_eq!(thousand_sep_number(100_000), "100.000".to_string());
+ assert_eq!(thousand_sep_number(512_854), "512.854".to_string());
+ assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());
+ }
+}
diff --git a/src/searcher/distributed.rs b/src/searcher/distributed.rs
new file mode 100644
index 00000000..f9e17a72
--- /dev/null
+++ b/src/searcher/distributed.rs
@@ -0,0 +1,612 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ collector::{self, BucketCollector},
+ exponential_backoff::ExponentialBackoff,
+ inverted_index::{self, RetrievedWebpage},
+ search_prettifier::DisplayedWebpage,
+ searcher::{
+ InitialWebsitesFormatting, PrettifiedWebsitesResult, SearchResult, WebsitesResult,
+ NUM_RESULTS_PER_PAGE,
+ },
+};
+
+use std::{cmp::Ordering, net::SocketAddr, time::Instant};
+
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{sonic, webpage::region::Region};
+
+use super::{InitialSearchResult, PrettifiedSearchResult};
+
+type Result<T> = std::result::Result<T, Error>;
+
+struct RemoteSearcher {
+ addr: SocketAddr,
+}
+
+#[derive(Error, Debug)]
+pub enum Error {
+ #[error("Failed to get search result")]
+ SearchFailed,
+
+ #[error("Query cannot be empty")]
+ EmptyQuery,
+}
+
+impl RemoteSearcher {
+ async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResult> {
+ for timeout in ExponentialBackoff::from_millis(30).take(5) {
+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
+ {
+ if let Ok(sonic::Response::Content(body)) =
+ connection.send(Request::Search(query.clone())).await
+ {
+ return Ok(body);
+ }
+ }
+ }
+
+ Err(Error::SearchFailed)
+ }
+
+ async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResult> {
+ for timeout in ExponentialBackoff::from_millis(30).take(5) {
+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
+ {
+ if let Ok(sonic::Response::Content(body)) = connection
+ .send(Request::SearchPrettified(query.clone()))
+ .await
+ {
+ return Ok(body);
+ }
+ }
+ }
+
+ Err(Error::SearchFailed)
+ }
+
+ async fn retrieve_websites(
+ &self,
+ pointers: &[inverted_index::WebsitePointer],
+ original_query: &str,
+ ) -> Result<Vec<RetrievedWebpage>> {
+ for timeout in ExponentialBackoff::from_millis(30).take(5) {
+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
+ {
+ if let Ok(sonic::Response::Content(body)) = connection
+ .send(Request::RetrieveWebites {
+ websites: pointers.to_vec(),
+ query: original_query.to_string(),
+ })
+ .await
+ {
+ return Ok(body);
+ }
+ }
+ }
+
+ Err(Error::SearchFailed)
+ }
+
+ async fn retrieve_websites_prettified(
+ &self,
+ pointers: &[inverted_index::WebsitePointer],
+ original_query: &str,
+ ) -> Result<Vec<DisplayedWebpage>> {
+ for timeout in ExponentialBackoff::from_millis(30).take(5) {
+ if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
+ {
+ if let Ok(sonic::Response::Content(body)) = connection
+ .send(Request::RetrievePrettifiedWebites {
+ websites: pointers.to_vec(),
+ query: original_query.to_string(),
+ })
+ .await
+ {
+ return Ok(body);
+ }
+ }
+ }
+
+ Err(Error::SearchFailed)
+ }
+}
+
+#[derive(Clone, PartialEq, Eq)]
+struct ShardId(u32);
+
+pub struct Shard {
+ id: ShardId,
+ replicas: Vec<RemoteSearcher>,
+}
+
+impl Shard {
+ pub fn new(id: u32, replicas: Vec<String>) -> Self {
+ let mut parsed_replicas = Vec::new();
+
+ for replica in replicas {
+ parsed_replicas.push(RemoteSearcher {
+ addr: replica.parse().unwrap(),
+ });
+ }
+
+ Self {
+ id: ShardId(id),
+ replicas: parsed_replicas,
+ }
+ }
+
+ async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {
+ match self
+ .replicas
+ .iter()
+ .map(|remote| remote.search(query))
+ .collect::<FuturesUnordered<_>>()
+ .next()
+ .await
+ {
+ Some(result) => Ok(InitialSearchResultShard {
+ local_result: result?,
+ shard: self.id.clone(),
+ }),
+ None => Err(Error::SearchFailed),
+ }
+ }
+
+ async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {
+ match self
+ .replicas
+ .iter()
+ .map(|remote| remote.search_prettified(query))
+ .collect::<FuturesUnordered<_>>()
+ .next()
+ .await
+ {
+ Some(result) => Ok(InitialSearchResultShard {
+ local_result: result?,
+ shard: self.id.clone(),
+ }),
+ None => Err(Error::SearchFailed),
+ }
+ }
+
+ async fn retrieve_websites(
+ &self,
+ pointers: &[inverted_index::WebsitePointer],
+ original_query: &str,
+ ) -> Result<Vec<RetrievedWebpage>> {
+ match self
+ .replicas
+ .iter()
+ .map(|remote| remote.retrieve_websites(pointers, original_query))
+ .collect::<FuturesUnordered<_>>()
+ .next()
+ .await
+ {
+ Some(Ok(websites)) => Ok(websites),
+ _ => Err(Error::SearchFailed),
+ }
+ }
+
+ async fn retrieve_websites_prettified(
+ &self,
+ pointers: &[inverted_index::WebsitePointer],
+ original_query: &str,
+ ) -> Result<Vec<DisplayedWebpage>> {
+ match self
+ .replicas
+ .iter()
+ .map(|remote| remote.retrieve_websites_prettified(pointers, original_query))
+ .collect::<FuturesUnordered<_>>()
+ .next()
+ .await
+ {
+ Some(Ok(websites)) => Ok(websites),
+ _ => Err(Error::SearchFailed),
+ }
+ }
+}
+
+struct InitialSearchResultShard {
+ local_result: InitialSearchResult,
+ shard: ShardId,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct SearchQuery {
+ pub original: String,
+ pub selected_region: Option<Region>,
+ pub goggle_program: Option<String>,
+ pub skip_pages: Option<usize>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum Request {
+ Search(SearchQuery),
+ SearchPrettified(SearchQuery),
+ RetrieveWebites {
+ websites: Vec<inverted_index::WebsitePointer>,
+ query: String,
+ },
+ RetrievePrettifiedWebites {
+ websites: Vec<inverted_index::WebsitePointer>,
+ query: String,
+ },
+}
+
+pub struct DistributedSearcher {
+ shards: Vec<Shard>,
+}
+
+#[derive(Clone)]
+struct ScoredWebsitePointer {
+ local_pointer: inverted_index::WebsitePointer,
+ shard: ShardId,
+}
+
+impl collector::Doc for ScoredWebsitePointer {
+ fn score(&self) -> &f64 {
+ &self.local_pointer.score
+ }
+
+ fn key(&self) -> &crate::prehashed::Prehashed {
+ &self.local_pointer.site_hash
+ }
+
+ fn id(&self) -> &tantivy::DocId {
+ &self.local_pointer.address.doc_id
+ }
+
+ fn set_score(&mut self, score: f64) {
+ self.local_pointer.score = score;
+ }
+
+ fn set_key(&mut self, key: crate::prehashed::Prehashed) {
+ self.local_pointer.site_hash = key;
+ }
+
+ fn set_id(&mut self, id: tantivy::DocId) {
+ self.local_pointer.address.doc_id = id;
+ }
+
+ fn update_values_from(&mut self, other: &Self) {
+ self.set_score(*other.score());
+ self.set_key(other.key().clone());
+ self.set_id(*other.id());
+
+ self.shard = other.shard.clone();
+ }
+}
+
+impl PartialOrd for ScoredWebsitePointer {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.local_pointer
+ .score
+ .partial_cmp(&other.local_pointer.score)
+ }
+}
+
+impl Ord for ScoredWebsitePointer {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
+
+impl PartialEq for ScoredWebsitePointer {
+ fn eq(&self, other: &Self) -> bool {
+ self.local_pointer.score == other.local_pointer.score
+ }
+}
+
+impl Eq for ScoredWebsitePointer {}
+
+impl DistributedSearcher {
+ pub fn new(shards: Vec<Shard>) -> Self {
+ Self { shards }
+ }
+
+ pub async fn search_api(
+ &self,
+ query: &str,
+ selected_region: Option<Region>,
+ goggle_program: Option<String>,
+ skip_pages: Option<usize>,
+ ) -> Result<SearchResult> {
+ let start = Instant::now();
+
+ if query.is_empty() {
+ return Err(Error::EmptyQuery);
+ }
+
+ let query = SearchQuery {
+ original: query.to_string(),
+ selected_region,
+ goggle_program,
+ skip_pages,
+ };
+
+ // search shards
+ let initial_results = self
+ .shards
+ .iter()
+ .map(|shard| shard.search(&query))
+ .collect::<FuturesUnordered<_>>()
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .filter_map(|result| result.ok())
+ .collect::<Vec<_>>();
+
+ // check if any result has a bang hit
+ if let Some(result) = initial_results
+ .iter()
+ .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))
+ {
+ if let InitialSearchResult::Bang(bang) = &result.local_result {
+ return Ok(SearchResult::Bang(bang.clone()));
+ }
+ }
+
+ let spell_corrected_query = initial_results.first().and_then(|result| {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =
+ &result.local_result
+ {
+ result.spell_corrected_query.clone()
+ } else {
+ None
+ }
+ });
+
+ let entity = initial_results.first().and_then(|result| {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =
+ &result.local_result
+ {
+ result.entity.clone()
+ } else {
+ None
+ }
+ });
+
+ let num_docs = initial_results
+ .iter()
+ .map(|result| {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =
+ &result.local_result
+ {
+ result.websites.num_websites
+ } else {
+ 0
+ }
+ })
+ .sum();
+
+ // combine results
+ let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);
+ let mut collector = BucketCollector::new(top_n);
+
+ for result in initial_results {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(local_result)) =
+ result.local_result
+ {
+ for website in local_result.websites.top_websites {
+ let pointer = ScoredWebsitePointer {
+ local_pointer: website,
+ shard: result.shard.clone(),
+ };
+
+ collector.insert(pointer);
+ }
+ }
+ }
+
+ let top_websites = collector
+ .into_sorted_vec(true)
+ .into_iter()
+ .skip(skip_pages.unwrap_or(0))
+ .take(NUM_RESULTS_PER_PAGE)
+ .collect::<Vec<_>>();
+
+ // retrieve webpages
+ let mut retrieved_webpages = Vec::new();
+
+ for _ in 0..top_websites.len() {
+ retrieved_webpages.push(None);
+ }
+
+ for shard in &self.shards {
+ let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites
+ .iter()
+ .enumerate()
+ .filter(|(_, pointer)| pointer.shard == shard.id)
+ .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))
+ .unzip();
+
+ if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {
+ for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
+ retrieved_webpages[index] = Some(website);
+ }
+ }
+ }
+
+ let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();
+
+ debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
+
+ if retrieved_webpages.is_empty() && !top_websites.is_empty() {
+ return Err(Error::SearchFailed);
+ }
+
+ Ok(SearchResult::Websites(WebsitesResult {
+ spell_corrected_query,
+ webpages: inverted_index::SearchResult {
+ num_docs,
+ documents: retrieved_webpages,
+ },
+ entity,
+ search_duration_ms: start.elapsed().as_millis(),
+ }))
+ }
+
+ pub async fn search_prettified(
+ &self,
+ query: &str,
+ selected_region: Option<Region>,
+ goggle_program: Option<String>,
+ skip_pages: Option<usize>,
+ ) -> Result<PrettifiedSearchResult> {
+ let start = Instant::now();
+
+ if query.is_empty() {
+ return Err(Error::EmptyQuery);
+ }
+
+ let query = SearchQuery {
+ original: query.to_string(),
+ selected_region,
+ goggle_program,
+ skip_pages,
+ };
+
+ // search shards
+ let initial_results = self
+ .shards
+ .iter()
+ .map(|shard| shard.search_prettified(&query))
+ .collect::<FuturesUnordered<_>>()
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .filter_map(|result| result.ok())
+ .collect::<Vec<_>>();
+
+ // check if any result has a bang hit
+ if let Some(result) = initial_results
+ .iter()
+ .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))
+ {
+ if let InitialSearchResult::Bang(bang) = &result.local_result {
+ return Ok(PrettifiedSearchResult::Bang(bang.clone()));
+ }
+ }
+
+ let spell_corrected_query = initial_results.first().and_then(|result| {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =
+ &result.local_result
+ {
+ result.spell_corrected_query.clone()
+ } else {
+ None
+ }
+ });
+
+ let entity = initial_results.first().and_then(|result| {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =
+ &result.local_result
+ {
+ result.entity.clone()
+ } else {
+ None
+ }
+ });
+
+ let num_docs = initial_results
+ .iter()
+ .map(|result| {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(
+ result,
+ )) = &result.local_result
+ {
+ result.websites.num_websites
+ } else {
+ 0
+ }
+ })
+ .sum();
+
+ // combine results
+ let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);
+ let mut collector = BucketCollector::new(top_n);
+
+ for result in initial_results {
+ if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(
+ local_result,
+ )) = result.local_result
+ {
+ for website in local_result.websites.top_websites {
+ let pointer = ScoredWebsitePointer {
+ local_pointer: website,
+ shard: result.shard.clone(),
+ };
+
+ collector.insert(pointer);
+ }
+ }
+ }
+
+ let top_websites = collector
+ .into_sorted_vec(true)
+ .into_iter()
+ .skip(skip_pages.unwrap_or(0))
+ .take(NUM_RESULTS_PER_PAGE)
+ .collect::<Vec<_>>();
+
+ // retrieve webpages
+ let mut retrieved_webpages = Vec::new();
+
+ for _ in 0..top_websites.len() {
+ retrieved_webpages.push(None);
+ }
+
+ for shard in &self.shards {
+ let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites
+ .iter()
+ .enumerate()
+ .filter(|(_, pointer)| pointer.shard == shard.id)
+ .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))
+ .unzip();
+
+ if let Ok(websites) = shard
+ .retrieve_websites_prettified(&pointers, &query.original)
+ .await
+ {
+ for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
+ retrieved_webpages[index] = Some(website);
+ }
+ }
+ }
+
+ let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();
+
+ debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
+
+ if retrieved_webpages.is_empty() && !top_websites.is_empty() {
+ return Err(Error::SearchFailed);
+ }
+
+ Ok(PrettifiedSearchResult::Websites(PrettifiedWebsitesResult {
+ spell_corrected_query,
+ num_docs,
+ webpages: retrieved_webpages,
+ entity,
+ search_duration_ms: start.elapsed().as_millis(),
+ }))
+ }
+}
diff --git a/src/searcher.rs b/src/searcher/local.rs
similarity index 68%
rename from src/searcher.rs
rename to src/searcher/local.rs
index 58b9dd20..f49ed41e 100644
--- a/src/searcher.rs
+++ b/src/searcher/local.rs
@@ -17,67 +17,61 @@
use std::str::FromStr;
use std::time::Instant;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use uuid::Uuid;
-use crate::bangs::{BangHit, Bangs};
+use crate::bangs::Bangs;
use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
-use crate::inverted_index::InvertedIndexSearchResult;
use crate::query::Query;
use crate::ranking::goggles;
use crate::ranking::{Ranker, SignalAggregator};
use crate::webpage::region::Region;
use crate::webpage::Url;
-use crate::{Error, Result};
+use crate::{inverted_index, Error, Result};
-pub const NUM_RESULTS_PER_PAGE: usize = 20;
+use super::{
+ InitialSearchResult, InitialWebsitesFormatting, SearchResult, WebsitesResult,
+ NUM_RESULTS_PER_PAGE,
+};
-#[derive(Debug, Serialize)]
-pub struct WebsitesResult {
- pub spell_corrected_query: Option<String>,
- pub webpages: InvertedIndexSearchResult,
- pub entity: Option<StoredEntity>,
- pub search_duration_ms: u128,
-}
-
-#[derive(Debug, Serialize)]
-pub enum SearchResult {
- Websites(WebsitesResult),
- Bang(BangHit),
-}
-
-pub struct Searcher {
+pub struct LocalSearcher {
index: Index,
entity_index: Option<EntityIndex>,
bangs: Option<Bangs>,
}
-impl From<Index> for Searcher {
+impl From<Index> for LocalSearcher {
fn from(index: Index) -> Self {
Self::new(index, None, None)
}
}
-impl Searcher {
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InitialWebsiteResult {
+ pub spell_corrected_query: Option<String>,
+ pub websites: inverted_index::InitialSearchResult,
+ pub entity: Option<StoredEntity>,
+}
+
+impl LocalSearcher {
pub fn new(index: Index, entity_index: Option<EntityIndex>, bangs: Option<Bangs>) -> Self {
- Searcher {
+ LocalSearcher {
index,
entity_index,
bangs,
}
}
- pub fn search(
+ pub fn search_initial(
&self,
query: &str,
selected_region: Option<Region>,
goggle_program: Option<String>,
skip_pages: Option<usize>,
- ) -> Result<SearchResult> {
- let start = Instant::now();
-
+ de_rank_similar: bool,
+ ) -> Result<InitialSearchResult> {
let raw_query = query.to_string();
let goggle = goggle_program.and_then(|program| goggles::parse(&program).ok());
@@ -94,16 +88,17 @@ impl Searcher {
if query.is_empty() {
return Err(Error::EmptyQuery);
}
- if let Some(goggle) = &goggle {
- query.set_goggle(goggle, &self.index.schema());
- }
if let Some(bangs) = self.bangs.as_ref() {
if let Some(bang) = bangs.get(&query) {
- return Ok(SearchResult::Bang(bang));
+ return Ok(InitialSearchResult::Bang(bang));
}
}
+ if let Some(goggle) = &goggle {
+ query.set_goggle(goggle, &self.index.schema());
+ }
+
let mut ranker = Ranker::new(
self.index.region_count.clone(),
goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),
@@ -120,8 +115,9 @@ impl Searcher {
}
ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());
+ ranker.de_rank_similar(de_rank_similar);
- let webpages = self.index.search(&query, ranker.collector())?;
+ let webpages = self.index.search_initial(&query, ranker.collector())?;
let correction = self.index.spell_correction(&query.simple_terms());
let entity = self
@@ -129,14 +125,64 @@ impl Searcher {
.as_ref()
.and_then(|index| index.search(&raw_query));
- let search_duration_ms = start.elapsed().as_millis();
+ Ok(InitialSearchResult::Websites(
+ InitialWebsitesFormatting::Raw(InitialWebsiteResult {
+ spell_corrected_query: correction,
+ websites: webpages,
+ entity,
+ }),
+ ))
+ }
+
+ pub fn retrieve_websites(
+ &self,
+ websites: &[inverted_index::WebsitePointer],
+ query: &str,
+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {
+ let query = Query::parse(
+ query,
+ self.index.schema(),
+ self.index.tokenizers(),
+ &SignalAggregator::default(),
+ )?;
+
+ if query.is_empty() {
+ return Err(Error::EmptyQuery);
+ }
+
+ self.index.retrieve_websites(websites, &query)
+ }
- Ok(SearchResult::Websites(WebsitesResult {
- webpages,
- entity,
- spell_corrected_query: correction,
- search_duration_ms,
- }))
+ pub fn search(
+ &self,
+ query: &str,
+ selected_region: Option<Region>,
+ goggle_program: Option<String>,
+ skip_pages: Option<usize>,
+ ) -> Result<SearchResult> {
+ let start = Instant::now();
+
+ let initial_result =
+ self.search_initial(query, selected_region, goggle_program, skip_pages, true)?;
+
+ match initial_result {
+ InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(search_result)) => {
+ let retrieved_sites =
+ self.retrieve_websites(&search_result.websites.top_websites, query)?;
+
+ Ok(SearchResult::Websites(WebsitesResult {
+ spell_corrected_query: search_result.spell_corrected_query,
+ webpages: inverted_index::SearchResult {
+ num_docs: search_result.websites.num_websites,
+ documents: retrieved_sites,
+ },
+ entity: search_result.entity,
+ search_duration_ms: start.elapsed().as_millis(),
+ }))
+ }
+ InitialSearchResult::Bang(bang) => Ok(SearchResult::Bang(bang)),
+ _ => unreachable!(),
+ }
}
pub fn favicon(&self, site: &Url) -> Option<Image> {
@@ -214,7 +260,7 @@ mod tests {
index.commit().unwrap();
- let searcher = Searcher::new(index, None, None);
+ let searcher = LocalSearcher::new(index, None, None);
for p in 0..NUM_PAGES {
let urls: Vec<_> = searcher
diff --git a/src/searcher/mod.rs b/src/searcher/mod.rs
new file mode 100644
index 00000000..6f9c3f04
--- /dev/null
+++ b/src/searcher/mod.rs
@@ -0,0 +1,72 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod distributed;
+pub mod local;
+
+pub use distributed::*;
+pub use local::*;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ bangs::BangHit,
+ entity_index::StoredEntity,
+ inverted_index,
+ search_prettifier::{self, DisplayedEntity, DisplayedWebpage},
+};
+
+pub const NUM_RESULTS_PER_PAGE: usize = 20;
+
+#[derive(Debug, Serialize)]
+pub struct WebsitesResult {
+ pub spell_corrected_query: Option<String>,
+ pub webpages: inverted_index::SearchResult,
+ pub entity: Option<StoredEntity>,
+ pub search_duration_ms: u128,
+}
+
+#[derive(Debug, Serialize)]
+pub enum SearchResult {
+ Websites(WebsitesResult),
+ Bang(BangHit),
+}
+
+#[derive(Debug, Serialize)]
+pub enum PrettifiedSearchResult {
+ Websites(PrettifiedWebsitesResult),
+ Bang(BangHit),
+}
+
+#[derive(Debug, Serialize)]
+pub struct PrettifiedWebsitesResult {
+ pub spell_corrected_query: Option<String>,
+ pub webpages: Vec<DisplayedWebpage>,
+ pub num_docs: usize,
+ pub entity: Option<DisplayedEntity>,
+ pub search_duration_ms: u128,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum InitialSearchResult {
+ Websites(InitialWebsitesFormatting),
+ Bang(BangHit),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum InitialWebsitesFormatting {
+ Raw(local::InitialWebsiteResult),
+ Prettified(search_prettifier::InitialWebsiteResult),
+}
diff --git a/src/snippet.rs b/src/snippet.rs
index 209fc730..264ca9c4 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::frontend::search::html_escape;
+use crate::search_prettifier::html_escape;
use crate::tokenizer::Stemmed;
use crate::webpage::region::Region;
use std::cmp::Ordering;
@@ -320,7 +320,7 @@ pub fn generate(
#[cfg(test)]
mod tests {
use super::*;
- use crate::{index::Index, searcher::Searcher, webpage::Webpage};
+ use crate::{index::Index, searcher::LocalSearcher, webpage::Webpage};
use maplit::btreemap;
use tantivy::tokenizer::SimpleTokenizer;
@@ -360,7 +360,7 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("rust language", None, None, None)
@@ -397,7 +397,7 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("describe", None, None, None)
@@ -589,7 +589,7 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = Searcher::from(index);
+ let searcher = LocalSearcher::from(index);
let result = searcher
.search("paradigms", None, None, None)
diff --git a/src/sonic.rs b/src/sonic.rs
new file mode 100644
index 00000000..3f87e570
--- /dev/null
+++ b/src/sonic.rs
@@ -0,0 +1,179 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::time::Duration;
+
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use thiserror::Error;
+use tokio::{
+ io::{AsyncReadExt, AsyncWriteExt},
+ net::{TcpListener, TcpStream, ToSocketAddrs},
+};
+
+type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Error, Debug)]
+pub enum Error {
+ #[error("Got an IO error")]
+ IO(#[from] std::io::Error),
+
+ #[error("Error while serializing/deserializing to/from bytes")]
+ Serialization(#[from] bincode::Error),
+
+ #[error("Failed to connect to peer: connection timeout")]
+ ConnectionTimeout,
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum Response<T: Serialize> {
+ Empty,
+ Content(T),
+}
+
+#[repr(C)]
+struct Header {
+ body_size: usize,
+}
+
+pub struct Request<T> {
+ stream: TcpStream,
+ pub body: T,
+}
+
+unsafe fn any_as_u8_slice<T: Sized>(p: &T) -> &[u8] {
+ std::slice::from_raw_parts((p as *const T) as *const u8, std::mem::size_of::<T>())
+}
+
+impl<T> Request<T> {
+ pub async fn respond<R: Serialize>(mut self, response: Response<R>) -> Result<()> {
+ let bytes = bincode::serialize(&response)?;
+ let header = Header {
+ body_size: bytes.len(),
+ };
+
+ self.stream
+ .write_all(unsafe { any_as_u8_slice(&header) })
+ .await?;
+ self.stream.write_all(&bytes).await?;
+ self.stream.flush().await?;
+
+ self.stream.shutdown().await?;
+
+ Ok(())
+ }
+}
+
+pub struct Server {
+ listener: TcpListener,
+}
+
+impl Server {
+ pub async fn bind(addr: impl ToSocketAddrs) -> Result<Self> {
+ let listener = TcpListener::bind(addr).await?;
+ Ok(Self { listener })
+ }
+
+ pub async fn accept<T>(&self) -> Result<Request<T>>
+ where
+ T: Serialize + DeserializeOwned,
+ {
+ let (mut stream, client) = self.listener.accept().await?;
+ tracing::debug!("accepted connection from: {}", &client);
+
+ let mut header_buf = vec![0; std::mem::size_of::<Header>()];
+ stream.read_exact(&mut header_buf).await?;
+ let header: Header = unsafe { std::ptr::read(header_buf.as_ptr() as *const _) };
+
+ let mut buf = vec![0; header.body_size];
+
+ stream.read_exact(&mut buf).await?;
+ tracing::debug!("received bytes: {:?}", &buf);
+
+ let body = bincode::deserialize(&buf)?;
+
+ Ok(Request { stream, body })
+ }
+}
+
+pub struct Connection {
+ stream: TcpStream,
+}
+
+impl Connection {
+ pub async fn create(server: impl ToSocketAddrs) -> Result<Self> {
+ Self::create_with_timeout(server, Duration::from_secs(30)).await
+ }
+
+ pub async fn create_with_timeout(
+ server: impl ToSocketAddrs,
+ timeout: Duration,
+ ) -> Result<Self> {
+ match tokio::time::timeout(timeout, TcpStream::connect(server)).await {
+ Ok(stream) => {
+ let stream = stream?;
+ Ok(Connection { stream })
+ }
+ Err(_) => Err(Error::ConnectionTimeout),
+ }
+ }
+
+ pub async fn send_without_timeout<T: Serialize, R: DeserializeOwned + Serialize>(
+ mut self,
+ request: T,
+ ) -> Result<Response<R>> {
+ let bytes = bincode::serialize(&request)?;
+
+ let header = Header {
+ body_size: bytes.len(),
+ };
+
+ self.stream
+ .write_all(unsafe { any_as_u8_slice(&header) })
+ .await?;
+ self.stream.write_all(&bytes).await?;
+ self.stream.flush().await?;
+
+ let mut header_buf = vec![0; std::mem::size_of::<Header>()];
+ self.stream.read_exact(&mut header_buf).await?;
+ let header: Header = unsafe { std::ptr::read(header_buf.as_ptr() as *const _) };
+
+ let mut buf = vec![0; header.body_size];
+ self.stream.read_exact(&mut buf).await?;
+
+ self.stream.shutdown().await?;
+
+ Ok(bincode::deserialize(&buf)?)
+ }
+
+ pub async fn send<T: Serialize, R: DeserializeOwned + Serialize>(
+ self,
+ request: T,
+ ) -> Result<Response<R>> {
+ self.send_with_timeout(request, Duration::from_secs(30))
+ .await
+ }
+
+ pub async fn send_with_timeout<T: Serialize, R: DeserializeOwned + Serialize>(
+ self,
+ request: T,
+ timeout: Duration,
+ ) -> Result<Response<R>> {
+ match tokio::time::timeout(timeout, self.send_without_timeout(request)).await {
+ Ok(res) => res,
+ Err(_) => Err(Error::ConnectionTimeout),
+ }
+ }
+}
diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rs
index 4459ffee..5cbf8cce 100644
--- a/src/spell/dictionary.rs
+++ b/src/spell/dictionary.rs
@@ -297,11 +297,10 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {
let searcher = searcher.unwrap();
let mut matches = self.perform_search(&searcher);
- let dist_metric = LevenshteinDistance::new(edit_distance);
while let Some((correction, freq)) = matches.next() {
let correction = std::str::from_utf8(correction).unwrap().to_owned();
- let dist = dist_metric.compare(term, &correction);
+ let dist = LevenshteinDistance::compare(term, &correction);
let prob = freq as f64 / self.total_freq as f64;
diff --git a/src/spell/distance.rs b/src/spell/distance.rs
index 515a0ae4..5317f439 100644
--- a/src/spell/distance.rs
+++ b/src/spell/distance.rs
@@ -39,11 +39,11 @@ impl LevenshteinDistance {
///
/// The runtime complexity is `O(m*n)`, where `m` and `n` are the
/// strings' lengths.
- pub fn compare(&self, a: &str, b: &str) -> usize {
+ pub fn compare(a: &str, b: &str) -> usize {
let len_a = a.chars().count();
let len_b = b.chars().count();
if len_a < len_b {
- return self.compare(b, a);
+ return Self::compare(b, a);
}
// handle special case of 0 length
if len_a == 0 {
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 352b3380..089eceee 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -627,7 +627,7 @@ impl Html {
}
}
Field::IsHomepage => {
- doc.add_u64(tantivy_field, if self.url().is_homepage() { 1 } else { 0 });
+ doc.add_u64(tantivy_field, self.url().is_homepage().into());
}
Field::LastUpdated => doc.add_u64(
tantivy_field,
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index 650a7785..bdb1e191 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -16,10 +16,10 @@
use std::{fmt::Display, time::Duration};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use tracing::debug;
-#[derive(Debug, Serialize, PartialEq, Eq, Clone, Hash)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
pub struct Url(String);
impl Display for Url {
@@ -71,7 +71,7 @@ impl Url {
pub fn domain(&self) -> &str {
let site = self.site();
- let num_punctuations: usize = site.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
+ let num_punctuations: usize = site.chars().map(|c| (c == '.') as usize).sum();
if num_punctuations > 1 {
let domain_index = site.rfind('.').unwrap();
let mut start_index = site[..domain_index].rfind('.').unwrap() + 1;