web search done right
Indexing using mapreduce, centrality store, mapreduce worker state
Mikkel Denkerda6c7503758d
diff --git a/.gitignore b/.gitignoreindex 79b16f44..129ac8b8 100644--- a/.gitignore+++ b/.gitignore@@ -3,3 +3,5 @@flamegraph.svgwarc_files/webgraph+centrality+indexdiff --git a/configs/centrality.toml b/configs/centrality.tomlnew file mode 100644index 00000000..a7300b3b--- /dev/null+++ b/configs/centrality.toml@@ -0,0 +1,3 @@+type = "Centrality"+webgraph_path = "webgraph"+output_path = "centrality"\ No newline at end of filediff --git a/configs/indexer.toml b/configs/indexer.tomlindex 4efb218b..3b2b1f95 100644--- a/configs/indexer.toml+++ b/configs/indexer.toml@@ -1,12 +1,12 @@type = "Indexer"-warc_paths_file = "warc.paths"+mode = "Local"-# [warc_source]-# type = "S3"-# name = "us-east-1"-# endpoint = "https://s3.us-east-1.amazonaws.com"-# bucket = "commoncrawl"+warc_paths_file = "warc.paths"+centrality_store_path = "centrality"[warc_source]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"+type = "Local"+folder = "./warc_files"+names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]+# type = "HTTP"+# base_url = "https://data.commoncrawl.org/"\ No newline at end of filediff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsindex 37fb6d26..f8206863 100644--- a/examples/mapreduce_manager.rs+++ b/examples/mapreduce_manager.rs@@ -1,6 +1,6 @@use std::net::SocketAddr;-use cuely::mapreduce::{Map, MapReduce, Reduce};+use cuely::mapreduce::{Map, MapReduce, Reduce, StatelessWorker};use serde::{Deserialize, Serialize};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -13,8 +13,8 @@ struct Job {#[derive(Serialize, Deserialize, Debug)]struct Count(usize);-impl Map<Count> for Job {- fn map(self) -> Count {+impl Map<StatelessWorker, Count> for Job {+ fn map(self, _worker: &StatelessWorker) -> Count {std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(1)}diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsindex 15c25a09..fdca5e9c 100644--- a/examples/mapreduce_worker.rs+++ b/examples/mapreduce_worker.rs@@ -1,6 +1,6 @@use std::net::SocketAddr;-use cuely::mapreduce::{Map, Reduce, Worker};+use cuely::mapreduce::{Map, Reduce, StatelessWorker, Worker};use serde::{Deserialize, Serialize};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -13,8 +13,8 @@ struct Job {#[derive(Serialize, Deserialize, Debug)]struct Count(usize);-impl Map<Count> for Job {- fn map(self) -> Count {+impl Map<StatelessWorker, Count> for Job {+ fn map(self, _worker: &StatelessWorker) -> Count {std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(1)}@@ -34,6 +34,7 @@ fn main() {let args: Vec<_> = std::env::args().collect();- Worker::run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())+ StatelessWorker::default()+ .run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap()).expect("failed to run worker");}diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex 494a0a47..6b80bff8 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use clap::Parser;-use cuely::entrypoint::{Indexer, WebgraphEntrypoint};+use cuely::entrypoint::{CentralityEntrypoint, Indexer, WebgraphEntrypoint};use cuely::Config;use std::fs;use tracing::Level;@@ -40,12 +40,14 @@ fn main() {match config {Config::Indexer(config) => {- Indexer::from(config)+ Indexer::new(config, args.mapreduce_worker_addr).run().expect("Failed to index documents");}Config::Webgraph(config) => WebgraphEntrypoint::new(config, args.mapreduce_worker_addr).run().expect("Failed to build webgraph"),++ Config::Centrality(config) => CentralityEntrypoint::from(config).run(),}}diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rsnew file mode 100644index 00000000..0d85a9ef--- /dev/null+++ b/src/entrypoint/centrality.rs@@ -0,0 +1,46 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder, CentralityConfig,+};++pub struct CentralityEntrypoint {+ config: CentralityConfig,+}++impl From<CentralityConfig> for CentralityEntrypoint {+ fn from(config: CentralityConfig) -> Self {+ Self { config }+ }+}++impl CentralityEntrypoint {+ pub fn run(self) {+ let graph = WebgraphBuilder::new(&self.config.webgraph_path)+ .with_host_graph()+ .open();++ let mut centrality_store = CentralityStore::new(&self.config.output_path);++ centrality_store.append(+ graph+ .host_harmonic_centrality()+ .into_iter()+ .map(|(node, centrality)| (node.name, centrality)),+ );+ }+}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 8163999f..52ba8ca2 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -13,55 +13,203 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::fs::File;-use std::io::{self, BufRead};+use std::net::SocketAddr;+use std::path::Path;+use serde::{Deserialize, Serialize};+use tracing::{debug, info, trace};++use crate::index::{FrozenIndex, Index};+use crate::mapreduce::{Map, MapReduce, Reduce, Worker};+use crate::ranking::centrality_store::CentralityStore;use crate::warc::WarcFile;-use crate::webpage::Html;-use crate::{IndexingConfig, Result};+use crate::webpage::{Html, Link, Webpage};+use crate::{+ HttpConfig, IndexingConfig, IndexingLocalConfig, IndexingMasterConfig, IndexingWorkerConfig,+ LocalConfig, Result, WarcSource,+};pub struct Indexer {- warc_paths: Vec<String>,config: IndexingConfig,+ worker_addr: Option<String>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+enum JobConfig {+ Http(HttpConfig),+ Local(LocalConfig),+}++#[derive(Debug, Serialize, Deserialize)]+struct Job {+ config: JobConfig,+ warc_path: String,+ base_path: String,+}++struct IndexingWorker {+ centrality_store: CentralityStore,}-impl From<IndexingConfig> for Indexer {- fn from(config: IndexingConfig) -> Self {- let file = File::open(&config.warc_paths_file).unwrap();- let mut warc_paths = Vec::new();+impl IndexingWorker {+ fn new(centrality_store_path: String) -> Self {+ Self {+ centrality_store: CentralityStore::new(centrality_store_path),+ }+ }+}++impl Worker for IndexingWorker {}++impl Map<IndexingWorker, FrozenIndex> for Job {+ fn map(self, worker: &IndexingWorker) -> FrozenIndex {+ let name = self.warc_path.split('/').last().unwrap();++ info!("processing {}", name);- for line in io::BufReader::new(file).lines() {- warc_paths.push(line.unwrap());+ let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();++ let source = match self.config {+ JobConfig::Http(config) => WarcSource::HTTP(config),+ JobConfig::Local(config) => WarcSource::Local(config),+ };++ debug!("downlooading warc file");+ let file = WarcFile::download(source, &self.warc_path).unwrap();+ debug!("finished downloading");++ for record in file.records().flatten() {+ let html = Html::parse(&record.response.body, &record.request.url);+ let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph+ let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();++ trace!("inserting webpage: {:?}", html.url());++ let webpage = Webpage {+ html,+ backlinks,+ centrality,+ };++ if let Err(err) = index.insert(webpage) {+ debug!("{:?}", err);+ }}+ index.commit().unwrap();++ info!("{} done", name);- Self { warc_paths, config }+ index.into()+ }+}++impl Reduce<FrozenIndex> for Index {+ fn reduce(self, element: FrozenIndex) -> Self {+ let other = element.into();++ self.merge(other)+ }+}++impl Reduce<Index> for Index {+ fn reduce(self, element: Index) -> Self {+ self.merge(element)}}impl Indexer {- pub fn run(self) -> Result<()> {- self.warc_paths- .into_iter()- .map(|warc_path| {- let source = self.config.warc_source.clone();+ pub fn new(config: IndexingConfig, worker_addr: Option<String>) -> Self {+ Self {+ config,+ worker_addr,+ }+ }+ fn run_master(config: &IndexingMasterConfig) -> Result<()> {+ info!("Running master for index construction");- WarcFile::download(source, &warc_path)- })- .map(|warc| {- if warc.is_err() {- return;- }- let warc = warc.unwrap();+ let warc_paths = config.warc_source.paths()?;++ let workers: Vec<SocketAddr> = config+ .workers+ .iter()+ .map(|worker| worker.parse().unwrap())+ .collect();- for record in warc.records().flatten() {- let _webpage = Html::parse(&record.response.body, &record.request.url);- // println!("TEST: {:?}", webpage.title());- // println!();+ let job_config = match config.warc_source.clone() {+ WarcSource::S3(_) => todo!("s3 not supported yet"),+ WarcSource::HTTP(config) => JobConfig::Http(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ };++ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> =+ Box::new(warc_paths.into_iter().map(|warc_path| {+ Job {+ config: job_config.clone(),+ warc_path,+ base_path: config+ .graph_base_path+ .clone()+ .unwrap_or_else(|| "index".to_string()),}- // panic!();+ }));++ if let Some(limit) = config.limit_warc_files {+ warc_paths = Box::new(warc_paths.take(limit));+ }++ let _index: Index = warc_paths+ .map_reduce(&workers)+ .expect("failed to build index");++ Ok(())+ }++ fn run_worker(worker_addr: String, config: &IndexingWorkerConfig) -> Result<()> {+ IndexingWorker::new(config.centrality_store_path.clone()).run::<Job, FrozenIndex>(+ worker_addr+ .parse::<SocketAddr>()+ .expect("Could not parse worker address"),+ )?;+ Ok(())+ }++ fn run_locally(config: &IndexingLocalConfig) -> Result<()> {+ let warc_paths = config.warc_source.paths()?;++ let job_config = match config.warc_source.clone() {+ WarcSource::S3(_) => todo!("s3 not supported yet"),+ WarcSource::HTTP(config) => JobConfig::Http(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ };++ let worker = IndexingWorker::new(config.centrality_store_path.clone());++ warc_paths+ .into_iter()+ .map(|path| Job {+ config: job_config.clone(),+ warc_path: path,+ base_path: "index".to_string(),})- .for_each(drop);+ .map(|job| job.map(&worker))+ .fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {+ Some(acc) => Some(acc.reduce(elem)),+ None => Some(elem.into()),+ });Ok(())}++ pub fn run(&self) -> Result<()> {+ match &self.config {+ IndexingConfig::Master(config) => Self::run_master(config),+ IndexingConfig::Worker(config) => Self::run_worker(+ self.worker_addr+ .clone()+ .expect("Worker address not specified"),+ config,+ ),+ IndexingConfig::Local(config) => Self::run_locally(config),+ }+ }}diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex d4f534ca..c7ca9826 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -13,8 +13,10 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+mod centrality;mod indexer;mod webgraph;+pub use centrality::CentralityEntrypoint;pub use indexer::Indexer;pub use webgraph::WebgraphEntrypoint;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex a6c9b62e..3140f606 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- mapreduce::{Map, MapReduce, Reduce, Worker},+ mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},warc::WarcFile,webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},webpage::{self, Html},@@ -38,8 +38,8 @@ struct Job {graph_base_path: String,}-impl Map<FrozenWebgraph> for Job {- fn map(self) -> FrozenWebgraph {+impl Map<StatelessWorker, FrozenWebgraph> for Job {+ fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {let name = self.warc_path.split('/').last().unwrap();info!("processing {}", name);@@ -102,7 +102,11 @@ impl Reduce<FrozenWebgraph> for Webgraph {impl Reduce<Webgraph> for Webgraph {fn reduce(mut self, element: Webgraph) -> Self {+ let other_path = element.path.clone();+self.merge(element);++ std::fs::remove_dir_all(other_path).unwrap();self}}@@ -160,7 +164,7 @@ impl WebgraphEntrypoint {}fn run_worker(worker_addr: String) -> Result<()> {- Worker::run::<Job, FrozenWebgraph>(+ StatelessWorker::default().run::<Job, FrozenWebgraph>(worker_addr.parse::<SocketAddr>().expect("Could not parse worker address"),@@ -176,6 +180,7 @@ impl WebgraphEntrypoint {WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};+ let worker = StatelessWorker::default();warc_paths.into_iter()@@ -184,7 +189,7 @@ impl WebgraphEntrypoint {warc_path: path,graph_base_path: "webgraph".to_string(),})- .map(|job| job.map())+ .map(|job| job.map(&worker)).fold(None,|acc: Option<Webgraph>, elem: FrozenWebgraph| match acc {diff --git a/src/index.rs b/src/index.rsindex 5f82e7ca..b5ea269f 100644--- a/src/index.rs+++ b/src/index.rs@@ -29,7 +29,7 @@ use std::fs;use std::path::Path;pub struct Index {- path: String,+ pub path: String,tantivy_index: tantivy::Index,writer: tantivy::IndexWriter,reader: tantivy::IndexReader,diff --git a/src/lib.rs b/src/lib.rsindex 356aa182..0bef245c 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -43,12 +43,40 @@ pub mod webpage;pub enum Config {Indexer(IndexingConfig),Webgraph(WebgraphConfig),+ Centrality(CentralityConfig),}#[derive(Debug, Deserialize, Clone)]-pub struct IndexingConfig {+pub struct CentralityConfig {+ pub webgraph_path: String,+ pub output_path: String,+}++#[derive(Debug, Deserialize, Clone)]+#[serde(tag = "mode")]+pub enum IndexingConfig {+ Master(IndexingMasterConfig),+ Worker(IndexingWorkerConfig),+ Local(IndexingLocalConfig),+}++#[derive(Debug, Deserialize, Clone)]+pub struct IndexingMasterConfig {+ limit_warc_files: Option<usize>,warc_source: WarcSource,- warc_paths_file: String,+ workers: Vec<String>,+ graph_base_path: Option<String>,+}++#[derive(Debug, Deserialize, Clone)]+pub struct IndexingWorkerConfig {+ centrality_store_path: String,+}++#[derive(Debug, Deserialize, Clone)]+pub struct IndexingLocalConfig {+ warc_source: WarcSource,+ centrality_store_path: String,}#[derive(Debug, Deserialize, Clone)]diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 9cb4372d..67ec53fe 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -1,4 +1,4 @@-use super::{Error, Result};+use super::{Error, Result, Worker};use super::{Map, Reduce};use crate::exponential_backoff::ExponentialBackoff;use crate::mapreduce::Task;@@ -43,9 +43,10 @@ impl RemoteWorker {Err(Error::NoResponse)}- fn perform<I, O>(&self, job: &I) -> Result<O>+ fn perform<W, I, O>(&self, job: &I) -> Result<O>where- I: Map<O> + Send,+ W: Worker,+ I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{let mut stream = self.connect()?;@@ -77,9 +78,10 @@ impl RemoteWorker {Ok(bincode::deserialize(&bytes)?)}- fn stop<I, O>(&self) -> Result<()>+ fn stop<W, I, O>(&self) -> Result<()>where- I: Map<O> + Send,+ W: Worker,+ I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{debug!("closing worker {:}", self.addr);@@ -174,14 +176,15 @@ impl WorkerPool {}}- fn stop_workers<I, O>(&self)+ fn stop_workers<W, I, O>(&self)where- I: Map<O> + Send,+ W: Worker,+ I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{let mut failing_workers = Vec::new();for worker in &self.all_workers {- if worker.stop::<I, O>().is_err() {+ if worker.stop::<W, I, O>().is_err() {failing_workers.push(worker)}}@@ -209,9 +212,10 @@ impl Manager {}}- fn try_map<I, O>(&self, job: &I) -> Result<O>+ fn try_map<W, I, O>(&self, job: &I) -> Result<O>where- I: Map<O> + Send,+ W: Worker,+ I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{loop {@@ -229,9 +233,10 @@ impl Manager {/// Execute job on one of the remote machines. If the remote machine fails for some reason,/// the job should be allocated to another machine.- pub fn map<I, O>(&self, job: I) -> O+ pub fn map<W, I, O>(&self, job: I) -> Owhere- I: Map<O> + Send,+ W: Worker,+ I: Map<W, O> + Send,O: Serialize + DeserializeOwned + Send,{loop {@@ -266,9 +271,10 @@ impl Manager {}}- fn get_results<I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>+ fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>where- I: Map<O1> + Send,+ W: Worker,+ I: Map<W, O1> + Send,O1: Serialize + DeserializeOwned + Send,O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{@@ -287,7 +293,7 @@ impl Manager {.progress_chars("#>-"),);jobs.par_bridge()- .map(|job| self.map::<I, O1>(job))+ .map(|job| self.map::<W, I, O1>(job)).progress_with(pb).fold(|| None,@@ -302,7 +308,7 @@ impl Manager {}None => {jobs.par_bridge()- .map(|job| self.map::<I, O1>(job))+ .map(|job| self.map::<W, I, O1>(job)).fold(|| None,|acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),@@ -320,14 +326,15 @@ impl Manager {x}- pub fn run<I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>+ pub fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>where- I: Map<O1> + Send,+ W: Worker,+ I: Map<W, O1> + Send,O1: Serialize + DeserializeOwned + Send,O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{let result = self.get_results(jobs);- self.pool.stop_workers::<I, O1>();+ self.pool.stop_workers::<W, I, O1>();result}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex 87d56a1b..d5e82e30 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -22,6 +22,7 @@ mod worker;pub use manager::Manager;use thiserror::Error;+pub use worker::StatelessWorker;pub use worker::Worker;pub(crate) type Result<T> = std::result::Result<T, Error>;@@ -44,22 +45,24 @@ pub enum Error {NoResponse,}-pub trait Map<T>+pub trait Map<W, T>whereSelf: Serialize + DeserializeOwned + Send,T: Serialize + DeserializeOwned + Send,+ W: Worker,{- fn map(self) -> T;+ fn map(self, worker: &W) -> T;}pub trait Reduce<T> {fn reduce(self, element: T) -> Self;}-pub trait MapReduce<I, O1, O2>+pub trait MapReduce<W, I, O1, O2>whereSelf: Sized + Iterator<Item = I> + Send,- I: Map<O1>,+ W: Worker,+ I: Map<W, O1>,O1: Serialize + DeserializeOwned + Send,O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{@@ -68,10 +71,11 @@ wheremanager.run(self)}}-impl<I, O1, O2, T> MapReduce<I, O1, O2> for T+impl<W, I, O1, O2, T> MapReduce<W, I, O1, O2> for Twhere+ W: Worker,T: Iterator<Item = I> + Sized + Send,- I: Map<O1>,+ I: Map<W, O1>,O1: Serialize + DeserializeOwned + Send,O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rsindex 289c59bd..f3c0e641 100644--- a/src/mapreduce/worker.rs+++ b/src/mapreduce/worker.rs@@ -9,17 +9,19 @@ use super::{Error, Map, Result, Task};use serde::{de::DeserializeOwned, Serialize};use tracing::{debug, info};-pub struct Worker {}+#[derive(Default)]+pub struct StatelessWorker {}-enum State {+pub enum State {Continue,Finished,}-impl Worker {- fn run_stream<I, O, S>(mut stream: S) -> Result<State>+pub trait Worker {+ fn run_stream<W, I, O, S>(&self, mut stream: S) -> Result<State>where- I: Map<O>,+ Self: Sized,+ I: Map<Self, O>,O: Serialize + DeserializeOwned + Send,S: Read + Write + Unpin,{@@ -47,7 +49,7 @@ impl Worker {match bincode::deserialize::<Task<I>>(&bytes)? {Task::Job(job) => {debug!("received job");- let res = job.map();+ let res = job.map(self);let bytes = bincode::serialize(&res)?;debug!("serialized result into {} bytes", bytes.len());stream.write_all(&bytes)?;@@ -62,9 +64,10 @@ impl Worker {Ok(State::Continue)}- pub fn run<I, O>(addr: SocketAddr) -> Result<()>+ fn run<I, O>(&self, addr: SocketAddr) -> Result<()>where- I: Map<O>,+ Self: Sized,+ I: Map<Self, O>,O: Serialize + DeserializeOwned + Send,{let listener = TcpListener::bind(addr)?;@@ -73,7 +76,7 @@ impl Worker {loop {let (socket, _) = listener.accept()?;debug!("received connection");- match Worker::run_stream::<I, O, _>(socket)? {+ match self.run_stream::<Self, I, O, _>(socket)? {State::Finished => break,State::Continue => {}}@@ -83,6 +86,8 @@ impl Worker {}}+impl Worker for StatelessWorker {}+#[cfg(test)]mod tests {use serde::Deserialize;@@ -138,8 +143,8 @@ mod tests {#[derive(Serialize, Deserialize, Debug)]struct Count(usize);- impl Map<Count> for MockJob {- fn map(self) -> Count {+ impl Map<StatelessWorker, Count> for MockJob {+ fn map(self, _worker: &StatelessWorker) -> Count {Count(self.contents.into_iter().filter(|d| *d == 0).count())}}@@ -150,7 +155,9 @@ mod tests {let job = bincode::serialize(&Task::Job(MockJob { contents })).unwrap();let mut stream = MockTcpStream::new(job);- Worker::run_stream::<MockJob, _, _>(&mut stream).expect("worker failed");+ StatelessWorker::default()+ .run_stream::<StatelessWorker, MockJob, _, _>(&mut stream)+ .expect("worker failed");let result_bytes = &stream.result[..stream.result.len() - END_OF_MESSAGE.len()];let res: Count = bincode::deserialize(result_bytes).unwrap();diff --git a/src/ranking/centrality_store.rs b/src/ranking/centrality_store.rsnew file mode 100644index 00000000..59fc675b--- /dev/null+++ b/src/ranking/centrality_store.rs@@ -0,0 +1,64 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;++pub struct CentralityStore {+ inner: sled::Tree,+}++impl CentralityStore {+ pub fn new<P: AsRef<Path>>(path: P) -> Self {+ let db = sled::Config::default()+ .path(path)+ .use_compression(true)+ .mode(sled::Mode::LowSpace)+ .open()+ .expect("Failed to open database");++ Self {+ inner: db+ .open_tree("centrality_store")+ .expect("Failed to open tree"),+ }+ }++ pub fn insert(&mut self, key: String, centrality: f64) {+ let key_bytes = bincode::serialize(&key).expect("Failed to serialize key");+ let centrality_bytes =+ bincode::serialize(¢rality).expect("Failed to serialize centrality");++ self.inner+ .insert(key_bytes, centrality_bytes)+ .expect("Failed to insert into tree");+ }++ pub fn get(&self, key: &str) -> Option<f64> {+ let key_bytes = bincode::serialize(key).expect("Failed to serialize key");+ self.inner+ .get(key_bytes)+ .expect("Failed to retrieve value from key")+ .map(|bytes| bincode::deserialize(&bytes).expect("Failed to deserialize value"))+ }++ pub fn append(&mut self, it: impl Iterator<Item = (String, f64)>) {+ it.filter(|(_, value)| *value != 0.0)+ .for_each(|(key, value)| {+ self.insert(key, value);+ });+ self.inner.flush().unwrap();+ }+}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 2c1279d2..dc3f7513 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -90,7 +90,7 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {fn score(&mut self, doc: DocId, score: Score) -> f64 {let centrality = self.centrality_reader.get(doc);let navigational = self.navigational_score(&self.sorted_dedupped_terms, doc);- score as f64 + 1000.0 * centrality + 100.0 * navigational+ score as f64 + 10_000.0 * centrality + 100.0 * navigational}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 43934c9f..f1f50334 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod centrality_store;mod initial;use std::sync::Arc;