Skip to content

web search done right

Indexing using mapreduce, centrality store, mapreduce worker state

Mikkel Denkerda6c7503758d
diff --git a/.gitignore b/.gitignore
index 79b16f44..129ac8b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,5 @@
flamegraph.svg
warc_files/
webgraph
+centrality
+index
diff --git a/configs/centrality.toml b/configs/centrality.toml
new file mode 100644
index 00000000..a7300b3b
--- /dev/null
+++ b/configs/centrality.toml
@@ -0,0 +1,3 @@
+type = "Centrality"
+webgraph_path = "webgraph"
+output_path = "centrality"
\ No newline at end of file
diff --git a/configs/indexer.toml b/configs/indexer.toml
index 4efb218b..3b2b1f95 100644
--- a/configs/indexer.toml
+++ b/configs/indexer.toml
@@ -1,12 +1,12 @@
type = "Indexer"
-warc_paths_file = "warc.paths"
+mode = "Local"
-# [warc_source]
-# type = "S3"
-# name = "us-east-1"
-# endpoint = "https://s3.us-east-1.amazonaws.com"
-# bucket = "commoncrawl"
+warc_paths_file = "warc.paths"
+centrality_store_path = "centrality"
[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
+type = "Local"
+folder = "./warc_files"
+names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]
+# type = "HTTP"
+# base_url = "https://data.commoncrawl.org/"
\ No newline at end of file
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
index 37fb6d26..f8206863 100644
--- a/examples/mapreduce_manager.rs
+++ b/examples/mapreduce_manager.rs
@@ -1,6 +1,6 @@
use std::net::SocketAddr;
-use cuely::mapreduce::{Map, MapReduce, Reduce};
+use cuely::mapreduce::{Map, MapReduce, Reduce, StatelessWorker};
use serde::{Deserialize, Serialize};
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -13,8 +13,8 @@ struct Job {
#[derive(Serialize, Deserialize, Debug)]
struct Count(usize);
-impl Map<Count> for Job {
- fn map(self) -> Count {
+impl Map<StatelessWorker, Count> for Job {
+ fn map(self, _worker: &StatelessWorker) -> Count {
std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(1)
}
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
index 15c25a09..fdca5e9c 100644
--- a/examples/mapreduce_worker.rs
+++ b/examples/mapreduce_worker.rs
@@ -1,6 +1,6 @@
use std::net::SocketAddr;
-use cuely::mapreduce::{Map, Reduce, Worker};
+use cuely::mapreduce::{Map, Reduce, StatelessWorker, Worker};
use serde::{Deserialize, Serialize};
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -13,8 +13,8 @@ struct Job {
#[derive(Serialize, Deserialize, Debug)]
struct Count(usize);
-impl Map<Count> for Job {
- fn map(self) -> Count {
+impl Map<StatelessWorker, Count> for Job {
+ fn map(self, _worker: &StatelessWorker) -> Count {
std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(1)
}
@@ -34,6 +34,7 @@ fn main() {
let args: Vec<_> = std::env::args().collect();
- Worker::run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())
+ StatelessWorker::default()
+ .run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())
.expect("failed to run worker");
}
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index 494a0a47..6b80bff8 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use clap::Parser;
-use cuely::entrypoint::{Indexer, WebgraphEntrypoint};
+use cuely::entrypoint::{CentralityEntrypoint, Indexer, WebgraphEntrypoint};
use cuely::Config;
use std::fs;
use tracing::Level;
@@ -40,12 +40,14 @@ fn main() {
match config {
Config::Indexer(config) => {
- Indexer::from(config)
+ Indexer::new(config, args.mapreduce_worker_addr)
.run()
.expect("Failed to index documents");
}
Config::Webgraph(config) => WebgraphEntrypoint::new(config, args.mapreduce_worker_addr)
.run()
.expect("Failed to build webgraph"),
+
+ Config::Centrality(config) => CentralityEntrypoint::from(config).run(),
}
}
diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rs
new file mode 100644
index 00000000..0d85a9ef
--- /dev/null
+++ b/src/entrypoint/centrality.rs
@@ -0,0 +1,46 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder, CentralityConfig,
+};
+
+pub struct CentralityEntrypoint {
+ config: CentralityConfig,
+}
+
+impl From<CentralityConfig> for CentralityEntrypoint {
+ fn from(config: CentralityConfig) -> Self {
+ Self { config }
+ }
+}
+
+impl CentralityEntrypoint {
+ pub fn run(self) {
+ let graph = WebgraphBuilder::new(&self.config.webgraph_path)
+ .with_host_graph()
+ .open();
+
+ let mut centrality_store = CentralityStore::new(&self.config.output_path);
+
+ centrality_store.append(
+ graph
+ .host_harmonic_centrality()
+ .into_iter()
+ .map(|(node, centrality)| (node.name, centrality)),
+ );
+ }
+}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 8163999f..52ba8ca2 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -13,55 +13,203 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::fs::File;
-use std::io::{self, BufRead};
+use std::net::SocketAddr;
+use std::path::Path;
+use serde::{Deserialize, Serialize};
+use tracing::{debug, info, trace};
+
+use crate::index::{FrozenIndex, Index};
+use crate::mapreduce::{Map, MapReduce, Reduce, Worker};
+use crate::ranking::centrality_store::CentralityStore;
use crate::warc::WarcFile;
-use crate::webpage::Html;
-use crate::{IndexingConfig, Result};
+use crate::webpage::{Html, Link, Webpage};
+use crate::{
+ HttpConfig, IndexingConfig, IndexingLocalConfig, IndexingMasterConfig, IndexingWorkerConfig,
+ LocalConfig, Result, WarcSource,
+};
pub struct Indexer {
- warc_paths: Vec<String>,
config: IndexingConfig,
+ worker_addr: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+enum JobConfig {
+ Http(HttpConfig),
+ Local(LocalConfig),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Job {
+ config: JobConfig,
+ warc_path: String,
+ base_path: String,
+}
+
+struct IndexingWorker {
+ centrality_store: CentralityStore,
}
-impl From<IndexingConfig> for Indexer {
- fn from(config: IndexingConfig) -> Self {
- let file = File::open(&config.warc_paths_file).unwrap();
- let mut warc_paths = Vec::new();
+impl IndexingWorker {
+ fn new(centrality_store_path: String) -> Self {
+ Self {
+ centrality_store: CentralityStore::new(centrality_store_path),
+ }
+ }
+}
+
+impl Worker for IndexingWorker {}
+
+impl Map<IndexingWorker, FrozenIndex> for Job {
+ fn map(self, worker: &IndexingWorker) -> FrozenIndex {
+ let name = self.warc_path.split('/').last().unwrap();
+
+ info!("processing {}", name);
- for line in io::BufReader::new(file).lines() {
- warc_paths.push(line.unwrap());
+ let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();
+
+ let source = match self.config {
+ JobConfig::Http(config) => WarcSource::HTTP(config),
+ JobConfig::Local(config) => WarcSource::Local(config),
+ };
+
+ debug!("downlooading warc file");
+ let file = WarcFile::download(source, &self.warc_path).unwrap();
+ debug!("finished downloading");
+
+ for record in file.records().flatten() {
+ let html = Html::parse(&record.response.body, &record.request.url);
+ let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph
+ let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();
+
+ trace!("inserting webpage: {:?}", html.url());
+
+ let webpage = Webpage {
+ html,
+ backlinks,
+ centrality,
+ };
+
+ if let Err(err) = index.insert(webpage) {
+ debug!("{:?}", err);
+ }
}
+ index.commit().unwrap();
+
+ info!("{} done", name);
- Self { warc_paths, config }
+ index.into()
+ }
+}
+
+impl Reduce<FrozenIndex> for Index {
+ fn reduce(self, element: FrozenIndex) -> Self {
+ let other = element.into();
+
+ self.merge(other)
+ }
+}
+
+impl Reduce<Index> for Index {
+ fn reduce(self, element: Index) -> Self {
+ self.merge(element)
}
}
impl Indexer {
- pub fn run(self) -> Result<()> {
- self.warc_paths
- .into_iter()
- .map(|warc_path| {
- let source = self.config.warc_source.clone();
+ pub fn new(config: IndexingConfig, worker_addr: Option<String>) -> Self {
+ Self {
+ config,
+ worker_addr,
+ }
+ }
+ fn run_master(config: &IndexingMasterConfig) -> Result<()> {
+ info!("Running master for index construction");
- WarcFile::download(source, &warc_path)
- })
- .map(|warc| {
- if warc.is_err() {
- return;
- }
- let warc = warc.unwrap();
+ let warc_paths = config.warc_source.paths()?;
+
+ let workers: Vec<SocketAddr> = config
+ .workers
+ .iter()
+ .map(|worker| worker.parse().unwrap())
+ .collect();
- for record in warc.records().flatten() {
- let _webpage = Html::parse(&record.response.body, &record.request.url);
- // println!("TEST: {:?}", webpage.title());
- // println!();
+ let job_config = match config.warc_source.clone() {
+ WarcSource::S3(_) => todo!("s3 not supported yet"),
+ WarcSource::HTTP(config) => JobConfig::Http(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ };
+
+ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> =
+ Box::new(warc_paths.into_iter().map(|warc_path| {
+ Job {
+ config: job_config.clone(),
+ warc_path,
+ base_path: config
+ .graph_base_path
+ .clone()
+ .unwrap_or_else(|| "index".to_string()),
}
- // panic!();
+ }));
+
+ if let Some(limit) = config.limit_warc_files {
+ warc_paths = Box::new(warc_paths.take(limit));
+ }
+
+ let _index: Index = warc_paths
+ .map_reduce(&workers)
+ .expect("failed to build index");
+
+ Ok(())
+ }
+
+ fn run_worker(worker_addr: String, config: &IndexingWorkerConfig) -> Result<()> {
+ IndexingWorker::new(config.centrality_store_path.clone()).run::<Job, FrozenIndex>(
+ worker_addr
+ .parse::<SocketAddr>()
+ .expect("Could not parse worker address"),
+ )?;
+ Ok(())
+ }
+
+ fn run_locally(config: &IndexingLocalConfig) -> Result<()> {
+ let warc_paths = config.warc_source.paths()?;
+
+ let job_config = match config.warc_source.clone() {
+ WarcSource::S3(_) => todo!("s3 not supported yet"),
+ WarcSource::HTTP(config) => JobConfig::Http(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ };
+
+ let worker = IndexingWorker::new(config.centrality_store_path.clone());
+
+ warc_paths
+ .into_iter()
+ .map(|path| Job {
+ config: job_config.clone(),
+ warc_path: path,
+ base_path: "index".to_string(),
})
- .for_each(drop);
+ .map(|job| job.map(&worker))
+ .fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {
+ Some(acc) => Some(acc.reduce(elem)),
+ None => Some(elem.into()),
+ });
Ok(())
}
+
+ pub fn run(&self) -> Result<()> {
+ match &self.config {
+ IndexingConfig::Master(config) => Self::run_master(config),
+ IndexingConfig::Worker(config) => Self::run_worker(
+ self.worker_addr
+ .clone()
+ .expect("Worker address not specified"),
+ config,
+ ),
+ IndexingConfig::Local(config) => Self::run_locally(config),
+ }
+ }
}
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index d4f534ca..c7ca9826 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -13,8 +13,10 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod centrality;
mod indexer;
mod webgraph;
+pub use centrality::CentralityEntrypoint;
pub use indexer::Indexer;
pub use webgraph::WebgraphEntrypoint;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index a6c9b62e..3140f606 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- mapreduce::{Map, MapReduce, Reduce, Worker},
+ mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},
warc::WarcFile,
webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},
webpage::{self, Html},
@@ -38,8 +38,8 @@ struct Job {
graph_base_path: String,
}
-impl Map<FrozenWebgraph> for Job {
- fn map(self) -> FrozenWebgraph {
+impl Map<StatelessWorker, FrozenWebgraph> for Job {
+ fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {
let name = self.warc_path.split('/').last().unwrap();
info!("processing {}", name);
@@ -102,7 +102,11 @@ impl Reduce<FrozenWebgraph> for Webgraph {
impl Reduce<Webgraph> for Webgraph {
fn reduce(mut self, element: Webgraph) -> Self {
+ let other_path = element.path.clone();
+
self.merge(element);
+
+ std::fs::remove_dir_all(other_path).unwrap();
self
}
}
@@ -160,7 +164,7 @@ impl WebgraphEntrypoint {
}
fn run_worker(worker_addr: String) -> Result<()> {
- Worker::run::<Job, FrozenWebgraph>(
+ StatelessWorker::default().run::<Job, FrozenWebgraph>(
worker_addr
.parse::<SocketAddr>()
.expect("Could not parse worker address"),
@@ -176,6 +180,7 @@ impl WebgraphEntrypoint {
WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
+ let worker = StatelessWorker::default();
warc_paths
.into_iter()
@@ -184,7 +189,7 @@ impl WebgraphEntrypoint {
warc_path: path,
graph_base_path: "webgraph".to_string(),
})
- .map(|job| job.map())
+ .map(|job| job.map(&worker))
.fold(
None,
|acc: Option<Webgraph>, elem: FrozenWebgraph| match acc {
diff --git a/src/index.rs b/src/index.rs
index 5f82e7ca..b5ea269f 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -29,7 +29,7 @@ use std::fs;
use std::path::Path;
pub struct Index {
- path: String,
+ pub path: String,
tantivy_index: tantivy::Index,
writer: tantivy::IndexWriter,
reader: tantivy::IndexReader,
diff --git a/src/lib.rs b/src/lib.rs
index 356aa182..0bef245c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -43,12 +43,40 @@ pub mod webpage;
pub enum Config {
Indexer(IndexingConfig),
Webgraph(WebgraphConfig),
+ Centrality(CentralityConfig),
}
#[derive(Debug, Deserialize, Clone)]
-pub struct IndexingConfig {
+pub struct CentralityConfig {
+ pub webgraph_path: String,
+ pub output_path: String,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+#[serde(tag = "mode")]
+pub enum IndexingConfig {
+ Master(IndexingMasterConfig),
+ Worker(IndexingWorkerConfig),
+ Local(IndexingLocalConfig),
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct IndexingMasterConfig {
+ limit_warc_files: Option<usize>,
warc_source: WarcSource,
- warc_paths_file: String,
+ workers: Vec<String>,
+ graph_base_path: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct IndexingWorkerConfig {
+ centrality_store_path: String,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct IndexingLocalConfig {
+ warc_source: WarcSource,
+ centrality_store_path: String,
}
#[derive(Debug, Deserialize, Clone)]
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 9cb4372d..67ec53fe 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -1,4 +1,4 @@
-use super::{Error, Result};
+use super::{Error, Result, Worker};
use super::{Map, Reduce};
use crate::exponential_backoff::ExponentialBackoff;
use crate::mapreduce::Task;
@@ -43,9 +43,10 @@ impl RemoteWorker {
Err(Error::NoResponse)
}
- fn perform<I, O>(&self, job: &I) -> Result<O>
+ fn perform<W, I, O>(&self, job: &I) -> Result<O>
where
- I: Map<O> + Send,
+ W: Worker,
+ I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
let mut stream = self.connect()?;
@@ -77,9 +78,10 @@ impl RemoteWorker {
Ok(bincode::deserialize(&bytes)?)
}
- fn stop<I, O>(&self) -> Result<()>
+ fn stop<W, I, O>(&self) -> Result<()>
where
- I: Map<O> + Send,
+ W: Worker,
+ I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
debug!("closing worker {:}", self.addr);
@@ -174,14 +176,15 @@ impl WorkerPool {
}
}
- fn stop_workers<I, O>(&self)
+ fn stop_workers<W, I, O>(&self)
where
- I: Map<O> + Send,
+ W: Worker,
+ I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
let mut failing_workers = Vec::new();
for worker in &self.all_workers {
- if worker.stop::<I, O>().is_err() {
+ if worker.stop::<W, I, O>().is_err() {
failing_workers.push(worker)
}
}
@@ -209,9 +212,10 @@ impl Manager {
}
}
- fn try_map<I, O>(&self, job: &I) -> Result<O>
+ fn try_map<W, I, O>(&self, job: &I) -> Result<O>
where
- I: Map<O> + Send,
+ W: Worker,
+ I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
loop {
@@ -229,9 +233,10 @@ impl Manager {
/// Execute job on one of the remote machines. If the remote machine fails for some reason,
/// the job should be allocated to another machine.
- pub fn map<I, O>(&self, job: I) -> O
+ pub fn map<W, I, O>(&self, job: I) -> O
where
- I: Map<O> + Send,
+ W: Worker,
+ I: Map<W, O> + Send,
O: Serialize + DeserializeOwned + Send,
{
loop {
@@ -266,9 +271,10 @@ impl Manager {
}
}
- fn get_results<I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
+ fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
- I: Map<O1> + Send,
+ W: Worker,
+ I: Map<W, O1> + Send,
O1: Serialize + DeserializeOwned + Send,
O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
@@ -287,7 +293,7 @@ impl Manager {
.progress_chars("#>-"),
);
jobs.par_bridge()
- .map(|job| self.map::<I, O1>(job))
+ .map(|job| self.map::<W, I, O1>(job))
.progress_with(pb)
.fold(
|| None,
@@ -302,7 +308,7 @@ impl Manager {
}
None => {
jobs.par_bridge()
- .map(|job| self.map::<I, O1>(job))
+ .map(|job| self.map::<W, I, O1>(job))
.fold(
|| None,
|acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),
@@ -320,14 +326,15 @@ impl Manager {
x
}
- pub fn run<I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
+ pub fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
- I: Map<O1> + Send,
+ W: Worker,
+ I: Map<W, O1> + Send,
O1: Serialize + DeserializeOwned + Send,
O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
let result = self.get_results(jobs);
- self.pool.stop_workers::<I, O1>();
+ self.pool.stop_workers::<W, I, O1>();
result
}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index 87d56a1b..d5e82e30 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -22,6 +22,7 @@ mod worker;
pub use manager::Manager;
use thiserror::Error;
+pub use worker::StatelessWorker;
pub use worker::Worker;
pub(crate) type Result<T> = std::result::Result<T, Error>;
@@ -44,22 +45,24 @@ pub enum Error {
NoResponse,
}
-pub trait Map<T>
+pub trait Map<W, T>
where
Self: Serialize + DeserializeOwned + Send,
T: Serialize + DeserializeOwned + Send,
+ W: Worker,
{
- fn map(self) -> T;
+ fn map(self, worker: &W) -> T;
}
pub trait Reduce<T> {
fn reduce(self, element: T) -> Self;
}
-pub trait MapReduce<I, O1, O2>
+pub trait MapReduce<W, I, O1, O2>
where
Self: Sized + Iterator<Item = I> + Send,
- I: Map<O1>,
+ W: Worker,
+ I: Map<W, O1>,
O1: Serialize + DeserializeOwned + Send,
O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
@@ -68,10 +71,11 @@ where
manager.run(self)
}
}
-impl<I, O1, O2, T> MapReduce<I, O1, O2> for T
+impl<W, I, O1, O2, T> MapReduce<W, I, O1, O2> for T
where
+ W: Worker,
T: Iterator<Item = I> + Sized + Send,
- I: Map<O1>,
+ I: Map<W, O1>,
O1: Serialize + DeserializeOwned + Send,
O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rs
index 289c59bd..f3c0e641 100644
--- a/src/mapreduce/worker.rs
+++ b/src/mapreduce/worker.rs
@@ -9,17 +9,19 @@ use super::{Error, Map, Result, Task};
use serde::{de::DeserializeOwned, Serialize};
use tracing::{debug, info};
-pub struct Worker {}
+#[derive(Default)]
+pub struct StatelessWorker {}
-enum State {
+pub enum State {
Continue,
Finished,
}
-impl Worker {
- fn run_stream<I, O, S>(mut stream: S) -> Result<State>
+pub trait Worker {
+ fn run_stream<W, I, O, S>(&self, mut stream: S) -> Result<State>
where
- I: Map<O>,
+ Self: Sized,
+ I: Map<Self, O>,
O: Serialize + DeserializeOwned + Send,
S: Read + Write + Unpin,
{
@@ -47,7 +49,7 @@ impl Worker {
match bincode::deserialize::<Task<I>>(&bytes)? {
Task::Job(job) => {
debug!("received job");
- let res = job.map();
+ let res = job.map(self);
let bytes = bincode::serialize(&res)?;
debug!("serialized result into {} bytes", bytes.len());
stream.write_all(&bytes)?;
@@ -62,9 +64,10 @@ impl Worker {
Ok(State::Continue)
}
- pub fn run<I, O>(addr: SocketAddr) -> Result<()>
+ fn run<I, O>(&self, addr: SocketAddr) -> Result<()>
where
- I: Map<O>,
+ Self: Sized,
+ I: Map<Self, O>,
O: Serialize + DeserializeOwned + Send,
{
let listener = TcpListener::bind(addr)?;
@@ -73,7 +76,7 @@ impl Worker {
loop {
let (socket, _) = listener.accept()?;
debug!("received connection");
- match Worker::run_stream::<I, O, _>(socket)? {
+ match self.run_stream::<Self, I, O, _>(socket)? {
State::Finished => break,
State::Continue => {}
}
@@ -83,6 +86,8 @@ impl Worker {
}
}
+impl Worker for StatelessWorker {}
+
#[cfg(test)]
mod tests {
use serde::Deserialize;
@@ -138,8 +143,8 @@ mod tests {
#[derive(Serialize, Deserialize, Debug)]
struct Count(usize);
- impl Map<Count> for MockJob {
- fn map(self) -> Count {
+ impl Map<StatelessWorker, Count> for MockJob {
+ fn map(self, _worker: &StatelessWorker) -> Count {
Count(self.contents.into_iter().filter(|d| *d == 0).count())
}
}
@@ -150,7 +155,9 @@ mod tests {
let job = bincode::serialize(&Task::Job(MockJob { contents })).unwrap();
let mut stream = MockTcpStream::new(job);
- Worker::run_stream::<MockJob, _, _>(&mut stream).expect("worker failed");
+ StatelessWorker::default()
+ .run_stream::<StatelessWorker, MockJob, _, _>(&mut stream)
+ .expect("worker failed");
let result_bytes = &stream.result[..stream.result.len() - END_OF_MESSAGE.len()];
let res: Count = bincode::deserialize(result_bytes).unwrap();
diff --git a/src/ranking/centrality_store.rs b/src/ranking/centrality_store.rs
new file mode 100644
index 00000000..59fc675b
--- /dev/null
+++ b/src/ranking/centrality_store.rs
@@ -0,0 +1,64 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::path::Path;
+
+pub struct CentralityStore {
+ inner: sled::Tree,
+}
+
+impl CentralityStore {
+ pub fn new<P: AsRef<Path>>(path: P) -> Self {
+ let db = sled::Config::default()
+ .path(path)
+ .use_compression(true)
+ .mode(sled::Mode::LowSpace)
+ .open()
+ .expect("Failed to open database");
+
+ Self {
+ inner: db
+ .open_tree("centrality_store")
+ .expect("Failed to open tree"),
+ }
+ }
+
+ pub fn insert(&mut self, key: String, centrality: f64) {
+ let key_bytes = bincode::serialize(&key).expect("Failed to serialize key");
+ let centrality_bytes =
+ bincode::serialize(&centrality).expect("Failed to serialize centrality");
+
+ self.inner
+ .insert(key_bytes, centrality_bytes)
+ .expect("Failed to insert into tree");
+ }
+
+ pub fn get(&self, key: &str) -> Option<f64> {
+ let key_bytes = bincode::serialize(key).expect("Failed to serialize key");
+ self.inner
+ .get(key_bytes)
+ .expect("Failed to retrieve value from key")
+ .map(|bytes| bincode::deserialize(&bytes).expect("Failed to deserialize value"))
+ }
+
+ pub fn append(&mut self, it: impl Iterator<Item = (String, f64)>) {
+ it.filter(|(_, value)| *value != 0.0)
+ .for_each(|(key, value)| {
+ self.insert(key, value);
+ });
+ self.inner.flush().unwrap();
+ }
+}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 2c1279d2..dc3f7513 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -90,7 +90,7 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
fn score(&mut self, doc: DocId, score: Score) -> f64 {
let centrality = self.centrality_reader.get(doc);
let navigational = self.navigational_score(&self.sorted_dedupped_terms, doc);
- score as f64 + 1000.0 * centrality + 100.0 * navigational
+ score as f64 + 10_000.0 * centrality + 100.0 * navigational
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 43934c9f..f1f50334 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod centrality_store;
mod initial;
use std::sync::Arc;