web search done right
make webgraph send + sync and retrieve backlinks from webgraph during indexing
Mikkel Denker9e79724e44de
diff --git a/configs/webgraph/profile.toml b/configs/webgraph/profile.tomlindex e9f56997..2070795d 100644--- a/configs/webgraph/profile.toml+++ b/configs/webgraph/profile.toml@@ -1,4 +1,6 @@+graph_base_path = "data/webgraph"+[warc_source]type = "Local"-folder = "./warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]+folder = "data/warc_files"+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 4e0d7436..33e4fe87 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -26,6 +26,7 @@ use crate::index::{FrozenIndex, Index};use crate::mapreduce::{Map, MapReduce, Reduce, Worker};use crate::ranking::centrality_store::CentralityStore;use crate::warc::WarcFile;+use crate::webgraph::{Node, Webgraph, WebgraphBuilder};use crate::webpage::{Html, Link, Webpage};use crate::{HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,@@ -48,12 +49,20 @@ struct Job {struct IndexingWorker {centrality_store: CentralityStore,+ webgraph: Option<Webgraph>,}impl IndexingWorker {- fn new(centrality_store_path: String) -> Self {+ fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {Self {centrality_store: CentralityStore::new(centrality_store_path),+ webgraph: webgraph_path.map(|path| {+ WebgraphBuilder::new(path)+ .with_full_graph()+ .with_host_graph()+ .read_only(true)+ .open()+ }),}}}@@ -88,7 +97,21 @@ impl Map<IndexingWorker, FrozenIndex> for Job {}){let html = Html::parse(&record.response.body, &record.request.url);- let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph+ let backlinks: Vec<Link> = worker+ .webgraph+ .as_ref()+ .map(|webgraph| {+ webgraph+ .ingoing_edges(Node::from(html.url()))+ .into_iter()+ .map(|edge| Link {+ source: edge.from.name.into(),+ destination: edge.to.name.into(),+ text: edge.label,+ })+ .collect()+ })+ .unwrap_or_else(Vec::new);let centrality = worker.centrality_store.get(html.url().host_without_specific_subdomains())@@ -197,8 +220,12 @@ impl Indexer {Ok(())}- pub fn run_worker(worker_addr: String, centrality_store_path: String) -> Result<()> {- IndexingWorker::new(centrality_store_path).run::<Job, FrozenIndex>(+ pub fn run_worker(+ worker_addr: String,+ centrality_store_path: String,+ webgraph_path: Option<String>,+ ) -> Result<()> {+ IndexingWorker::new(centrality_store_path, webgraph_path).run::<Job, FrozenIndex>(worker_addr.parse::<SocketAddr>().expect("Could not parse worker address"),@@ -214,7 +241,10 @@ impl Indexer {WarcSource::Local(config) => JobConfig::Local(config),};- let worker = IndexingWorker::new(config.centrality_store_path.clone());+ let worker = IndexingWorker::new(+ config.centrality_store_path.clone(),+ config.webgraph_path.clone(),+ );warc_paths.into_iter()diff --git a/src/kv/rocksdb_store.rs b/src/kv/rocksdb_store.rsindex 435a3885..e8881581 100644--- a/src/kv/rocksdb_store.rs+++ b/src/kv/rocksdb_store.rs@@ -16,7 +16,9 @@use std::{fs, marker::PhantomData};-use rocksdb::{DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, SingleThreaded, DB};+use rocksdb::{+ DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options, SingleThreaded, DB,+};use serde::{de::DeserializeOwned, Serialize};use crate::kv::Kv;@@ -36,6 +38,22 @@ impl RocksDbStore {Box::new(DB::open_default(path).expect("unable to open rocks db"))}++ pub fn open_read_only<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>+ where+ P: AsRef<std::path::Path>,+ K: Serialize + DeserializeOwned + 'static,+ V: Serialize + DeserializeOwned + 'static,+ {+ if !path.as_ref().exists() {+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");+ }++ let mut options = Options::default();+ options.create_if_missing(true);++ Box::new(DB::open_for_read_only(&options, path, false).expect("unable to open rocks db"))+ }}impl<K, V> Kv<K, V> for rocksdb::DBdiff --git a/src/lib.rs b/src/lib.rsindex 96b0d7b7..cf8d4406 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -67,6 +67,7 @@ pub struct IndexingLocalConfig {limit_warc_files: Option<usize>,warc_source: WarcSource,batch_size: Option<usize>,+ webgraph_path: Option<String>,centrality_store_path: String,}diff --git a/src/main.rs b/src/main.rsindex c9cc364c..c1962384 100644--- a/src/main.rs+++ b/src/main.rs@@ -68,6 +68,7 @@ enum IndexingOptions {Worker {address: String,centrality_store_path: String,+ webgraph_path: Option<String>,},Local {config_path: String,@@ -100,8 +101,9 @@ fn main() -> Result<()> {IndexingOptions::Worker {address,centrality_store_path,+ webgraph_path,} => {- entrypoint::Indexer::run_worker(address, centrality_store_path)?;+ entrypoint::Indexer::run_worker(address, centrality_store_path, webgraph_path)?;}IndexingOptions::Local { config_path } => {let config = load_toml_config(&config_path);diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rsindex 1f4b6ad1..e5420797 100644--- a/src/webgraph/graph_store.rs+++ b/src/webgraph/graph_store.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::{- cell::RefCell, collections::HashMap, hash::Hash, marker::PhantomData, ops::Div, path::Path,+ collections::HashMap, hash::Hash, marker::PhantomData, ops::Div, path::Path, sync::Mutex,};use lru::LruCache;@@ -146,11 +146,11 @@ where}pub struct GraphStore<S> {- pub(crate) adjacency: RefCell<Adjacency>,- pub(crate) reversed_adjacency: RefCell<Adjacency>,- pub(crate) node2id: RefCell<CachedTree<Node, NodeID>>,- pub(crate) id2node: RefCell<BlockedCachedTree<NodeID, Node>>,- pub(crate) meta: RefCell<CachedTree<String, u64>>,+ pub(crate) adjacency: Mutex<Adjacency>,+ pub(crate) reversed_adjacency: Mutex<Adjacency>,+ pub(crate) node2id: Mutex<CachedTree<Node, NodeID>>,+ pub(crate) id2node: Mutex<BlockedCachedTree<NodeID, Node>>,+ pub(crate) meta: Mutex<CachedTree<String, u64>>,pub(crate) store: PhantomData<S>,}@@ -164,9 +164,14 @@ impl<S: Store> GraphStore<S> {S::open(path)}+ pub fn open_read_only<P: AsRef<Path>>(path: P) -> Self {+ S::open_read_only(path)+ }+fn next_id(&self) -> NodeID {self.meta- .borrow_mut()+ .lock()+ .unwrap().get(&"next_id".to_string()).cloned().unwrap_or(0)@@ -176,7 +181,8 @@ impl<S: Store> GraphStore<S> {let current_id = self.next_id();let next_id = current_id + 1;self.meta- .borrow_mut()+ .lock()+ .unwrap().insert("next_id".to_string(), next_id);}@@ -187,14 +193,14 @@ impl<S: Store> GraphStore<S> {}fn assign_id(&self, node: Node, id: NodeID) {- self.node2id.borrow_mut().insert(node.clone(), id);- self.id2node.borrow_mut().insert(id, &mut |block| {+ self.node2id.lock().unwrap().insert(node.clone(), id);+ self.id2node.lock().unwrap().insert(id, &mut |block| {block.insert(id, node.clone());});}fn id_or_assign(&self, node: Node) -> NodeID {- if let Some(id) = self.node2id.borrow_mut().get(&node) {+ if let Some(id) = self.node2id.lock().unwrap().get(&node) {return *id;}let id = self.id_and_increment();@@ -205,7 +211,8 @@ impl<S: Store> GraphStore<S> {#[allow(unused)]pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {self.adjacency- .borrow_mut()+ .lock()+ .unwrap().edges(node).into_iter().map(|edge| Edge {@@ -218,7 +225,8 @@ impl<S: Store> GraphStore<S> {pub fn ingoing_edges(&self, node: NodeID) -> Vec<Edge> {self.reversed_adjacency- .borrow_mut()+ .lock()+ .unwrap().edges(node).into_iter().map(|edge| Edge {@@ -231,7 +239,8 @@ impl<S: Store> GraphStore<S> {pub fn nodes(&self) -> impl Iterator<Item = NodeID> {self.node2id- .borrow_mut()+ .lock()+ .unwrap().iter().map(|(_, id)| id).collect::<Vec<u64>>()@@ -243,27 +252,29 @@ impl<S: Store> GraphStore<S> {let to_id = self.id_or_assign(to);self.adjacency- .borrow_mut()+ .lock()+ .unwrap().insert(from_id, to_id, label.clone());self.reversed_adjacency- .borrow_mut()+ .lock()+ .unwrap().insert(to_id, from_id, label);}pub fn node2id(&self, node: &Node) -> Option<NodeID> {- self.node2id.borrow_mut().get(node).cloned()+ self.node2id.lock().unwrap().get(node).cloned()}pub fn id2node(&self, id: &NodeID) -> Option<Node> {- self.id2node.borrow_mut().get(id).cloned()+ self.id2node.lock().unwrap().get(id).cloned()}pub fn flush(&self) {- self.adjacency.borrow_mut().tree.inner.flush();- self.reversed_adjacency.borrow_mut().tree.inner.flush();- self.node2id.borrow_mut().flush();- self.id2node.borrow_mut().inner.flush();- self.meta.borrow_mut().flush();+ self.adjacency.lock().unwrap().tree.inner.flush();+ self.reversed_adjacency.lock().unwrap().tree.inner.flush();+ self.node2id.lock().unwrap().flush();+ self.id2node.lock().unwrap().inner.flush();+ self.meta.lock().unwrap().flush();}pub fn edges(&self) -> EdgeIterator<'_> {@@ -291,24 +302,55 @@ impl Store for RocksDbStore {let meta = RocksDbStore::open(path.as_ref().join("meta"));GraphStore {- adjacency: RefCell::new(Adjacency {+ adjacency: Mutex::new(Adjacency {+ tree: BlockedCachedTree {+ inner: CachedTree::new(adjacency, 10_000),+ block_size: 1_024,+ },+ }),+ reversed_adjacency: Mutex::new(Adjacency {+ tree: BlockedCachedTree {+ inner: CachedTree::new(reversed_adjacency, 10_000),+ block_size: 1_024,+ },+ }),+ node2id: Mutex::new(CachedTree::new(node2id, 100_000)),+ id2node: Mutex::new(BlockedCachedTree {+ inner: CachedTree::new(id2node, 100_000),+ block_size: 1_024,+ }),+ meta: Mutex::new(CachedTree::new(meta, 1_000)),+ store: Default::default(),+ }+ }++ fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self> {+ let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));+ let reversed_adjacency =+ RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));+ let node2id = RocksDbStore::open_read_only(path.as_ref().join("node2id"));+ let id2node = RocksDbStore::open_read_only(path.as_ref().join("id2node"));+ let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));++ GraphStore {+ adjacency: Mutex::new(Adjacency {tree: BlockedCachedTree {inner: CachedTree::new(adjacency, 10_000),block_size: 1_024,},}),- reversed_adjacency: RefCell::new(Adjacency {+ reversed_adjacency: Mutex::new(Adjacency {tree: BlockedCachedTree {inner: CachedTree::new(reversed_adjacency, 10_000),block_size: 1_024,},}),- node2id: RefCell::new(CachedTree::new(node2id, 100_000)),- id2node: RefCell::new(BlockedCachedTree {+ node2id: Mutex::new(CachedTree::new(node2id, 100_000)),+ id2node: Mutex::new(BlockedCachedTree {inner: CachedTree::new(id2node, 100_000),block_size: 1_024,}),- meta: RefCell::new(CachedTree::new(meta, 1_000)),+ meta: Mutex::new(CachedTree::new(meta, 1_000)),store: Default::default(),}}@@ -348,7 +390,7 @@ mod test {let nodes: Vec<Node> = store.nodes()- .map(|id| store.id2node.borrow_mut().get(&id).unwrap().clone())+ .map(|id| store.id2node.lock().unwrap().get(&id).unwrap().clone()).collect();assert_eq!(nodes, vec![a.clone(), b.clone(), c.clone()]);diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 60069883..ce776e1a 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -17,9 +17,9 @@ mod graph_store;use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use serde::{Deserialize, Serialize};-use std::cell::RefCell;use std::collections::{BinaryHeap, HashMap};use std::path::Path;+use std::sync::Mutex;use std::{cmp, fs};use tracing::info;@@ -62,6 +62,14 @@ impl From<String> for Node {}}+impl From<&Url> for Node {+ fn from(url: &Url) -> Self {+ Self {+ name: url.raw().to_string(),+ }+ }+}+impl From<&str> for Node {fn from(name: &str) -> Self {Self::from(name.to_string())@@ -79,14 +87,15 @@ impl From<Url> for Node {pub struct EdgeIterator<'a> {current_block_idx: usize,blocks: Vec<u64>,- adjacency: &'a RefCell<Adjacency>,+ adjacency: &'a Mutex<Adjacency>,current_block: Option<Box<dyn Iterator<Item = Edge>>>,}impl<'a> EdgeIterator<'a> {- fn new(adjacency: &'a RefCell<Adjacency>) -> EdgeIterator<'a> {+ fn new(adjacency: &'a Mutex<Adjacency>) -> EdgeIterator<'a> {let blocks: Vec<_> = adjacency- .borrow()+ .lock()+ .unwrap().tree.inner.store@@ -107,7 +116,8 @@ impl<'a> EdgeIterator<'a> {let block_id = self.blocks[self.current_block_idx];let block = self.adjacency- .borrow_mut()+ .lock()+ .unwrap().tree.inner.get(&block_id)@@ -150,10 +160,18 @@ pub struct Edge {label: String,}+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct FullEdge {+ pub from: Node,+ pub to: Node,+ pub label: String,+}+pub struct WebgraphBuilder {path: Box<Path>,full_graph_path: Option<Box<Path>>,host_graph_path: Option<Box<Path>>,+ read_only: bool,}impl WebgraphBuilder {@@ -170,6 +188,7 @@ impl WebgraphBuilder {path: path.as_ref().into(),full_graph_path: None,host_graph_path: None,+ read_only: false,}}@@ -183,11 +202,25 @@ impl WebgraphBuilder {self}+ pub fn read_only(mut self, read_only: bool) -> Self {+ self.read_only = read_only;++ self+ }+pub fn open(self) -> Webgraph {- Webgraph {- full_graph: self.full_graph_path.map(GraphStore::open),- host_graph: self.host_graph_path.map(GraphStore::open),- path: self.path.to_str().unwrap().to_string(),+ if self.read_only {+ Webgraph {+ full_graph: self.full_graph_path.map(GraphStore::open_read_only),+ host_graph: self.host_graph_path.map(GraphStore::open_read_only),+ path: self.path.to_str().unwrap().to_string(),+ }+ } else {+ Webgraph {+ full_graph: self.full_graph_path.map(GraphStore::open),+ host_graph: self.host_graph_path.map(GraphStore::open),+ path: self.path.to_str().unwrap().to_string(),+ }}}}@@ -197,6 +230,7 @@ whereSelf: Sized,{fn open<P: AsRef<Path>>(path: P) -> GraphStore<Self>;+ fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self>;fn temporary() -> GraphStore<Self> {Self::open(crate::gen_temp_path())@@ -441,6 +475,26 @@ impl<S: Store> Webgraph<S> {host_graph.flush();}}++ pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {+ if let Some(graph) = &self.full_graph {+ if let Some(node_id) = graph.node2id(&node) {+ graph+ .ingoing_edges(node_id)+ .into_iter()+ .map(|edge| FullEdge {+ from: graph.id2node(&edge.from).unwrap(),+ to: graph.id2node(&edge.to).unwrap(),+ label: edge.label,+ })+ .collect()+ } else {+ Vec::new()+ }+ } else {+ Vec::new()+ }+ }}impl From<FrozenWebgraph> for Webgraph {