Skip to content

web search done right

make webgraph send + sync and retrieve backlinks from webgraph during indexing

Mikkel Denker9e79724e44de
diff --git a/configs/webgraph/profile.toml b/configs/webgraph/profile.toml
index e9f56997..2070795d 100644
--- a/configs/webgraph/profile.toml
+++ b/configs/webgraph/profile.toml
@@ -1,4 +1,6 @@
+graph_base_path = "data/webgraph"
+
[warc_source]
type = "Local"
-folder = "./warc_files"
-names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]
+folder = "data/warc_files"
+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 4e0d7436..33e4fe87 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -26,6 +26,7 @@ use crate::index::{FrozenIndex, Index};
use crate::mapreduce::{Map, MapReduce, Reduce, Worker};
use crate::ranking::centrality_store::CentralityStore;
use crate::warc::WarcFile;
+use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
use crate::webpage::{Html, Link, Webpage};
use crate::{
HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,
@@ -48,12 +49,20 @@ struct Job {
struct IndexingWorker {
centrality_store: CentralityStore,
+ webgraph: Option<Webgraph>,
}
impl IndexingWorker {
- fn new(centrality_store_path: String) -> Self {
+ fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {
Self {
centrality_store: CentralityStore::new(centrality_store_path),
+ webgraph: webgraph_path.map(|path| {
+ WebgraphBuilder::new(path)
+ .with_full_graph()
+ .with_host_graph()
+ .read_only(true)
+ .open()
+ }),
}
}
}
@@ -88,7 +97,21 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
})
{
let html = Html::parse(&record.response.body, &record.request.url);
- let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph
+ let backlinks: Vec<Link> = worker
+ .webgraph
+ .as_ref()
+ .map(|webgraph| {
+ webgraph
+ .ingoing_edges(Node::from(html.url()))
+ .into_iter()
+ .map(|edge| Link {
+ source: edge.from.name.into(),
+ destination: edge.to.name.into(),
+ text: edge.label,
+ })
+ .collect()
+ })
+ .unwrap_or_else(Vec::new);
let centrality = worker
.centrality_store
.get(html.url().host_without_specific_subdomains())
@@ -197,8 +220,12 @@ impl Indexer {
Ok(())
}
- pub fn run_worker(worker_addr: String, centrality_store_path: String) -> Result<()> {
- IndexingWorker::new(centrality_store_path).run::<Job, FrozenIndex>(
+ pub fn run_worker(
+ worker_addr: String,
+ centrality_store_path: String,
+ webgraph_path: Option<String>,
+ ) -> Result<()> {
+ IndexingWorker::new(centrality_store_path, webgraph_path).run::<Job, FrozenIndex>(
worker_addr
.parse::<SocketAddr>()
.expect("Could not parse worker address"),
@@ -214,7 +241,10 @@ impl Indexer {
WarcSource::Local(config) => JobConfig::Local(config),
};
- let worker = IndexingWorker::new(config.centrality_store_path.clone());
+ let worker = IndexingWorker::new(
+ config.centrality_store_path.clone(),
+ config.webgraph_path.clone(),
+ );
warc_paths
.into_iter()
diff --git a/src/kv/rocksdb_store.rs b/src/kv/rocksdb_store.rs
index 435a3885..e8881581 100644
--- a/src/kv/rocksdb_store.rs
+++ b/src/kv/rocksdb_store.rs
@@ -16,7 +16,9 @@
use std::{fs, marker::PhantomData};
-use rocksdb::{DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, SingleThreaded, DB};
+use rocksdb::{
+ DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options, SingleThreaded, DB,
+};
use serde::{de::DeserializeOwned, Serialize};
use crate::kv::Kv;
@@ -36,6 +38,22 @@ impl RocksDbStore {
Box::new(DB::open_default(path).expect("unable to open rocks db"))
}
+
+ pub fn open_read_only<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>
+ where
+ P: AsRef<std::path::Path>,
+ K: Serialize + DeserializeOwned + 'static,
+ V: Serialize + DeserializeOwned + 'static,
+ {
+ if !path.as_ref().exists() {
+ fs::create_dir_all(path.as_ref()).expect("faild to create dir");
+ }
+
+ let mut options = Options::default();
+ options.create_if_missing(true);
+
+ Box::new(DB::open_for_read_only(&options, path, false).expect("unable to open rocks db"))
+ }
}
impl<K, V> Kv<K, V> for rocksdb::DB
diff --git a/src/lib.rs b/src/lib.rs
index 96b0d7b7..cf8d4406 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -67,6 +67,7 @@ pub struct IndexingLocalConfig {
limit_warc_files: Option<usize>,
warc_source: WarcSource,
batch_size: Option<usize>,
+ webgraph_path: Option<String>,
centrality_store_path: String,
}
diff --git a/src/main.rs b/src/main.rs
index c9cc364c..c1962384 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -68,6 +68,7 @@ enum IndexingOptions {
Worker {
address: String,
centrality_store_path: String,
+ webgraph_path: Option<String>,
},
Local {
config_path: String,
@@ -100,8 +101,9 @@ fn main() -> Result<()> {
IndexingOptions::Worker {
address,
centrality_store_path,
+ webgraph_path,
} => {
- entrypoint::Indexer::run_worker(address, centrality_store_path)?;
+ entrypoint::Indexer::run_worker(address, centrality_store_path, webgraph_path)?;
}
IndexingOptions::Local { config_path } => {
let config = load_toml_config(&config_path);
diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rs
index 1f4b6ad1..e5420797 100644
--- a/src/webgraph/graph_store.rs
+++ b/src/webgraph/graph_store.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
- cell::RefCell, collections::HashMap, hash::Hash, marker::PhantomData, ops::Div, path::Path,
+ collections::HashMap, hash::Hash, marker::PhantomData, ops::Div, path::Path, sync::Mutex,
};
use lru::LruCache;
@@ -146,11 +146,11 @@ where
}
pub struct GraphStore<S> {
- pub(crate) adjacency: RefCell<Adjacency>,
- pub(crate) reversed_adjacency: RefCell<Adjacency>,
- pub(crate) node2id: RefCell<CachedTree<Node, NodeID>>,
- pub(crate) id2node: RefCell<BlockedCachedTree<NodeID, Node>>,
- pub(crate) meta: RefCell<CachedTree<String, u64>>,
+ pub(crate) adjacency: Mutex<Adjacency>,
+ pub(crate) reversed_adjacency: Mutex<Adjacency>,
+ pub(crate) node2id: Mutex<CachedTree<Node, NodeID>>,
+ pub(crate) id2node: Mutex<BlockedCachedTree<NodeID, Node>>,
+ pub(crate) meta: Mutex<CachedTree<String, u64>>,
pub(crate) store: PhantomData<S>,
}
@@ -164,9 +164,14 @@ impl<S: Store> GraphStore<S> {
S::open(path)
}
+ pub fn open_read_only<P: AsRef<Path>>(path: P) -> Self {
+ S::open_read_only(path)
+ }
+
fn next_id(&self) -> NodeID {
self.meta
- .borrow_mut()
+ .lock()
+ .unwrap()
.get(&"next_id".to_string())
.cloned()
.unwrap_or(0)
@@ -176,7 +181,8 @@ impl<S: Store> GraphStore<S> {
let current_id = self.next_id();
let next_id = current_id + 1;
self.meta
- .borrow_mut()
+ .lock()
+ .unwrap()
.insert("next_id".to_string(), next_id);
}
@@ -187,14 +193,14 @@ impl<S: Store> GraphStore<S> {
}
fn assign_id(&self, node: Node, id: NodeID) {
- self.node2id.borrow_mut().insert(node.clone(), id);
- self.id2node.borrow_mut().insert(id, &mut |block| {
+ self.node2id.lock().unwrap().insert(node.clone(), id);
+ self.id2node.lock().unwrap().insert(id, &mut |block| {
block.insert(id, node.clone());
});
}
fn id_or_assign(&self, node: Node) -> NodeID {
- if let Some(id) = self.node2id.borrow_mut().get(&node) {
+ if let Some(id) = self.node2id.lock().unwrap().get(&node) {
return *id;
}
let id = self.id_and_increment();
@@ -205,7 +211,8 @@ impl<S: Store> GraphStore<S> {
#[allow(unused)]
pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {
self.adjacency
- .borrow_mut()
+ .lock()
+ .unwrap()
.edges(node)
.into_iter()
.map(|edge| Edge {
@@ -218,7 +225,8 @@ impl<S: Store> GraphStore<S> {
pub fn ingoing_edges(&self, node: NodeID) -> Vec<Edge> {
self.reversed_adjacency
- .borrow_mut()
+ .lock()
+ .unwrap()
.edges(node)
.into_iter()
.map(|edge| Edge {
@@ -231,7 +239,8 @@ impl<S: Store> GraphStore<S> {
pub fn nodes(&self) -> impl Iterator<Item = NodeID> {
self.node2id
- .borrow_mut()
+ .lock()
+ .unwrap()
.iter()
.map(|(_, id)| id)
.collect::<Vec<u64>>()
@@ -243,27 +252,29 @@ impl<S: Store> GraphStore<S> {
let to_id = self.id_or_assign(to);
self.adjacency
- .borrow_mut()
+ .lock()
+ .unwrap()
.insert(from_id, to_id, label.clone());
self.reversed_adjacency
- .borrow_mut()
+ .lock()
+ .unwrap()
.insert(to_id, from_id, label);
}
pub fn node2id(&self, node: &Node) -> Option<NodeID> {
- self.node2id.borrow_mut().get(node).cloned()
+ self.node2id.lock().unwrap().get(node).cloned()
}
pub fn id2node(&self, id: &NodeID) -> Option<Node> {
- self.id2node.borrow_mut().get(id).cloned()
+ self.id2node.lock().unwrap().get(id).cloned()
}
pub fn flush(&self) {
- self.adjacency.borrow_mut().tree.inner.flush();
- self.reversed_adjacency.borrow_mut().tree.inner.flush();
- self.node2id.borrow_mut().flush();
- self.id2node.borrow_mut().inner.flush();
- self.meta.borrow_mut().flush();
+ self.adjacency.lock().unwrap().tree.inner.flush();
+ self.reversed_adjacency.lock().unwrap().tree.inner.flush();
+ self.node2id.lock().unwrap().flush();
+ self.id2node.lock().unwrap().inner.flush();
+ self.meta.lock().unwrap().flush();
}
pub fn edges(&self) -> EdgeIterator<'_> {
@@ -291,24 +302,55 @@ impl Store for RocksDbStore {
let meta = RocksDbStore::open(path.as_ref().join("meta"));
GraphStore {
- adjacency: RefCell::new(Adjacency {
+ adjacency: Mutex::new(Adjacency {
+ tree: BlockedCachedTree {
+ inner: CachedTree::new(adjacency, 10_000),
+ block_size: 1_024,
+ },
+ }),
+ reversed_adjacency: Mutex::new(Adjacency {
+ tree: BlockedCachedTree {
+ inner: CachedTree::new(reversed_adjacency, 10_000),
+ block_size: 1_024,
+ },
+ }),
+ node2id: Mutex::new(CachedTree::new(node2id, 100_000)),
+ id2node: Mutex::new(BlockedCachedTree {
+ inner: CachedTree::new(id2node, 100_000),
+ block_size: 1_024,
+ }),
+ meta: Mutex::new(CachedTree::new(meta, 1_000)),
+ store: Default::default(),
+ }
+ }
+
+ fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self> {
+ let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));
+ let reversed_adjacency =
+ RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));
+ let node2id = RocksDbStore::open_read_only(path.as_ref().join("node2id"));
+ let id2node = RocksDbStore::open_read_only(path.as_ref().join("id2node"));
+ let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));
+
+ GraphStore {
+ adjacency: Mutex::new(Adjacency {
tree: BlockedCachedTree {
inner: CachedTree::new(adjacency, 10_000),
block_size: 1_024,
},
}),
- reversed_adjacency: RefCell::new(Adjacency {
+ reversed_adjacency: Mutex::new(Adjacency {
tree: BlockedCachedTree {
inner: CachedTree::new(reversed_adjacency, 10_000),
block_size: 1_024,
},
}),
- node2id: RefCell::new(CachedTree::new(node2id, 100_000)),
- id2node: RefCell::new(BlockedCachedTree {
+ node2id: Mutex::new(CachedTree::new(node2id, 100_000)),
+ id2node: Mutex::new(BlockedCachedTree {
inner: CachedTree::new(id2node, 100_000),
block_size: 1_024,
}),
- meta: RefCell::new(CachedTree::new(meta, 1_000)),
+ meta: Mutex::new(CachedTree::new(meta, 1_000)),
store: Default::default(),
}
}
@@ -348,7 +390,7 @@ mod test {
let nodes: Vec<Node> = store
.nodes()
- .map(|id| store.id2node.borrow_mut().get(&id).unwrap().clone())
+ .map(|id| store.id2node.lock().unwrap().get(&id).unwrap().clone())
.collect();
assert_eq!(nodes, vec![a.clone(), b.clone(), c.clone()]);
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index 60069883..ce776e1a 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -17,9 +17,9 @@ mod graph_store;
use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use serde::{Deserialize, Serialize};
-use std::cell::RefCell;
use std::collections::{BinaryHeap, HashMap};
use std::path::Path;
+use std::sync::Mutex;
use std::{cmp, fs};
use tracing::info;
@@ -62,6 +62,14 @@ impl From<String> for Node {
}
}
+impl From<&Url> for Node {
+ fn from(url: &Url) -> Self {
+ Self {
+ name: url.raw().to_string(),
+ }
+ }
+}
+
impl From<&str> for Node {
fn from(name: &str) -> Self {
Self::from(name.to_string())
@@ -79,14 +87,15 @@ impl From<Url> for Node {
pub struct EdgeIterator<'a> {
current_block_idx: usize,
blocks: Vec<u64>,
- adjacency: &'a RefCell<Adjacency>,
+ adjacency: &'a Mutex<Adjacency>,
current_block: Option<Box<dyn Iterator<Item = Edge>>>,
}
impl<'a> EdgeIterator<'a> {
- fn new(adjacency: &'a RefCell<Adjacency>) -> EdgeIterator<'a> {
+ fn new(adjacency: &'a Mutex<Adjacency>) -> EdgeIterator<'a> {
let blocks: Vec<_> = adjacency
- .borrow()
+ .lock()
+ .unwrap()
.tree
.inner
.store
@@ -107,7 +116,8 @@ impl<'a> EdgeIterator<'a> {
let block_id = self.blocks[self.current_block_idx];
let block = self
.adjacency
- .borrow_mut()
+ .lock()
+ .unwrap()
.tree
.inner
.get(&block_id)
@@ -150,10 +160,18 @@ pub struct Edge {
label: String,
}
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FullEdge {
+ pub from: Node,
+ pub to: Node,
+ pub label: String,
+}
+
pub struct WebgraphBuilder {
path: Box<Path>,
full_graph_path: Option<Box<Path>>,
host_graph_path: Option<Box<Path>>,
+ read_only: bool,
}
impl WebgraphBuilder {
@@ -170,6 +188,7 @@ impl WebgraphBuilder {
path: path.as_ref().into(),
full_graph_path: None,
host_graph_path: None,
+ read_only: false,
}
}
@@ -183,11 +202,25 @@ impl WebgraphBuilder {
self
}
+ pub fn read_only(mut self, read_only: bool) -> Self {
+ self.read_only = read_only;
+
+ self
+ }
+
pub fn open(self) -> Webgraph {
- Webgraph {
- full_graph: self.full_graph_path.map(GraphStore::open),
- host_graph: self.host_graph_path.map(GraphStore::open),
- path: self.path.to_str().unwrap().to_string(),
+ if self.read_only {
+ Webgraph {
+ full_graph: self.full_graph_path.map(GraphStore::open_read_only),
+ host_graph: self.host_graph_path.map(GraphStore::open_read_only),
+ path: self.path.to_str().unwrap().to_string(),
+ }
+ } else {
+ Webgraph {
+ full_graph: self.full_graph_path.map(GraphStore::open),
+ host_graph: self.host_graph_path.map(GraphStore::open),
+ path: self.path.to_str().unwrap().to_string(),
+ }
}
}
}
@@ -197,6 +230,7 @@ where
Self: Sized,
{
fn open<P: AsRef<Path>>(path: P) -> GraphStore<Self>;
+ fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self>;
fn temporary() -> GraphStore<Self> {
Self::open(crate::gen_temp_path())
@@ -441,6 +475,26 @@ impl<S: Store> Webgraph<S> {
host_graph.flush();
}
}
+
+ pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
+ if let Some(graph) = &self.full_graph {
+ if let Some(node_id) = graph.node2id(&node) {
+ graph
+ .ingoing_edges(node_id)
+ .into_iter()
+ .map(|edge| FullEdge {
+ from: graph.id2node(&edge.from).unwrap(),
+ to: graph.id2node(&edge.to).unwrap(),
+ label: edge.label,
+ })
+ .collect()
+ } else {
+ Vec::new()
+ }
+ } else {
+ Vec::new()
+ }
+ }
}
impl From<FrozenWebgraph> for Webgraph {