Skip to content

web search done right

webgraph writer directly in memory. reduces number of disk writes, and it turns out it is actually feasible to keep it in memory

Mikkel Denker2c88387943af
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index a565b9a6..b767c297 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -57,9 +57,7 @@ impl SegmentWriter {
}
}
- pub fn finalize(mut self) -> Segment {
- self.flush();
-
+ pub fn finalize(self) -> Segment {
Segment {
adjacency: self.adjacency.finalize(),
reversed_adjacency: self.reversed_adjacency.finalize(),
@@ -68,14 +66,9 @@ impl SegmentWriter {
}
}
- pub fn flush(&mut self) {
- self.adjacency.flush();
- self.reversed_adjacency.flush();
- }
-
- pub fn insert(&mut self, edges: &[InnerEdge<String>]) {
- self.adjacency.put(edges.iter());
- self.reversed_adjacency.put(edges.iter());
+ pub fn insert(&mut self, edge: InnerEdge<String>) {
+ self.adjacency.put(edge.clone());
+ self.reversed_adjacency.put(edge);
}
}
@@ -210,7 +203,9 @@ mod test {
label: String::new(),
});
- writer.insert(&edges);
+ for edge in &edges {
+ writer.insert(edge.clone());
+ }
let segment = writer.finalize();
let mut out: Vec<_> = segment.outgoing_edges(&a.id);
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 6cb515ca..a267fa68 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -14,7 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{fs::File, io::Write, ops::Range, path::Path};
+use std::{
+ fs::File,
+ io::Write,
+ ops::Range,
+ path::{Path, PathBuf},
+};
use fst::Automaton;
use itertools::Itertools;
@@ -41,114 +46,68 @@ struct SerializedEdge {
label: Vec<u8>,
}
+struct SortableEdge<L: EdgeLabel> {
+ sort_node: NodeID,
+ edge: InnerEdge<L>,
+}
+
+impl<L: EdgeLabel> PartialOrd for SortableEdge<L> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl<L: EdgeLabel> Ord for SortableEdge<L> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.sort_node.cmp(&other.sort_node)
+ }
+}
+
+impl<L: EdgeLabel> PartialEq for SortableEdge<L> {
+ fn eq(&self, other: &Self) -> bool {
+ self.sort_node == other.sort_node
+ }
+}
+
+impl<L: EdgeLabel> Eq for SortableEdge<L> {}
+
pub struct EdgeStoreWriter {
reversed: bool,
- db: speedy_kv::Db<Vec<u8>, Vec<u8>>,
+ path: PathBuf,
+ edges: Vec<SortableEdge<String>>,
compression: Compression,
}
impl EdgeStoreWriter {
pub fn open<P: AsRef<Path>>(path: P, compression: Compression, reversed: bool) -> Self {
- let db = speedy_kv::Db::open_or_create(path.as_ref().join("writer")).unwrap();
-
Self {
- db,
+ edges: Vec::new(),
reversed,
+ path: path.as_ref().to_path_buf(),
compression,
}
}
- pub fn put<'a, L: EdgeLabel + 'a>(&'a mut self, edges: impl Iterator<Item = &'a InnerEdge<L>>) {
- for edge in edges {
- let value_bytes = L::to_bytes(&edge.label).unwrap();
-
- let value_bytes = bincode::encode_to_vec(
- &SerializedEdge {
- from_prefix: edge.from.prefix,
- to_prefix: edge.to.prefix,
- label: value_bytes.clone(),
- },
- bincode::config::standard(),
- )
- .unwrap();
-
- let key_bytes = if self.reversed {
- [
- edge.to.id.as_u64().to_le_bytes(),
- edge.from.id.as_u64().to_le_bytes(),
- ]
- .concat()
- } else {
- [
- edge.from.id.as_u64().to_le_bytes(),
- edge.to.id.as_u64().to_le_bytes(),
- ]
- .concat()
- };
-
- self.db.insert_raw(key_bytes, value_bytes);
- }
- }
-
- pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = InnerEdge<L>> + '_ + Send + Sync {
- self.db.iter_raw().map(|(key, val)| {
- let (from, to) = if self.reversed {
- (
- u64::from_le_bytes(
- key.as_bytes()[u64::BITS as usize / 8..].try_into().unwrap(),
- ),
- u64::from_le_bytes(
- key.as_bytes()[..u64::BITS as usize / 8].try_into().unwrap(),
- ),
- )
- } else {
- (
- u64::from_le_bytes(
- key.as_bytes()[..u64::BITS as usize / 8].try_into().unwrap(),
- ),
- u64::from_le_bytes(
- key.as_bytes()[u64::BITS as usize / 8..].try_into().unwrap(),
- ),
- )
- };
-
- let (val, _): (SerializedEdge, _) =
- bincode::decode_from_slice(val.as_bytes(), bincode::config::standard()).unwrap();
-
- InnerEdge {
- from: FullNodeID {
- prefix: val.from_prefix,
- id: NodeID::from(from),
- },
- to: FullNodeID {
- prefix: val.to_prefix,
- id: NodeID::from(to),
- },
- label: L::from_bytes(&val.label).unwrap(),
- }
- })
- }
+ pub fn put(&mut self, edge: InnerEdge<String>) {
+ let sort_node = if self.reversed {
+ edge.to.id
+ } else {
+ edge.from.id
+ };
- pub fn flush(&mut self) {
- self.db.commit().unwrap();
- self.db.merge_all_segments().unwrap();
+ self.edges.push(SortableEdge { sort_node, edge });
}
pub fn finalize(mut self) -> EdgeStore {
- self.flush();
+ self.edges.sort_unstable();
let s = EdgeStore::build(
- self.db.folder().parent().unwrap(),
+ self.path,
self.compression,
self.reversed,
- self.iter(),
+ self.edges.into_iter().map(|e| e.edge),
);
- // delete the writer db
- let p = self.db.folder().to_owned();
- drop(self.db);
- std::fs::remove_dir_all(p).unwrap();
-
s
}
}
@@ -550,7 +509,7 @@ mod tests {
label: "test".to_string(),
};
- kv.put([e.clone()].iter());
+ kv.put(e.clone());
let store = kv.finalize();
@@ -584,7 +543,7 @@ mod tests {
label: "test".to_string(),
};
- kv.put([e.clone()].iter());
+ kv.put(e.clone());
let store = kv.finalize();
diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rs
index 9ac066ab..2e619d14 100644
--- a/crates/core/src/webgraph/writer.rs
+++ b/crates/core/src/webgraph/writer.rs
@@ -26,7 +26,6 @@ use super::{
pub struct WebgraphWriter {
pub path: String,
segment: SegmentWriter,
- insert_batch: Vec<InnerEdge<String>>,
id2node: Id2NodeDb,
executor: Executor,
meta: Meta,
@@ -60,7 +59,6 @@ impl WebgraphWriter {
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
segment,
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
- insert_batch: Vec::new(),
executor,
meta,
compression,
@@ -95,16 +93,10 @@ impl WebgraphWriter {
label: label.chars().take(MAX_LABEL_LENGTH).collect(),
};
- self.insert_batch.push(edge);
+ self.segment.insert(edge);
}
pub fn commit(&mut self) {
- if !self.insert_batch.is_empty() {
- self.segment.insert(&self.insert_batch);
- self.segment.flush();
- self.insert_batch.clear();
- }
-
self.save_metadata();
self.id2node.flush();
}