Skip to content

web search done right

reduce webgraph storage by 33% this is primarily done by using truncating nodeids to u64 and increasing compression for writer. while the truncation might increase the number of collisions, it seems to be worth it in order to store larger page graphs

Mikkel Denker89d11512476a
diff --git a/core/src/api/user_count.rs b/core/src/api/user_count.rs
index 96cdc3f7..6635b2e6 100644
--- a/core/src/api/user_count.rs
+++ b/core/src/api/user_count.rs
@@ -97,7 +97,7 @@ impl<F: Frequency> InnerUserCount<F> {
let mut user_id = [0u8; 8];
user_id.copy_from_slice(&hash[..8]);
- self.counter.add(u64::from_be_bytes(user_id));
+ self.counter.add(u64::from_le_bytes(user_id));
self.metric.store(self.counter.size() as u64);
Ok(())
diff --git a/core/src/crawler/file_queue.rs b/core/src/crawler/file_queue.rs
index 92883dae..ab0315f9 100644
--- a/core/src/crawler/file_queue.rs
+++ b/core/src/crawler/file_queue.rs
@@ -118,12 +118,12 @@ impl FilePointer {
return 0;
}
- usize::from_be_bytes(buf)
+ usize::from_le_bytes(buf)
}
fn set(&mut self, pointer: usize) -> Result<()> {
self.file.seek(std::io::SeekFrom::Start(0))?;
- self.file.write_all(&pointer.to_be_bytes())?;
+ self.file.write_all(&pointer.to_le_bytes())?;
self.file.flush()?;
Ok(())
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 01e20295..1544d1ca 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -109,13 +109,14 @@ impl WebgraphWorker {
}
};
- for link in webpage
+ for mut link in webpage
.anchor_links()
.into_iter()
.filter(|link| matches!(link.destination.scheme(), "http" | "https"))
{
let source = link.source.clone();
let destination = link.destination.clone();
+ link.text = link.text.chars().take(128).collect();
trace!("inserting link {:?}", link);
let mut source = Node::from(source);
@@ -194,13 +195,14 @@ impl Webgraph {
worker.process_job(job);
}
- (worker.host_graph.finalize(), worker.page_graph.finalize())
+ worker
}));
}
let mut graphs = Vec::new();
for handler in handlers {
- graphs.push(handler.join().unwrap());
+ let worker = handler.join().unwrap();
+ graphs.push((worker.host_graph.finalize(), worker.page_graph.finalize()));
}
let (mut host_graph, mut page_graph) = graphs.pop().unwrap();
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 5e61f64b..ff47af9c 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -45,6 +45,7 @@ use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::SignalAggregator;
use crate::schema::{FastField, Field, TextField};
use crate::search_ctx::Ctx;
+use crate::snippet;
use crate::snippet::TextSnippet;
use crate::tokenizer::{
BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,
@@ -54,7 +55,6 @@ use crate::webpage::region::Region;
use crate::webpage::url_ext::UrlExt;
use crate::webpage::{schema_org, Webpage};
use crate::Result;
-use crate::{combine_u64s, snippet};
use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::collections::HashSet;
use std::fs;
@@ -381,22 +381,16 @@ impl InvertedIndex {
let searcher = self.reader.searcher();
let doc: TantivyDocument = searcher.doc(website.address.into())?;
- let field1 = self
+ let field = self
.schema()
- .get_field(Field::Fast(FastField::HostNodeID1).name())
- .unwrap();
- let field2 = self
- .schema()
- .get_field(Field::Fast(FastField::HostNodeID2).name())
+ .get_field(Field::Fast(FastField::HostNodeID).name())
.unwrap();
- let id1 = doc.get_first(field1).unwrap().as_u64().unwrap();
- let id2 = doc.get_first(field2).unwrap().as_u64().unwrap();
+ let id = doc.get_first(field).unwrap().as_u64().unwrap();
- if id1 == u64::MAX && id2 == u64::MAX {
+ if id == u64::MAX {
Ok(None)
} else {
- let id = combine_u64s([id1, id2]);
Ok(Some(id.into()))
}
}
diff --git a/core/src/prehashed.rs b/core/src/prehashed.rs
index b4b133e4..b1405cf8 100644
--- a/core/src/prehashed.rs
+++ b/core/src/prehashed.rs
@@ -69,5 +69,5 @@ impl PartialEq for Prehashed {
pub fn hash<T: AsRef<[u8]>>(val: T) -> Prehashed {
let digest = md5::compute(val);
- Prehashed(u128::from_be_bytes(*digest))
+ Prehashed(u128::from_le_bytes(*digest))
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index ce05bd0e..3d0646d0 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -197,7 +197,7 @@ impl VecMap {
for (node_id, inbound) in adjacency {
map.insert(
node_id,
- bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.bit_64()).collect()),
+ bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.as_u64()).collect()),
);
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 791d7f7b..8985291a 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -17,7 +17,7 @@
use crate::fastfield_reader::FieldValue;
use crate::query::optic::AsSearchableRule;
use crate::query::Query;
-use crate::{combine_u64s, Result};
+use crate::Result;
use crate::{
enum_map::EnumMap,
fastfield_reader,
@@ -301,26 +301,17 @@ impl Signal {
fn host_id(&self, aggregator: &SignalAggregator, doc: DocId) -> Option<NodeID> {
aggregator.segment_reader.as_ref().and_then(|reader| {
- let node_id1: Option<u64> = reader
+ let node_id: Option<u64> = reader
.fastfield_reader
- .get_field_reader(&FastField::HostNodeID1)
+ .get_field_reader(&FastField::HostNodeID)
.get(&doc)
.into();
- let node_id1 = node_id1.unwrap();
+ let node_id = node_id.unwrap();
- let node_id2: Option<u64> = reader
- .fastfield_reader
- .get_field_reader(&FastField::HostNodeID2)
- .get(&doc)
- .into();
- let node_id2 = node_id2.unwrap();
-
- if node_id1 == u64::MAX && node_id2 == u64::MAX {
+ if node_id == u64::MAX {
None
} else {
- let id = combine_u64s([node_id1, node_id2]);
-
- Some(NodeID::from(id))
+ Some(node_id.into())
}
})
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index e9853ee8..99cf3ffd 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -226,8 +226,7 @@ pub enum FastField {
UrlWithoutTldHash1,
UrlWithoutTldHash2,
PreComputedScore,
- HostNodeID1,
- HostNodeID2,
+ HostNodeID,
SimHash,
NumFlattenedSchemaTokens,
NumPathAndQuerySlashes,
@@ -268,8 +267,7 @@ impl FastField {
FastField::UrlWithoutTldHash1 => "url_without_tld_hash1",
FastField::UrlWithoutTldHash2 => "url_without_tld_hash2",
FastField::PreComputedScore => "pre_computed_score",
- FastField::HostNodeID1 => "host_node_id1",
- FastField::HostNodeID2 => "host_node_id2",
+ FastField::HostNodeID => "host_node_id",
FastField::SimHash => "sim_hash",
FastField::NumPathAndQuerySlashes => "num_path_and_query_slashes",
FastField::NumPathAndQueryDigits => "num_path_and_query_digits",
@@ -292,7 +290,7 @@ pub enum Field {
Text(TextField),
}
-static ALL_FIELDS: [Field; 65] = [
+static ALL_FIELDS: [Field; 64] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -352,8 +350,7 @@ static ALL_FIELDS: [Field; 65] = [
Field::Fast(FastField::UrlWithoutTldHash1),
Field::Fast(FastField::UrlWithoutTldHash2),
Field::Fast(FastField::PreComputedScore),
- Field::Fast(FastField::HostNodeID1),
- Field::Fast(FastField::HostNodeID2),
+ Field::Fast(FastField::HostNodeID),
Field::Fast(FastField::SimHash),
Field::Fast(FastField::NumPathAndQuerySlashes),
Field::Fast(FastField::NumPathAndQueryDigits),
@@ -571,13 +568,7 @@ impl Field {
.set_indexed()
.set_stored(),
),
- Field::Fast(FastField::HostNodeID1) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::HostNodeID2) => IndexingOption::Integer(
+ Field::Fast(FastField::HostNodeID) => IndexingOption::Integer(
NumericOptions::default()
.set_fast()
.set_indexed()
@@ -728,8 +719,7 @@ impl FastField {
FastField::UrlWithoutTldHash1 => DataType::U64,
FastField::UrlWithoutTldHash2 => DataType::U64,
FastField::PreComputedScore => DataType::U64,
- FastField::HostNodeID1 => DataType::U64,
- FastField::HostNodeID2 => DataType::U64,
+ FastField::HostNodeID => DataType::U64,
FastField::SimHash => DataType::U64,
FastField::NumPathAndQuerySlashes => DataType::U64,
FastField::NumPathAndQueryDigits => DataType::U64,
diff --git a/core/src/webgraph/centrality/derived_harmonic.rs b/core/src/webgraph/centrality/derived_harmonic.rs
index 94fedb4f..9523a3f8 100644
--- a/core/src/webgraph/centrality/derived_harmonic.rs
+++ b/core/src/webgraph/centrality/derived_harmonic.rs
@@ -44,7 +44,7 @@ impl BloomMap {
}
fn insert(&self, item: &NodeID) {
- let h = item.bit_64();
+ let h = item.as_u64();
self.map[(h as usize) % self.map.len()]
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -98,7 +98,7 @@ impl DerivedCentrality {
page_graph.node_ids().for_each(|(node, id)| {
pb.inc(1);
- if has_outgoing.contains(&id.bit_64()) {
+ if has_outgoing.contains(&id.as_u64()) {
let host_node = node.clone().into_host().id();
if let Some(harmonic) = host_harmonic.get(&host_node) {
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index 3b1d55f6..8218b21b 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -41,7 +41,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
for node in graph.nodes() {
let mut counter = HyperLogLog::default();
- counter.add(node.bit_64());
+ counter.add(node.as_u64());
counters.insert(node, counter);
centralities.insert(node, KahanSum::default());
@@ -52,7 +52,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);
for node in graph.nodes() {
- changed_nodes.insert(node.bit_64());
+ changed_nodes.insert(node.as_u64());
}
info!("Found {} nodes in the graph", num_nodes);
@@ -92,7 +92,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.bit_64());
+ new_changed_nodes.insert(edge.to.as_u64());
new_exact_changed_nodes.insert(edge.to);
@@ -106,7 +106,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
} else {
exact_changed_nodes = BTreeSet::default();
graph.edges().for_each(|edge| {
- if changed_nodes.contains(&edge.from.bit_64()) {
+ if changed_nodes.contains(&edge.from.as_u64()) {
if let (Some(counter_to), Some(counter_from)) =
(new_counters.get_mut(&edge.to), counters.get(&edge.from))
{
@@ -117,7 +117,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.bit_64());
+ new_changed_nodes.insert(edge.to.as_u64());
if exact_counting {
exact_changed_nodes.insert(edge.to);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 84e296af..fdeb80fd 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -39,20 +39,22 @@ pub const MAX_LABEL_LENGTH: usize = 1024;
#[derive(
Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,
)]
-pub struct NodeID(u128);
+pub struct NodeID(u64);
impl NodeID {
- pub fn bit_64(self) -> u64 {
- self.0 as u64
- }
-
- pub fn bit_128(self) -> u128 {
+ pub fn as_u64(self) -> u64 {
self.0
}
}
impl From<u128> for NodeID {
fn from(val: u128) -> Self {
+ NodeID(val as u64)
+ }
+}
+
+impl From<u64> for NodeID {
+ fn from(val: u64) -> Self {
NodeID(val)
}
}
@@ -73,7 +75,7 @@ impl From<Node> for FullNodeID {
}
impl intmap::Key for NodeID {
- const BIG_PRIME: Self = NodeID(335579573203413586826293107669396558523);
+ const BIG_PRIME: Self = NodeID(11400714819323198549);
fn wrapping_mul(self, rhs: Self) -> Self {
NodeID(self.0.wrapping_mul(rhs.0))
@@ -84,7 +86,7 @@ impl intmap::Key for NodeID {
}
fn modulus_usize(self, rhs: usize) -> usize {
- (self.0 % (rhs as u128)) as usize
+ (self.0 % (rhs as u64)) as usize
}
}
@@ -195,7 +197,7 @@ impl Node {
pub fn id(&self) -> NodeID {
let digest = md5::compute(self.name.as_bytes());
- NodeID(u128::from_be_bytes(*digest))
+ u128::from_le_bytes(*digest).into()
}
}
@@ -453,10 +455,25 @@ impl Id2NodeDb {
opts.create_if_missing(true);
opts.optimize_for_point_lookup(512);
+ opts.set_allow_mmap_reads(true);
+ opts.set_allow_mmap_writes(true);
+ opts.set_write_buffer_size(128 * 1024 * 1024); // 128 MB
+ opts.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ opts.set_target_file_size_multiplier(10);
+
let mut block_opts = rocksdb::BlockBasedOptions::default();
block_opts.set_ribbon_filter(5.0);
+ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
+ opts.set_level_compaction_dynamic_level_bytes(true);
+ opts.set_bytes_per_sync(1048576);
+ block_opts.set_block_size(16 * 1024);
+ block_opts.set_format_version(5);
+ block_opts.set_cache_index_and_filter_blocks(true);
+ block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);
+
opts.set_block_based_table_factory(&block_opts);
+ opts.set_compression_type(rocksdb::DBCompressionType::Zstd);
let db = rocksdb::DB::open(&opts, path).unwrap();
@@ -469,7 +486,7 @@ impl Id2NodeDb {
self.db
.put_opt(
- id.bit_128().to_be_bytes(),
+ id.as_u64().to_le_bytes(),
bincode::serialize(node).unwrap(),
&opts,
)
@@ -478,7 +495,7 @@ impl Id2NodeDb {
fn get(&self, id: &NodeID) -> Option<Node> {
self.db
- .get(id.bit_128().to_be_bytes())
+ .get(id.as_u64().to_le_bytes())
.unwrap()
.map(|bytes| bincode::deserialize(&bytes).unwrap())
}
@@ -488,7 +505,7 @@ impl Id2NodeDb {
.iterator(rocksdb::IteratorMode::Start)
.filter_map(|r| {
let (key, _) = r.ok()?;
- Some(NodeID(u128::from_be_bytes((*key).try_into().unwrap())))
+ Some(NodeID(u64::from_le_bytes((*key).try_into().unwrap())))
})
}
@@ -499,7 +516,7 @@ impl Id2NodeDb {
let (key, value) = r.ok()?;
Some((
- NodeID(u128::from_be_bytes((*key).try_into().unwrap())),
+ NodeID(u64::from_le_bytes((*key).try_into().unwrap())),
bincode::deserialize(&value).unwrap(),
))
})
@@ -510,7 +527,7 @@ impl Id2NodeDb {
for (id, node) in iter {
batch.put(
- id.bit_128().to_be_bytes(),
+ id.as_u64().to_le_bytes(),
bincode::serialize(&node).unwrap(),
);
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index b3081d88..b23f6b78 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -25,8 +25,8 @@ const ADJACENCY_STORE: &str = "adjacency";
const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";
pub struct SegmentWriter {
- full_adjacency: EdgeStoreWriter,
- full_reversed_adjacency: EdgeStoreWriter,
+ adjacency: EdgeStoreWriter,
+ reversed_adjacency: EdgeStoreWriter,
id: String,
folder_path: String,
}
@@ -34,12 +34,12 @@ pub struct SegmentWriter {
impl SegmentWriter {
pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {
SegmentWriter {
- full_adjacency: EdgeStoreWriter::open(
+ adjacency: EdgeStoreWriter::open(
folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
compression,
false,
),
- full_reversed_adjacency: EdgeStoreWriter::open(
+ reversed_adjacency: EdgeStoreWriter::open(
folder_path
.as_ref()
.join(&id)
@@ -61,27 +61,27 @@ impl SegmentWriter {
self.flush();
Segment {
- full_adjacency: self.full_adjacency.finalize(),
- full_reversed_adjacency: self.full_reversed_adjacency.finalize(),
+ adjacency: self.adjacency.finalize(),
+ reversed_adjacency: self.reversed_adjacency.finalize(),
folder_path: self.folder_path,
id: self.id,
}
}
pub fn flush(&mut self) {
- self.full_adjacency.flush();
- self.full_reversed_adjacency.flush();
+ self.adjacency.flush();
+ self.reversed_adjacency.flush();
}
pub fn insert(&mut self, edges: &[InnerEdge<String>]) {
- self.full_adjacency.put(edges.iter());
- self.full_reversed_adjacency.put(edges.iter());
+ self.adjacency.put(edges.iter());
+ self.reversed_adjacency.put(edges.iter());
}
}
pub struct Segment {
- full_adjacency: EdgeStore,
- full_reversed_adjacency: EdgeStore,
+ adjacency: EdgeStore,
+ reversed_adjacency: EdgeStore,
id: String,
folder_path: String,
}
@@ -89,12 +89,12 @@ pub struct Segment {
impl Segment {
pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {
Segment {
- full_adjacency: EdgeStore::open(
+ adjacency: EdgeStore::open(
folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
false,
compression,
),
- full_reversed_adjacency: EdgeStore::open(
+ reversed_adjacency: EdgeStore::open(
folder_path
.as_ref()
.join(&id)
@@ -113,23 +113,23 @@ impl Segment {
}
pub fn outgoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {
- self.full_adjacency.get_with_label(node)
+ self.adjacency.get_with_label(node)
}
pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
- self.full_adjacency.get_without_label(node)
+ self.adjacency.get_without_label(node)
}
pub fn ingoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {
- self.full_reversed_adjacency.get_with_label(node)
+ self.reversed_adjacency.get_with_label(node)
}
pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
- self.full_reversed_adjacency.get_without_label(node)
+ self.reversed_adjacency.get_without_label(node)
}
pub fn ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {
- self.full_reversed_adjacency
+ self.reversed_adjacency
.nodes_by_prefix(host_node)
.into_iter()
.flat_map(|node| self.ingoing_edges(&node))
@@ -137,7 +137,7 @@ impl Segment {
}
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
- self.full_reversed_adjacency.nodes_by_prefix(host_node)
+ self.reversed_adjacency.nodes_by_prefix(host_node)
}
pub fn id(&self) -> String {
@@ -149,7 +149,7 @@ impl Segment {
}
pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
- self.full_adjacency.iter_without_label()
+ self.adjacency.iter_without_label()
}
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index cf9e4ed4..abf65f35 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -23,9 +23,9 @@ use rocksdb::BlockBasedOptions;
use super::{Compression, Edge, EdgeLabel, FullNodeID, InnerEdge, NodeID};
#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
-pub struct SerializedEdge {
- from: FullNodeID,
- to: FullNodeID,
+struct SerializedEdge {
+ from_prefix: NodeID,
+ to_prefix: NodeID,
label: Vec<u8>,
}
@@ -49,7 +49,7 @@ impl EdgeStoreWriter {
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB
- options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ options.set_target_file_size_base(128 * 1024 * 1024); // 128 MB
options.set_target_file_size_multiplier(10);
options.set_max_write_buffer_number(4);
@@ -67,7 +67,7 @@ impl EdgeStoreWriter {
block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
options.set_block_based_table_factory(&block_options);
- options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+ options.set_compression_type(rocksdb::DBCompressionType::Zstd);
let db = rocksdb::DB::open(&options, path.as_ref().join("writer")).unwrap();
@@ -88,22 +88,22 @@ impl EdgeStoreWriter {
let value_bytes = L::to_bytes(&edge.label).unwrap();
let value_bytes = bincode::serialize(&SerializedEdge {
- from: edge.from.clone(),
- to: edge.to.clone(),
+ from_prefix: edge.from.prefix,
+ to_prefix: edge.to.prefix,
label: value_bytes.clone(),
})
.unwrap();
let key_bytes = if self.reversed {
[
- edge.to.id.bit_128().to_be_bytes(),
- edge.from.id.bit_128().to_be_bytes(),
+ edge.to.id.as_u64().to_le_bytes(),
+ edge.from.id.as_u64().to_le_bytes(),
]
.concat()
} else {
[
- edge.from.id.bit_128().to_be_bytes(),
- edge.to.id.bit_128().to_be_bytes(),
+ edge.from.id.as_u64().to_le_bytes(),
+ edge.to.id.as_u64().to_le_bytes(),
]
.concat()
};
@@ -124,16 +124,34 @@ impl EdgeStoreWriter {
self.db
.iterator_opt(rocksdb::IteratorMode::Start, read_opts)
- .map(|res| {
- let (_, val) = res.unwrap();
+ .filter_map(|res| {
+ let (key, val) = res.ok()?;
+
+ let (from, to) = if self.reversed {
+ (
+ u64::from_le_bytes(key[u64::BITS as usize / 8..].try_into().unwrap()),
+ u64::from_le_bytes(key[..u64::BITS as usize / 8].try_into().unwrap()),
+ )
+ } else {
+ (
+ u64::from_le_bytes(key[..u64::BITS as usize / 8].try_into().unwrap()),
+ u64::from_le_bytes(key[u64::BITS as usize / 8..].try_into().unwrap()),
+ )
+ };
let val: SerializedEdge = bincode::deserialize(&val).unwrap();
- InnerEdge {
- from: val.from,
- to: val.to,
+ Some(InnerEdge {
+ from: FullNodeID {
+ prefix: val.from_prefix,
+ id: NodeID(from),
+ },
+ to: FullNodeID {
+ prefix: val.to_prefix,
+ id: NodeID(to),
+ },
label: L::from_bytes(&val.label).unwrap(),
- }
+ })
})
}
@@ -167,9 +185,15 @@ impl PrefixDb {
let mut options = rocksdb::Options::default();
options.create_if_missing(true);
- options.set_max_background_jobs(8);
- options.increase_parallelism(8);
- options.set_max_subcompactions(8);
+ options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB
+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ options.set_target_file_size_multiplier(10);
+
+ options.set_max_write_buffer_number(4);
+ options.set_min_write_buffer_number_to_merge(2);
+
+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ options.set_target_file_size_multiplier(10);
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
@@ -199,8 +223,8 @@ impl PrefixDb {
opts.disable_wal(true);
let key = [
- node.prefix.bit_128().to_be_bytes(),
- node.id.bit_128().to_be_bytes(),
+ node.prefix.as_u64().to_le_bytes(),
+ node.id.as_u64().to_le_bytes(),
]
.concat();
let value = [];
@@ -210,8 +234,8 @@ impl PrefixDb {
fn get(&self, prefix: &NodeID) -> Vec<NodeID> {
let start = [
- prefix.bit_128().to_be_bytes().to_vec(),
- [0].repeat(u128::BITS as usize / 8).to_vec(),
+ prefix.as_u64().to_le_bytes().to_vec(),
+ [0].repeat(u64::BITS as usize / 8).to_vec(),
]
.concat();
@@ -225,10 +249,10 @@ impl PrefixDb {
for item in iter {
let (key, _) = item.unwrap();
- let p = u128::from_be_bytes(key[..(u128::BITS as usize / 8)].try_into().unwrap());
- let node = u128::from_be_bytes(key[(u128::BITS as usize / 8)..].try_into().unwrap());
+ let p = u64::from_le_bytes(key[..(u64::BITS as usize / 8)].try_into().unwrap());
+ let node = u64::from_le_bytes(key[(u64::BITS as usize / 8)..].try_into().unwrap());
- if p != prefix.bit_128() {
+ if p != prefix.as_u64() {
break;
}
@@ -269,6 +293,9 @@ impl EdgeStore {
options.increase_parallelism(8);
options.set_max_subcompactions(8);
+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ options.set_target_file_size_multiplier(10);
+
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
@@ -290,7 +317,7 @@ impl EdgeStore {
block_options.set_ribbon_filter(10.0);
options.set_block_based_table_factory(&block_options);
- options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+ options.set_compression_type(rocksdb::DBCompressionType::Zstd);
options.optimize_for_point_lookup(512);
let ranges = match rocksdb::DB::open_cf_with_opts(
@@ -358,7 +385,7 @@ impl EdgeStore {
};
self.prefixes.insert(&node);
- let node_bytes = node.id.bit_128().to_be_bytes();
+ let node_bytes = node.id.as_u64().to_le_bytes();
let node_cf = self.ranges.cf_handle("nodes").unwrap();
let label_cf = self.ranges.cf_handle("labels").unwrap();
@@ -432,6 +459,9 @@ impl EdgeStore {
if let Some(last_node) = last_node {
if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node)
{
+ // batch
+ // .sort_by_key(|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id });
+ // batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
s.put(&batch);
batch.clear();
}
@@ -442,6 +472,8 @@ impl EdgeStore {
}
if !batch.is_empty() {
+ // batch.sort_by_key(|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id });
+ // batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
s.put(&batch);
}
@@ -472,7 +504,7 @@ impl EdgeStore {
}
pub fn get_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {
- let node_bytes = node.bit_128().to_be_bytes();
+ let node_bytes = node.as_u64().to_le_bytes();
let node_cf = self.ranges.cf_handle("nodes").unwrap();
let edge_cf = self.ranges.cf_handle("labels").unwrap();
@@ -518,7 +550,7 @@ impl EdgeStore {
}
pub fn get_without_label(&self, node: &NodeID) -> Vec<Edge<()>> {
- let node_bytes = node.bit_128().to_be_bytes();
+ let node_bytes = node.as_u64().to_le_bytes();
let node_cf = self.ranges.cf_handle("nodes").unwrap();
@@ -565,7 +597,7 @@ impl EdgeStore {
.flat_map(move |res| {
let (key, val) = res.unwrap();
- let node = u128::from_be_bytes((*key).try_into().unwrap());
+ let node = u64::from_le_bytes((*key).try_into().unwrap());
let node = NodeID(node);
let node_range = bincode::deserialize::<Range<usize>>(&val).unwrap();
diff --git a/core/src/webpage/html/into_tantivy.rs b/core/src/webpage/html/into_tantivy.rs
index 06f58910..1c32cf4c 100644
--- a/core/src/webpage/html/into_tantivy.rs
+++ b/core/src/webpage/html/into_tantivy.rs
@@ -526,8 +526,7 @@ impl Html {
| Field::Fast(FastField::FetchTimeMs)
| Field::Fast(FastField::PreComputedScore)
| Field::Fast(FastField::Region)
- | Field::Fast(FastField::HostNodeID1)
- | Field::Fast(FastField::HostNodeID2)
+ | Field::Fast(FastField::HostNodeID)
| Field::Text(TextField::DmozDescription) => {}
}
}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 7602719c..1f4926a2 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -16,7 +16,6 @@
use crate::{
schema::{FastField, TextField},
- split_u128,
webgraph::NodeID,
Result,
};
@@ -199,31 +198,18 @@ impl Webpage {
match &self.node_id {
Some(node_id) => {
- let [node_id1, node_id2] = split_u128(node_id.bit_128());
doc.add_u64(
schema
- .get_field(Field::Fast(FastField::HostNodeID1).name())
- .expect("Failed to get node_id field 1"),
- node_id1,
- );
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::HostNodeID2).name())
- .expect("Failed to get node_id field 2"),
- node_id2,
+ .get_field(Field::Fast(FastField::HostNodeID).name())
+ .expect("Failed to get node_id field"),
+ node_id.as_u64(),
);
}
None => {
doc.add_u64(
schema
- .get_field(Field::Fast(FastField::HostNodeID1).name())
- .expect("Failed to get node_id field 1"),
- u64::MAX,
- );
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::HostNodeID2).name())
- .expect("Failed to get node_id field 2"),
+ .get_field(Field::Fast(FastField::HostNodeID).name())
+ .expect("Failed to get node_id field"),
u64::MAX,
);
}