web search done right
store node ids as big endian in webgraph so sort is correct during merge
Mikkel Denker308388262fd3
diff --git a/Cargo.lock b/Cargo.lockindex 60b6d1de..36fe23a8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -927,6 +927,13 @@ version = "1.0.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"+[[package]]+name = "common"+version = "0.1.0"+dependencies = [+ "bincode",+]+[[package]]name = "console"version = "0.15.8"@@ -1603,6 +1610,7 @@ version = "0.1.0"dependencies = ["anyhow","bincode",+ "common","memmap2","ownedbytes",]@@ -4906,6 +4914,7 @@ dependencies = ["anyhow","bincode","bloom",+ "common","file_store","fst","memmap2",@@ -4970,6 +4979,7 @@ dependencies = ["chitchat","chrono","clap",+ "common","criterion","crossbeam-channel","csv",diff --git a/Cargo.toml b/Cargo.tomlindex 423bb260..c43146ad 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -14,6 +14,7 @@ members = ["crates/robotstxt","crates/tantivy","crates/leechy",+ "crates/common","fuzz",]resolver = "2"@@ -60,6 +61,7 @@ chitchat = "0.5.0"chrono = {version = "0.4.23", features = ["serde"]}clap = {version = "4.4.6", features = ["derive"]}cmake = "0.1"+common = {path = "crates/common"}crc32fast = "1.3.2"criterion = "0.5.1"crossbeam-channel = "0.5.6"diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rsindex 81dbc86d..2927a674 100644--- a/crates/bloom/src/lib.rs+++ b/crates/bloom/src/lib.rs@@ -210,22 +210,22 @@ mod tests {#[test]fn test_bloom_filter_bytes() {let mut bf = BytesBloomFilter::new(100, 0.01);- bf.insert(&1u64.to_be_bytes());- bf.insert(&2u64.to_be_bytes());- bf.insert(&3u64.to_be_bytes());- bf.insert(&4u64.to_be_bytes());- bf.insert(&5u64.to_be_bytes());-- assert!(bf.contains(&1u64.to_be_bytes()));- assert!(bf.contains(&2u64.to_be_bytes()));- assert!(bf.contains(&3u64.to_be_bytes()));- assert!(bf.contains(&4u64.to_be_bytes()));- assert!(bf.contains(&5u64.to_be_bytes()));- assert!(!bf.contains(&6u64.to_be_bytes()));- assert!(!bf.contains(&7u64.to_be_bytes()));- assert!(!bf.contains(&8u64.to_be_bytes()));- assert!(!bf.contains(&9u64.to_be_bytes()));- assert!(!bf.contains(&10u64.to_be_bytes()));+ bf.insert(&1u64.to_le_bytes());+ bf.insert(&2u64.to_le_bytes());+ bf.insert(&3u64.to_le_bytes());+ bf.insert(&4u64.to_le_bytes());+ bf.insert(&5u64.to_le_bytes());++ assert!(bf.contains(&1u64.to_le_bytes()));+ assert!(bf.contains(&2u64.to_le_bytes()));+ assert!(bf.contains(&3u64.to_le_bytes()));+ assert!(bf.contains(&4u64.to_le_bytes()));+ assert!(bf.contains(&5u64.to_le_bytes()));+ assert!(!bf.contains(&6u64.to_le_bytes()));+ assert!(!bf.contains(&7u64.to_le_bytes()));+ assert!(!bf.contains(&8u64.to_le_bytes()));+ assert!(!bf.contains(&9u64.to_le_bytes()));+ assert!(!bf.contains(&10u64.to_le_bytes()));}#[test]diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.tomlnew file mode 100644index 00000000..392c244d--- /dev/null+++ b/crates/common/Cargo.toml@@ -0,0 +1,10 @@+[package]+edition = "2021"+license = "AGPL-3.0"+name = "common"+version = "0.1.0"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html++[dependencies]+bincode.workspace = truediff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rsnew file mode 100644index 00000000..be3248f5--- /dev/null+++ b/crates/common/src/lib.rs@@ -0,0 +1,3 @@+pub fn bincode_config() -> bincode::config::Configuration {+ bincode::config::standard()+}diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex db9d2289..91bc7fc7 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -40,6 +40,7 @@ chardetng.workspace = truechitchat.workspace = truechrono.workspace = trueclap.workspace = true+common.workspace = truecrossbeam-channel.workspace = truecsv.workspace = truedashmap.workspace = truediff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rsindex 6ed973e1..6457e8e7 100644--- a/crates/core/src/ampc/dht/store.rs+++ b/crates/core/src/ampc/dht/store.rs@@ -274,7 +274,7 @@ impl RaftSnapshotBuilder<TypeConfig> for Arc<StateMachineStore> {{// Serialize the data of the state machine.let state_machine = self.state_machine.read().await;- let encoded = bincode::encode_to_vec(&*state_machine, bincode::config::standard())+ let encoded = bincode::encode_to_vec(&*state_machine, common::bincode_config()).map_err(|e| StorageIOError::read_state_machine(&e))?;data = encoded;@@ -433,10 +433,9 @@ impl RaftStateMachine<TypeConfig> for Arc<StateMachineStore> {// Update the state machine.{let (data, _): (Db, _) =- bincode::decode_from_slice(&new_snapshot.data, bincode::config::standard())- .map_err(|e| {- StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e)- })?;+ bincode::decode_from_slice(&new_snapshot.data, common::bincode_config()).map_err(+ |e| StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e),+ )?;let mut state_machine = self.state_machine.write().await;state_machine.db = data;diff --git a/crates/core/src/api/user_count.rs b/crates/core/src/api/user_count.rsindex 846bed4f..678c982d 100644--- a/crates/core/src/api/user_count.rs+++ b/crates/core/src/api/user_count.rs@@ -83,7 +83,7 @@ impl<F: Frequency> InnerUserCount<F> {// It is first hashed with a salt for good measure, and then we only use// the id for a probabilistic count using hyperloglog.self.maybe_reset();- let bytes = bincode::encode_to_vec(user_id, bincode::config::standard())?;+ let bytes = bincode::encode_to_vec(user_id, common::bincode_config())?;let mut hash = [0u8; digest::SHA512_OUTPUT_LEN];pbkdf2::derive(diff --git a/crates/core/src/crawler/file_queue.rs b/crates/core/src/crawler/file_queue.rsindex c3159883..c6a311c2 100644--- a/crates/core/src/crawler/file_queue.rs+++ b/crates/core/src/crawler/file_queue.rs@@ -61,7 +61,7 @@ where}pub fn push(&mut self, item: T) -> Result<()> {- let body = bincode::encode_to_vec(&item, bincode::config::standard())?;+ let body = bincode::encode_to_vec(&item, common::bincode_config())?;let header = Header {body_size: body.len(),};@@ -177,7 +177,7 @@ wherelet body =&self.file[cur_pointer + header_size..cur_pointer + header_size + header.body_size];- let (item, _) = bincode::decode_from_slice(body, bincode::config::standard())?;+ let (item, _) = bincode::decode_from_slice(body, common::bincode_config())?;self.pointer.set(cur_pointer + header_size + header.body_size)?;diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rsindex 84a686ec..713483c3 100644--- a/crates/core/src/distributed/sonic/mod.rs+++ b/crates/core/src/distributed/sonic/mod.rs@@ -121,7 +121,7 @@ whereasync fn send_without_timeout(&mut self, request: &Req) -> Result<Res> {self.awaiting_res = true;- let bytes = bincode::encode_to_vec(request, bincode::config::standard()).unwrap();+ let bytes = bincode::encode_to_vec(request, common::bincode_config()).unwrap();let header = Header {body_size: bytes.len(),@@ -147,7 +147,7 @@ whereself.stream.flush().await?;tracing::debug!("deserializing {:?}", std::any::type_name::<(Req, Res)>());- let (res, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();+ let (res, _) = bincode::decode_from_slice(&buf, common::bincode_config()).unwrap();self.awaiting_res = false;@@ -249,7 +249,7 @@ whereself.stream.read_exact(&mut buf).await?;- let (body, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();+ let (body, _) = bincode::decode_from_slice(&buf, common::bincode_config()).unwrap();Ok(Request {conn: self,@@ -268,7 +268,7 @@ whereRes: bincode::Encode,{async fn respond_without_timeout(self, response: Res) -> Result<()> {- let bytes = bincode::encode_to_vec(&response, bincode::config::standard()).unwrap();+ let bytes = bincode::encode_to_vec(&response, common::bincode_config()).unwrap();let header = Header {body_size: bytes.len(),};diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rsindex 88187572..2b090830 100644--- a/crates/core/src/entity_index/mod.rs+++ b/crates/core/src/entity_index/mod.rs@@ -96,11 +96,11 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> Tantiv);doc.add_bytes(schema.get_field("info").unwrap(),- &bincode::encode_to_vec(&entity.info, bincode::config::standard()).unwrap(),+ &bincode::encode_to_vec(&entity.info, common::bincode_config()).unwrap(),);doc.add_bytes(schema.get_field("links").unwrap(),- &bincode::encode_to_vec(&entity.page_abstract.links, bincode::config::standard()).unwrap(),+ &bincode::encode_to_vec(&entity.page_abstract.links, common::bincode_config()).unwrap(),);let has_image = if entity.image.is_some() {"true"@@ -346,7 +346,7 @@ impl EntityIndex {let info = if decode_info {let (info, _) = bincode::decode_from_slice(doc.get_first(info).and_then(|val| val.as_bytes()).unwrap(),- bincode::config::standard(),+ common::bincode_config(),).unwrap();@@ -383,7 +383,7 @@ impl EntityIndex {let links: Vec<Link> = if get_links {let (links, _) = bincode::decode_from_slice(doc.get_first(links).and_then(|val| val.as_bytes()).unwrap(),- bincode::config::standard(),+ common::bincode_config(),).unwrap();diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rsindex 9d619d3e..39f706f5 100644--- a/crates/core/src/external_sort.rs+++ b/crates/core/src/external_sort.rs@@ -57,7 +57,7 @@ impl<T> Chunk<T> {let mut wrt = BufWriter::new(file);for item in self.data {- let bytes = bincode::encode_to_vec(&item, bincode::config::standard())?;+ let bytes = bincode::encode_to_vec(&item, common::bincode_config())?;let size = bytes.len() as u64;wrt.write_all(&size.to_le_bytes())?;@@ -179,7 +179,7 @@ impl<T> StoredChunk<T> {self.data.read_exact(&mut self.buf).ok()?;- let (next, _) = bincode::decode_from_slice(&self.buf, bincode::config::standard()).ok()?;+ let (next, _) = bincode::decode_from_slice(&self.buf, common::bincode_config()).ok()?;Some(next)}diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rsindex b64861b5..688901a6 100644--- a/crates/core/src/image_store.rs+++ b/crates/core/src/image_store.rs@@ -243,9 +243,9 @@ mod tests {.into(),);- let bytes = bincode::encode_to_vec(&image, bincode::config::standard()).unwrap();+ let bytes = bincode::encode_to_vec(&image, common::bincode_config()).unwrap();let (decoded_image, _) =- bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();+ bincode::decode_from_slice(&bytes, common::bincode_config()).unwrap();assert_eq!(image, decoded_image);}diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex b0807c96..ad3e3d32 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::sync::Arc;+use std::{ops::Add, sync::Arc};use crate::{collector::{self, BucketCollector},@@ -25,7 +25,7 @@ use crate::{use super::{models::lambdamart::{self, LambdaMART},- SignalCoefficient, SignalEnum, SignalScore,+ InboundSimilarity, SignalCoefficient, SignalEnum, SignalScore,};mod scorers;@@ -105,7 +105,13 @@ impl<T: RankableWebpage> RankingStage<T> {.map(|(signal, score)| self.coefficients.get(&signal) * score).sum()} else {- coeff * model.predict(signals)+ signals+ .get(InboundSimilarity.into())+ .copied()+ .unwrap_or(0.0)+ .add(1.0)+ * coeff+ * model.predict(signals)}}None => signalsdiff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex fb2fe0ec..7fbd1723 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -198,7 +198,7 @@ whereT: Graph,{async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {- self.batch_raw_ingoing(nodes, EdgeLimit::Limit(128))+ self.batch_raw_ingoing(nodes, EdgeLimit::Limit(4096)).await.into_iter().map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rsindex 6981176c..4783e92b 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/core/src/web_spell/stupid_backoff.rs@@ -143,7 +143,7 @@ impl StupidBackoffTrainer {let f = File::create(path.as_ref().join("n_counts.bin"))?;let mut wrt = BufWriter::new(f);- bincode::encode_into_std_write(&self.n_counts, &mut wrt, bincode::config::standard())?;+ bincode::encode_into_std_write(&self.n_counts, &mut wrt, common::bincode_config())?;wrt.flush()?;Ok(())@@ -226,7 +226,7 @@ impl StupidBackoff {let file = File::open(folder.as_ref().join("n_counts.bin"))?;let mut reader = std::io::BufReader::new(file);- let n_counts = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;+ let n_counts = bincode::decode_from_std_read(&mut reader, common::bincode_config())?;Ok(Self {ngrams,@@ -257,7 +257,7 @@ impl StupidBackoff {.open(folder.as_ref().join("n_counts.bin"))?;let mut wrt = BufWriter::new(file);- bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;+ bincode::encode_into_std_write(&n_counts, &mut wrt, common::bincode_config())?;wrt.flush()?;let file = OpenOptions::new()diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex 8c1e7fd5..8e1f8f0b 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -170,7 +170,7 @@ where}}-#[derive(Clone)]+#[derive(Debug, Clone)]pub struct StoredEdge<L = ()> {pub other: NodeDatum,pub rel: RelFlags,diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rsindex 9eeffb55..d39f28ae 100644--- a/crates/core/src/webgraph/merge.rs+++ b/crates/core/src/webgraph/merge.rs@@ -19,6 +19,7 @@ use std::{cmp::Reverse, collections::BinaryHeap};use super::{store::EdgeRange, NodeID, StoredEdge};use std::hash::Hash;+#[derive(Debug)]pub struct MergeNode<O = ()> {node: NodeID,range: EdgeRange,@@ -83,7 +84,7 @@ impl<O> PartialEq for MergeNode<O> {}impl<O> Eq for MergeNode<O> {}-#[derive(Clone, Copy)]+#[derive(Debug, Clone, Copy)]pub struct MergeSegmentOrd(usize);impl MergeSegmentOrd {pub fn new(ord: usize) -> Self {@@ -103,18 +104,18 @@ pub struct MergeIter<'a> {}impl<'a> MergeIter<'a> {pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {- let iters = iters- .into_iter()- .enumerate()- .map(|(ord, iter)| {- let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))- as Box<dyn Iterator<Item = _>>;-- Reverse(file_store::Peekable::new(it))- })- .collect();-- Self { iters }+ let mut heap = MinHeap::new();++ for item in iters.into_iter().enumerate().map(|(ord, iter)| {+ let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))+ as Box<dyn Iterator<Item = _>>;++ Reverse(file_store::Peekable::new(it))+ }) {+ heap.push(item);+ }++ Self { iters: heap }}pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {@@ -159,15 +160,15 @@ pub struct EdgeMerger<'a, L = String> {impl<'a, L> EdgeMerger<'a, L> {pub fn new(iters: Vec<impl Iterator<Item = StoredEdge<L>> + 'a>) -> Self {- let iters = iters- .into_iter()- .map(|iter| {- let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;- Reverse(file_store::Peekable::new(it))- })- .collect();-- Self { iters }+ let mut heap = MinHeap::new();+ for item in iters.into_iter().map(|iter| {+ let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;+ Reverse(file_store::Peekable::new(it))+ }) {+ heap.push(item);+ }++ Self { iters: heap }}}@@ -222,6 +223,7 @@ impl<L> Hash for StoredEdge<L> {#[cfg(test)]mod tests {use crate::{webgraph::NodeDatum, webpage::html::links::RelFlags};+ use proptest::prelude::*;use super::*;@@ -267,7 +269,7 @@ mod tests {}#[test]- fn test_merge_nodes_unequal_len() {+ fn test_merge_nodes_unequal_len1() {let a = vec![MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),@@ -298,6 +300,37 @@ mod tests {assert!(!merger.advance(&mut buf));}+ #[test]+ fn test_merge_nodes_unequal_len2() {+ let a = vec![MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10)];+ let b = vec![+ MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),+ MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),+ MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),+ ];++ let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);+ let mut buf = Vec::new();++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 1u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 2u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 4u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 5u64.into());++ assert!(!merger.advance(&mut buf));+ }+#[test]fn test_datum_merge() {let a = vec![@@ -343,4 +376,65 @@ mod tests {assert_eq!(merger.next().unwrap().other.host_rank(), 5);assert!(merger.next().is_none());}++ #[test]+ fn test_mergenode_heap_order() {+ let mut heap = MinHeap::new();++ heap.push(Reverse(MergeNode::new(+ 15414505588350686497u64.into(),+ EdgeRange::new(0..10, 1),+ 0..10,+ )));+ heap.push(Reverse(MergeNode::new(+ 470624832116206178u64.into(),+ EdgeRange::new(0..10, 1),+ 0..10,+ )));+ heap.push(Reverse(MergeNode::new(+ 15414505588350686497u64.into(),+ EdgeRange::new(0..10, 1),+ 0..10,+ )));++ assert_eq!(+ heap.into_iter()+ .map(|x| x.0.id().as_u64())+ .collect::<Vec<_>>(),+ vec![+ 470624832116206178,+ 15414505588350686497,+ 15414505588350686497+ ]+ );+ }++ proptest! {+ #[test]+ fn merge(mut ids: Vec<u64>) {+ ids.sort();++ let mut a: Vec<MergeNode> = Vec::new();+ let mut b: Vec<MergeNode> = Vec::new();++ for id in ids.clone() {+ if rand::random() {+ a.push(MergeNode::new(NodeID::from(id), EdgeRange::new(0..10, 4), 0..10));+ } else {+ b.push(MergeNode::new(NodeID::from(id), EdgeRange::new(0..10, 4), 0..10));+ }+ }++ let mut merge = MergeIter::new(vec![a.into_iter() , b.into_iter()]);++ let mut merged = Vec::new();+ let mut buf = Vec::new();++ while merge.advance(&mut buf) {+ merged.extend(buf.iter().map(|n| n.id().as_u64()));+ }++ prop_assert_eq!(merged, ids);+ }+ }}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 838aabcc..d66c8315 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -438,6 +438,7 @@ pub mod tests {use crate::webpage::html::links::RelFlags;use super::*;+ use proptest::prelude::*;pub fn test_edges() -> Vec<(Node, Node, String)> {vec![@@ -547,6 +548,13 @@ pub mod tests {graph.distances(Node::from("A")).get(&Node::from("H")),Some(&7));++ assert_eq!(+ graph+ .reversed_distances(Node::from("H"))+ .get(&Node::from("A")),+ Some(&7)+ );}#[test]@@ -709,6 +717,27 @@ pub mod tests {.len(),4);++ assert_eq!(+ graph+ .ingoing_edges(Node::from("B"), EdgeLimit::Unlimited)+ .len(),+ 1+ );++ assert_eq!(+ graph+ .ingoing_edges(Node::from("C"), EdgeLimit::Unlimited)+ .len(),+ 1+ );++ assert_eq!(+ graph+ .ingoing_edges(Node::from("D"), EdgeLimit::Unlimited)+ .len(),+ 1+ );}#[test]@@ -745,6 +774,162 @@ pub mod tests {.len(),4);++ assert_eq!(+ graph+ .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)+ .len(),+ 1+ );++ assert_eq!(+ graph+ .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)+ .len(),+ 1+ );++ assert_eq!(+ graph+ .outgoing_edges(Node::from("D"), EdgeLimit::Unlimited)+ .len(),+ 1+ );+ }++ proptest! {+ #[test]+ fn prop_merge(+ nodes in+ proptest::collection::vec(+ ("[a-z]", "[a-z]"), 0..100+ )+ ) {+ let mut graphs = Vec::new();+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ for (from, to) in nodes.clone() {+ wrt.insert(Node::new_for_test(from.as_str()), Node::new_for_test(to.as_str()), String::new(), RelFlags::default());++ if rand::random::<usize>() % 10 == 0 {+ graphs.push(wrt.finalize());+ wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ }+ }++ if graphs.is_empty() {+ return Ok(());+ }++ graphs.push(wrt.finalize());++ let mut graph = graphs.pop().unwrap();+++ for other in graphs {+ graph.merge(other).unwrap();+ }++ graph.merge_all_segments(Compression::default()).unwrap();++ for (from, to) in nodes {+ if from == to {+ continue;+ }++ let from = Node::new_for_test(from.as_str());+ let to = Node::new_for_test(to.as_str());++ let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);+ let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);++ prop_assert!(outgoing.iter().any(|e| e.to == to));+ prop_assert!(ingoing.iter().any(|e| e.from == from));+ }+ }+ }++ fn proptest_case(nodes: &[(&str, &str)]) {+ let mut graphs = Vec::new();+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );++ for (i, (from, to)) in nodes.iter().enumerate() {+ wrt.insert(+ Node::new_for_test(from),+ Node::new_for_test(to),+ String::new(),+ RelFlags::default(),+ );++ if i % 2 == 0 {+ graphs.push(wrt.finalize());+ wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ }+ }++ graphs.push(wrt.finalize());++ let mut graph = graphs.pop().unwrap();++ for other in graphs {+ graph.merge(other).unwrap();+ }++ graph.merge_all_segments(Compression::default()).unwrap();++ for (from, to) in nodes {+ if from == to {+ continue;+ }++ let from = Node::new_for_test(from);+ let to = Node::new_for_test(to);++ let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);+ let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);++ assert!(outgoing.iter().any(|e| e.to == to));+ assert!(ingoing.iter().any(|e| e.from == from));+ }+ }++ #[test]+ fn merge_proptest_case1() {+ let nodes = [("k", "d"), ("k", "t"), ("y", "m")];+ proptest_case(&nodes);+ }++ #[test]+ fn merge_proptest_case2() {+ let nodes = [("i", "k"), ("k", "g"), ("y", "m"), ("q", "r"), ("e", "g")];++ proptest_case(&nodes);+ }++ #[test]+ fn merge_proptest_case3() {+ let nodes = [("h", "c"), ("r", "r")];++ proptest_case(&nodes);}#[test]diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex 63a5b22f..8e964bd3 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -40,6 +40,14 @@ impl NodeID {pub fn as_u64(self) -> u64 {self.0}++ pub fn from_be_bytes(bytes: [u8; 8]) -> Self {+ NodeID(u64::from_be_bytes(bytes))+ }++ pub fn to_be_bytes(self) -> [u8; 8] {+ self.0.to_be_bytes()+ }}impl From<u128> for NodeID {@@ -111,6 +119,13 @@ pub struct Node {}impl Node {+ #[cfg(test)]+ pub fn new_for_test(name: &str) -> Self {+ Node {+ name: name.to_string(),+ }+ }+pub fn into_host(self) -> Node {let url = if self.name.contains("://") {Url::parse(&self.name)diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 2c140643..0805ca97 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -54,8 +54,8 @@ impl HostDb {pub fn insert(&mut self, node: &FullNodeID) {let key = [- node.host.as_u64().to_le_bytes(),- node.id.as_u64().to_le_bytes(),+ node.host.as_u64().to_be_bytes(),+ node.id.as_u64().to_be_bytes(),].concat();@@ -63,14 +63,14 @@ impl HostDb {}fn get(&self, host: &NodeID) -> Vec<NodeID> {- let host = host.as_u64().to_le_bytes().to_vec();+ let host = host.as_u64().to_be_bytes().to_vec();let query = speedy_kv::automaton::ExactMatch(&host).starts_with();self.db.search_raw(query).map(|(key, _)| {- let id = u64::from_le_bytes(+ let id = u64::from_be_bytes(key.as_bytes()[u64::BITS as usize / 8..].try_into().unwrap(),);NodeID::from(id)@@ -154,11 +154,13 @@ impl RangesDb {}fn merge_nodes(&self) -> impl Iterator<Item = MergeNode> + '_ {- self.edges.iter_raw().zip_eq(self.labels.iter_raw()).map(- move |((key_node, val), (key_label, labels))| {+ self.edges+ .sorted_iter_raw()+ .zip_eq(self.labels.sorted_iter_raw())+ .map(move |((key_node, val), (key_label, labels))| {debug_assert_eq!(key_node, key_label);- let node = NodeID::deserialize(key_node.as_bytes());+ let node = NodeID::from_be_bytes(key_node.as_bytes().try_into().unwrap());let range = EdgeRange::deserialize(val.as_bytes());let labels = {@@ -167,8 +169,7 @@ impl RangesDb {};MergeNode::new(node, range, labels)- },- )+ })}}@@ -201,13 +202,14 @@ impl ConstSerializable for NodeDatum {const BYTES: usize = std::mem::size_of::<NodeDatum>();fn serialize(&self, buf: &mut [u8]) {- self.node().serialize(&mut buf[..NodeID::BYTES]);- self.host_rank().serialize(&mut buf[NodeID::BYTES..]);+ let node = self.node().to_be_bytes();+ buf[..8].copy_from_slice(&node);+ self.host_rank().serialize(&mut buf[8..]);}fn deserialize(buf: &[u8]) -> Self {- let node = NodeID::deserialize(&buf[..NodeID::BYTES]);- let host_rank = u64::deserialize(&buf[NodeID::BYTES..]);+ let node = NodeID::from_be_bytes(buf[..8].try_into().unwrap());+ let host_rank = u64::deserialize(&buf[8..]);Self::new(node, host_rank)}@@ -260,7 +262,7 @@ impl LabelBlock {}pub fn compress(&self, compression: Compression) -> CompressedLabelBlock {- let bytes = bincode::encode_to_vec(self, bincode::config::standard()).unwrap();+ let bytes = bincode::encode_to_vec(self, common::bincode_config()).unwrap();let compressed = compression.compress(&bytes);CompressedLabelBlock {@@ -279,7 +281,7 @@ pub struct CompressedLabelBlock {impl CompressedLabelBlock {pub fn decompress(&self) -> LabelBlock {let bytes = self.compressions.decompress(&self.data);- let (res, _) = bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();+ let (res, _) = bincode::decode_from_slice(&bytes, common::bincode_config()).unwrap();res}}@@ -386,6 +388,7 @@ impl EdgeStore {// write postingslet node_host_rank = buf[0].range().host_rank;let node_id = buf[0].id();+ debug_assert!(buf.iter().all(|node| node.id() == node_id));let mut first_label_offset = None;let mut last_label_offset = None;let mut first_node_offset = None;@@ -431,7 +434,7 @@ impl EdgeStore {);let node_range_bytes = node_range.serialize_to_vec();- let node_bytes = node_id.serialize_to_vec();+ let node_bytes = node_id.as_u64().to_be_bytes().to_vec();ranges.insert_raw_node(node_bytes.clone(), node_range_bytes);let label_range_bytes = label_range.serialize_to_vec();@@ -484,7 +487,7 @@ impl EdgeStore {}pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {- let node_bytes = node.as_u64().to_le_bytes();+ let node_bytes = node.as_u64().to_be_bytes();match (self.ranges.edges.get_raw(&node_bytes),@@ -492,11 +495,11 @@ impl EdgeStore {) {(Some(node_range_bytes), Some(edge_range_bytes)) => {let node_range = EdgeRange::deserialize(node_range_bytes.as_bytes());- let edge_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());+ let label_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());let labels = self.edge_labels- .slice(usize_range(edge_range))+ .slice(usize_range(label_range)).map(|r| r.decompress()).flat_map(|block| block.labels.into_iter());@@ -531,7 +534,7 @@ impl EdgeStore {}pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {- let node_bytes = node.as_u64().to_le_bytes();+ let node_bytes = node.as_u64().to_be_bytes();match self.ranges.edges.get_raw(&node_bytes) {Some(node_range_bytes) => {@@ -572,7 +575,7 @@ impl EdgeStore {pub fn iter_without_label(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {self.ranges.edges.iter_raw().flat_map(move |(key, val)| {- let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());+ let node = u64::from_be_bytes((key.as_bytes()).try_into().unwrap());let node = NodeID::from(node);let edge_range = EdgeRange::deserialize(val.as_bytes());diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex b6588bcc..91e2a546 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -297,7 +297,7 @@ impl FinalEdgeStoreWriter {}self.hosts.insert(&node);- let node_bytes = node.id.serialize_to_vec();+ let node_bytes = node.id.as_u64().to_be_bytes().to_vec();debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());diff --git a/crates/core/src/webpage/safety_classifier.rs b/crates/core/src/webpage/safety_classifier.rsindex ceac0791..924f7ba8 100644--- a/crates/core/src/webpage/safety_classifier.rs+++ b/crates/core/src/webpage/safety_classifier.rs@@ -187,7 +187,7 @@ impl Model {.truncate(true).open(path)?;- bincode::encode_into_std_write(&self, &mut file, bincode::config::standard())?;+ bincode::encode_into_std_write(&self, &mut file, common::bincode_config())?;file.flush()?;Ok(())@@ -197,7 +197,7 @@ impl Model {let file = OpenOptions::new().read(true).open(path)?;let mut reader = std::io::BufReader::new(file);- let model = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;+ let model = bincode::decode_from_std_read(&mut reader, common::bincode_config())?;Ok(model)}diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.tomlindex 961fe387..c883142b 100644--- a/crates/file-store/Cargo.toml+++ b/crates/file-store/Cargo.toml@@ -9,5 +9,6 @@ version = "0.1.0"[dependencies]anyhow.workspace = truebincode.workspace = true+common.workspace = truememmap2.workspace = trueownedbytes.workspace = truediff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rsindex 4d343494..5f06828a 100644--- a/crates/file-store/src/iterable.rs+++ b/crates/file-store/src/iterable.rs@@ -102,7 +102,7 @@ where}pub fn write(&mut self, item: &T) -> Result<WrittenOffset> {- let serialized = bincode::encode_to_vec(item, bincode::config::standard())?;+ let serialized = bincode::encode_to_vec(item, common::bincode_config())?;let header = IterableHeader {num_upcoming_bytes: serialized.len() as u64,};@@ -187,8 +187,7 @@ wherelet serialized = &self.data[self.offset..self.offset + header.num_upcoming_bytes as usize];self.offset += header.num_upcoming_bytes as usize;- let (item, _) =- bincode::decode_from_slice(serialized, bincode::config::standard()).unwrap();+ let (item, _) = bincode::decode_from_slice(serialized, common::bincode_config()).unwrap();Some(item)}diff --git a/crates/file-store/src/peekable.rs b/crates/file-store/src/peekable.rsindex 65aab11c..1509ef65 100644--- a/crates/file-store/src/peekable.rs+++ b/crates/file-store/src/peekable.rs@@ -108,3 +108,15 @@ whereT: Eq,{}++impl<I, T> std::fmt::Debug for Peekable<I>+where+ I: Iterator<Item = T>,+ T: std::fmt::Debug,+{+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("Peekable")+ .field("peeked", &self.peeked)+ .finish()+ }+}diff --git a/crates/leechy/src/conf.toml b/crates/leechy/src/conf.tomlindex 1c492b01..a358d6b6 100644--- a/crates/leechy/src/conf.toml+++ b/crates/leechy/src/conf.toml@@ -1,4 +1,4 @@[[engines]]name = "startpage"-search = "https://www.startpage.com/search?q={query}&qsr=en_US"+search = "https://www.startpage.com/sp/search?q={query}&qsr=en_US"xpath = "./div[contains(@class, 'result')]/a"diff --git a/crates/speedy-kv/Cargo.toml b/crates/speedy-kv/Cargo.tomlindex 364195ea..01762f9c 100644--- a/crates/speedy-kv/Cargo.toml+++ b/crates/speedy-kv/Cargo.toml@@ -1,16 +1,17 @@[package]-name = "speedy_kv"-version = "0.1.0"edition = "2021"license = "AGPL-3.0"+name = "speedy_kv"+version = "0.1.0"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[dependencies]anyhow.workspace = truebincode.workspace = true-bloom = { path = "../bloom" }-file_store = { path = "../file-store" }+bloom = {path = "../bloom"}+common.workspace = true+file_store = {path = "../file-store"}fst.workspace = truememmap2.workspace = trueserde.workspace = truediff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rsindex 68cf8dd1..08a2c64e 100644--- a/crates/speedy-kv/src/lib.rs+++ b/crates/speedy-kv/src/lib.rs@@ -446,11 +446,11 @@ whereV: bincode::Decode,{pub fn get(&self, key: &K) -> Result<Option<V>> {- let key = bincode::encode_to_vec(key, bincode::config::standard())?;+ let key = bincode::encode_to_vec(key, common::bincode_config())?;match self.get_raw(key.as_slice()) {Some(v) => {- let (v, _) = bincode::decode_from_slice(v.as_bytes(), bincode::config::standard())?;+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config())?;Ok(Some(v))}None => Ok(None),@@ -467,10 +467,8 @@ whereA: fst::Automaton + Clone + 'a,{self.search_raw(query).filter_map(|(k, v)| {- let (k, _) =- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;- let (v, _) =- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;+ let (k, _) = bincode::decode_from_slice(k.as_bytes(), common::bincode_config()).ok()?;+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config()).ok()?;Some((k, v))})@@ -492,10 +490,8 @@ where{pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {self.iter_raw().filter_map(|(k, v)| {- let (k, _) =- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;- let (v, _) =- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;+ let (k, _) = bincode::decode_from_slice(k.as_bytes(), common::bincode_config()).ok()?;+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config()).ok()?;Some((k, v))})@@ -503,10 +499,8 @@ wherepub fn sorted_iter(&self) -> impl Iterator<Item = (K, V)> + '_ {self.sorted_iter_raw().filter_map(|(k, v)| {- let (k, _) =- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;- let (v, _) =- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;+ let (k, _) = bincode::decode_from_slice(k.as_bytes(), common::bincode_config()).ok()?;+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config()).ok()?;Some((k, v))})diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rsindex 458a55d2..196aacf3 100644--- a/crates/speedy-kv/src/segment.rs+++ b/crates/speedy-kv/src/segment.rs@@ -79,7 +79,7 @@ whereself.writers.store.finish()?;let mut wrt = BufWriter::new(self.writers.bloom);- bincode::encode_into_std_write(self.bloom, &mut wrt, bincode::config::standard())?;+ bincode::encode_into_std_write(self.bloom, &mut wrt, common::bincode_config())?;wrt.flush()?;Ok(())@@ -117,7 +117,7 @@ impl<K, V> Segment<K, V> {.read(true).open(folder.as_ref().join(Segment::<K, V>::bloom_file_name(uuid)))?;- let bloom = bincode::decode_from_std_read(&mut bloom, bincode::config::standard())?;+ let bloom = bincode::decode_from_std_read(&mut bloom, common::bincode_config())?;Ok(Self {uuid,@@ -363,13 +363,17 @@ whereI: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,{pub fn new(segments: Vec<Peekable<I>>) -> Self {- Self {- segments: segments- .into_iter()- .enumerate()- .map(|(segment_ord, iter)| SortedPeekable { segment_ord, iter })- .collect(),+ let mut heap = BinaryHeap::new();++ for item in segments+ .into_iter()+ .enumerate()+ .map(|(segment_ord, iter)| SortedPeekable { segment_ord, iter })+ {+ heap.push(item);}++ Self { segments: heap }}}diff --git a/crates/speedy-kv/src/serialized.rs b/crates/speedy-kv/src/serialized.rsindex aef8e47f..38491db4 100644--- a/crates/speedy-kv/src/serialized.rs+++ b/crates/speedy-kv/src/serialized.rs@@ -84,7 +84,7 @@ whereT: bincode::Encode,{pub fn new(value: &T) -> Result<Self> {- let bytes = bincode::encode_to_vec(value, bincode::config::standard())?;+ let bytes = bincode::encode_to_vec(value, common::bincode_config())?;Ok(Self {bytes,_marker: std::marker::PhantomData,diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 26a27a30..a67d3659 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -13,10 +13,12 @@ param_grid = {"verbosity": [-1],"metric": ["ndcg"],"ndcg_at": [[1, 2, 3, 5, 10]],- "learning_rate": [0.025],- "max_depth": [-1, 2, 4, 8],- "num_leaves": [7, 15, 31],- "lambda_l2": [0.1, 0.5, 1.0],+ "learning_rate": [0.003],+ # "max_depth": [-1, 2, 4, 8],+ "max_depth": [-1],+ # "num_leaves": [7, 15, 31],+ "num_leaves": [31],+ "lambda_l2": [1.0, 1.5, 2.0],"linear_tree": [False],}@@ -149,10 +151,10 @@ for param in params:scores = []for metric, vals in res.items():- if metric in ['cvbooster']:+ if metric in ["cvbooster"]:continue- if 'stdv' in metric:+ if "stdv" in metric:continuescores.append(vals[0])@@ -188,7 +190,11 @@ print()print("Feature importance:")pprint(sorted(- [(id2feature[i], v) for i, v in enumerate(booster.feature_importance()) if v > 0],+ [+ (id2feature[i], v)+ for i, v in enumerate(booster.feature_importance())+ if v > 0+ ],key=lambda x: x[1],reverse=True,)diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.pyindex 6f38b471..cf975ffe 100644--- a/ltr/leechy_annotate.py+++ b/ltr/leechy_annotate.py@@ -4,6 +4,7 @@ import stractimport numpy as npfrom tqdm import tqdmimport time+import sysNUM_LABELS = 4@@ -34,7 +35,16 @@ unannotated_queries = db.get_unannotated_queries()eng = leechy.Engine()for qid, query in tqdm(unannotated_queries.items()):tqdm.write(query)- leechy_results = eng.search(query)+ leechy_results = []++ for i in range(3):+ leechy_results = eng.search(query)+ if len(leechy_results) > 0:+ break++ if len(leechy_results) == 0:+ tqdm.write(f"No results found for {query}")+ sys.exit(-1)for i, result in enumerate(leechy_results):label = NUM_LABELS - int(np.log2(i + 1))diff --git a/ltr/stract.py b/ltr/stract.pyindex 25026f27..2617ffd8 100644--- a/ltr/stract.py+++ b/ltr/stract.py@@ -4,6 +4,7 @@ import requestsAPI = "http://localhost:3000/beta/api/search"NUM_RESULTS_PER_QUERY = 20+def simplify_snippet(snippet):if "text" not in snippet:return ""@@ -15,7 +16,7 @@ def search(query, page=0):payload = {"query": query,"numResults": NUM_RESULTS_PER_QUERY,- 'page': page,+ "page": page,"returnRankingSignals": True,}@@ -29,4 +30,4 @@ def search(query, page=0):for w in requests.post(API, json=payload).json()["webpages"][:NUM_RESULTS_PER_QUERY]- ]\ No newline at end of file+ ]