Skip to content

web search done right

store node ids as big endian in webgraph so sort is correct during merge

Mikkel Denker308388262fd3
diff --git a/Cargo.lock b/Cargo.lock
index 60b6d1de..36fe23a8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -927,6 +927,13 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+[[package]]
+name = "common"
+version = "0.1.0"
+dependencies = [
+ "bincode",
+]
+
[[package]]
name = "console"
version = "0.15.8"
@@ -1603,6 +1610,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bincode",
+ "common",
"memmap2",
"ownedbytes",
]
@@ -4906,6 +4914,7 @@ dependencies = [
"anyhow",
"bincode",
"bloom",
+ "common",
"file_store",
"fst",
"memmap2",
@@ -4970,6 +4979,7 @@ dependencies = [
"chitchat",
"chrono",
"clap",
+ "common",
"criterion",
"crossbeam-channel",
"csv",
diff --git a/Cargo.toml b/Cargo.toml
index 423bb260..c43146ad 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,6 +14,7 @@ members = [
"crates/robotstxt",
"crates/tantivy",
"crates/leechy",
+ "crates/common",
"fuzz",
]
resolver = "2"
@@ -60,6 +61,7 @@ chitchat = "0.5.0"
chrono = {version = "0.4.23", features = ["serde"]}
clap = {version = "4.4.6", features = ["derive"]}
cmake = "0.1"
+common = {path = "crates/common"}
crc32fast = "1.3.2"
criterion = "0.5.1"
crossbeam-channel = "0.5.6"
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rs
index 81dbc86d..2927a674 100644
--- a/crates/bloom/src/lib.rs
+++ b/crates/bloom/src/lib.rs
@@ -210,22 +210,22 @@ mod tests {
#[test]
fn test_bloom_filter_bytes() {
let mut bf = BytesBloomFilter::new(100, 0.01);
- bf.insert(&1u64.to_be_bytes());
- bf.insert(&2u64.to_be_bytes());
- bf.insert(&3u64.to_be_bytes());
- bf.insert(&4u64.to_be_bytes());
- bf.insert(&5u64.to_be_bytes());
-
- assert!(bf.contains(&1u64.to_be_bytes()));
- assert!(bf.contains(&2u64.to_be_bytes()));
- assert!(bf.contains(&3u64.to_be_bytes()));
- assert!(bf.contains(&4u64.to_be_bytes()));
- assert!(bf.contains(&5u64.to_be_bytes()));
- assert!(!bf.contains(&6u64.to_be_bytes()));
- assert!(!bf.contains(&7u64.to_be_bytes()));
- assert!(!bf.contains(&8u64.to_be_bytes()));
- assert!(!bf.contains(&9u64.to_be_bytes()));
- assert!(!bf.contains(&10u64.to_be_bytes()));
+ bf.insert(&1u64.to_le_bytes());
+ bf.insert(&2u64.to_le_bytes());
+ bf.insert(&3u64.to_le_bytes());
+ bf.insert(&4u64.to_le_bytes());
+ bf.insert(&5u64.to_le_bytes());
+
+ assert!(bf.contains(&1u64.to_le_bytes()));
+ assert!(bf.contains(&2u64.to_le_bytes()));
+ assert!(bf.contains(&3u64.to_le_bytes()));
+ assert!(bf.contains(&4u64.to_le_bytes()));
+ assert!(bf.contains(&5u64.to_le_bytes()));
+ assert!(!bf.contains(&6u64.to_le_bytes()));
+ assert!(!bf.contains(&7u64.to_le_bytes()));
+ assert!(!bf.contains(&8u64.to_le_bytes()));
+ assert!(!bf.contains(&9u64.to_le_bytes()));
+ assert!(!bf.contains(&10u64.to_le_bytes()));
}
#[test]
diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml
new file mode 100644
index 00000000..392c244d
--- /dev/null
+++ b/crates/common/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+edition = "2021"
+license = "AGPL-3.0"
+name = "common"
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bincode.workspace = true
diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs
new file mode 100644
index 00000000..be3248f5
--- /dev/null
+++ b/crates/common/src/lib.rs
@@ -0,0 +1,3 @@
+pub fn bincode_config() -> bincode::config::Configuration {
+ bincode::config::standard()
+}
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index db9d2289..91bc7fc7 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -40,6 +40,7 @@ chardetng.workspace = true
chitchat.workspace = true
chrono.workspace = true
clap.workspace = true
+common.workspace = true
crossbeam-channel.workspace = true
csv.workspace = true
dashmap.workspace = true
diff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rs
index 6ed973e1..6457e8e7 100644
--- a/crates/core/src/ampc/dht/store.rs
+++ b/crates/core/src/ampc/dht/store.rs
@@ -274,7 +274,7 @@ impl RaftSnapshotBuilder<TypeConfig> for Arc<StateMachineStore> {
{
// Serialize the data of the state machine.
let state_machine = self.state_machine.read().await;
- let encoded = bincode::encode_to_vec(&*state_machine, bincode::config::standard())
+ let encoded = bincode::encode_to_vec(&*state_machine, common::bincode_config())
.map_err(|e| StorageIOError::read_state_machine(&e))?;
data = encoded;
@@ -433,10 +433,9 @@ impl RaftStateMachine<TypeConfig> for Arc<StateMachineStore> {
// Update the state machine.
{
let (data, _): (Db, _) =
- bincode::decode_from_slice(&new_snapshot.data, bincode::config::standard())
- .map_err(|e| {
- StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e)
- })?;
+ bincode::decode_from_slice(&new_snapshot.data, common::bincode_config()).map_err(
+ |e| StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e),
+ )?;
let mut state_machine = self.state_machine.write().await;
state_machine.db = data;
diff --git a/crates/core/src/api/user_count.rs b/crates/core/src/api/user_count.rs
index 846bed4f..678c982d 100644
--- a/crates/core/src/api/user_count.rs
+++ b/crates/core/src/api/user_count.rs
@@ -83,7 +83,7 @@ impl<F: Frequency> InnerUserCount<F> {
// It is first hashed with a salt for good measure, and then we only use
// the id for a probabilistic count using hyperloglog.
self.maybe_reset();
- let bytes = bincode::encode_to_vec(user_id, bincode::config::standard())?;
+ let bytes = bincode::encode_to_vec(user_id, common::bincode_config())?;
let mut hash = [0u8; digest::SHA512_OUTPUT_LEN];
pbkdf2::derive(
diff --git a/crates/core/src/crawler/file_queue.rs b/crates/core/src/crawler/file_queue.rs
index c3159883..c6a311c2 100644
--- a/crates/core/src/crawler/file_queue.rs
+++ b/crates/core/src/crawler/file_queue.rs
@@ -61,7 +61,7 @@ where
}
pub fn push(&mut self, item: T) -> Result<()> {
- let body = bincode::encode_to_vec(&item, bincode::config::standard())?;
+ let body = bincode::encode_to_vec(&item, common::bincode_config())?;
let header = Header {
body_size: body.len(),
};
@@ -177,7 +177,7 @@ where
let body =
&self.file[cur_pointer + header_size..cur_pointer + header_size + header.body_size];
- let (item, _) = bincode::decode_from_slice(body, bincode::config::standard())?;
+ let (item, _) = bincode::decode_from_slice(body, common::bincode_config())?;
self.pointer
.set(cur_pointer + header_size + header.body_size)?;
diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rs
index 84a686ec..713483c3 100644
--- a/crates/core/src/distributed/sonic/mod.rs
+++ b/crates/core/src/distributed/sonic/mod.rs
@@ -121,7 +121,7 @@ where
async fn send_without_timeout(&mut self, request: &Req) -> Result<Res> {
self.awaiting_res = true;
- let bytes = bincode::encode_to_vec(request, bincode::config::standard()).unwrap();
+ let bytes = bincode::encode_to_vec(request, common::bincode_config()).unwrap();
let header = Header {
body_size: bytes.len(),
@@ -147,7 +147,7 @@ where
self.stream.flush().await?;
tracing::debug!("deserializing {:?}", std::any::type_name::<(Req, Res)>());
- let (res, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();
+ let (res, _) = bincode::decode_from_slice(&buf, common::bincode_config()).unwrap();
self.awaiting_res = false;
@@ -249,7 +249,7 @@ where
self.stream.read_exact(&mut buf).await?;
- let (body, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();
+ let (body, _) = bincode::decode_from_slice(&buf, common::bincode_config()).unwrap();
Ok(Request {
conn: self,
@@ -268,7 +268,7 @@ where
Res: bincode::Encode,
{
async fn respond_without_timeout(self, response: Res) -> Result<()> {
- let bytes = bincode::encode_to_vec(&response, bincode::config::standard()).unwrap();
+ let bytes = bincode::encode_to_vec(&response, common::bincode_config()).unwrap();
let header = Header {
body_size: bytes.len(),
};
diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rs
index 88187572..2b090830 100644
--- a/crates/core/src/entity_index/mod.rs
+++ b/crates/core/src/entity_index/mod.rs
@@ -96,11 +96,11 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> Tantiv
);
doc.add_bytes(
schema.get_field("info").unwrap(),
- &bincode::encode_to_vec(&entity.info, bincode::config::standard()).unwrap(),
+ &bincode::encode_to_vec(&entity.info, common::bincode_config()).unwrap(),
);
doc.add_bytes(
schema.get_field("links").unwrap(),
- &bincode::encode_to_vec(&entity.page_abstract.links, bincode::config::standard()).unwrap(),
+ &bincode::encode_to_vec(&entity.page_abstract.links, common::bincode_config()).unwrap(),
);
let has_image = if entity.image.is_some() {
"true"
@@ -346,7 +346,7 @@ impl EntityIndex {
let info = if decode_info {
let (info, _) = bincode::decode_from_slice(
doc.get_first(info).and_then(|val| val.as_bytes()).unwrap(),
- bincode::config::standard(),
+ common::bincode_config(),
)
.unwrap();
@@ -383,7 +383,7 @@ impl EntityIndex {
let links: Vec<Link> = if get_links {
let (links, _) = bincode::decode_from_slice(
doc.get_first(links).and_then(|val| val.as_bytes()).unwrap(),
- bincode::config::standard(),
+ common::bincode_config(),
)
.unwrap();
diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rs
index 9d619d3e..39f706f5 100644
--- a/crates/core/src/external_sort.rs
+++ b/crates/core/src/external_sort.rs
@@ -57,7 +57,7 @@ impl<T> Chunk<T> {
let mut wrt = BufWriter::new(file);
for item in self.data {
- let bytes = bincode::encode_to_vec(&item, bincode::config::standard())?;
+ let bytes = bincode::encode_to_vec(&item, common::bincode_config())?;
let size = bytes.len() as u64;
wrt.write_all(&size.to_le_bytes())?;
@@ -179,7 +179,7 @@ impl<T> StoredChunk<T> {
self.data.read_exact(&mut self.buf).ok()?;
- let (next, _) = bincode::decode_from_slice(&self.buf, bincode::config::standard()).ok()?;
+ let (next, _) = bincode::decode_from_slice(&self.buf, common::bincode_config()).ok()?;
Some(next)
}
diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rs
index b64861b5..688901a6 100644
--- a/crates/core/src/image_store.rs
+++ b/crates/core/src/image_store.rs
@@ -243,9 +243,9 @@ mod tests {
.into(),
);
- let bytes = bincode::encode_to_vec(&image, bincode::config::standard()).unwrap();
+ let bytes = bincode::encode_to_vec(&image, common::bincode_config()).unwrap();
let (decoded_image, _) =
- bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();
+ bincode::decode_from_slice(&bytes, common::bincode_config()).unwrap();
assert_eq!(image, decoded_image);
}
diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rs
index b0807c96..ad3e3d32 100644
--- a/crates/core/src/ranking/pipeline/mod.rs
+++ b/crates/core/src/ranking/pipeline/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::sync::Arc;
+use std::{ops::Add, sync::Arc};
use crate::{
collector::{self, BucketCollector},
@@ -25,7 +25,7 @@ use crate::{
use super::{
models::lambdamart::{self, LambdaMART},
- SignalCoefficient, SignalEnum, SignalScore,
+ InboundSimilarity, SignalCoefficient, SignalEnum, SignalScore,
};
mod scorers;
@@ -105,7 +105,13 @@ impl<T: RankableWebpage> RankingStage<T> {
.map(|(signal, score)| self.coefficients.get(&signal) * score)
.sum()
} else {
- coeff * model.predict(signals)
+ signals
+ .get(InboundSimilarity.into())
+ .copied()
+ .unwrap_or(0.0)
+ .add(1.0)
+ * coeff
+ * model.predict(signals)
}
}
None => signals
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index fb2fe0ec..7fbd1723 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -198,7 +198,7 @@ where
T: Graph,
{
async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {
- self.batch_raw_ingoing(nodes, EdgeLimit::Limit(128))
+ self.batch_raw_ingoing(nodes, EdgeLimit::Limit(4096))
.await
.into_iter()
.map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())
diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rs
index 6981176c..4783e92b 100644
--- a/crates/core/src/web_spell/stupid_backoff.rs
+++ b/crates/core/src/web_spell/stupid_backoff.rs
@@ -143,7 +143,7 @@ impl StupidBackoffTrainer {
let f = File::create(path.as_ref().join("n_counts.bin"))?;
let mut wrt = BufWriter::new(f);
- bincode::encode_into_std_write(&self.n_counts, &mut wrt, bincode::config::standard())?;
+ bincode::encode_into_std_write(&self.n_counts, &mut wrt, common::bincode_config())?;
wrt.flush()?;
Ok(())
@@ -226,7 +226,7 @@ impl StupidBackoff {
let file = File::open(folder.as_ref().join("n_counts.bin"))?;
let mut reader = std::io::BufReader::new(file);
- let n_counts = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;
+ let n_counts = bincode::decode_from_std_read(&mut reader, common::bincode_config())?;
Ok(Self {
ngrams,
@@ -257,7 +257,7 @@ impl StupidBackoff {
.open(folder.as_ref().join("n_counts.bin"))?;
let mut wrt = BufWriter::new(file);
- bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;
+ bincode::encode_into_std_write(&n_counts, &mut wrt, common::bincode_config())?;
wrt.flush()?;
let file = OpenOptions::new()
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rs
index 8c1e7fd5..8e1f8f0b 100644
--- a/crates/core/src/webgraph/edge.rs
+++ b/crates/core/src/webgraph/edge.rs
@@ -170,7 +170,7 @@ where
}
}
-#[derive(Clone)]
+#[derive(Debug, Clone)]
pub struct StoredEdge<L = ()> {
pub other: NodeDatum,
pub rel: RelFlags,
diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rs
index 9eeffb55..d39f28ae 100644
--- a/crates/core/src/webgraph/merge.rs
+++ b/crates/core/src/webgraph/merge.rs
@@ -19,6 +19,7 @@ use std::{cmp::Reverse, collections::BinaryHeap};
use super::{store::EdgeRange, NodeID, StoredEdge};
use std::hash::Hash;
+#[derive(Debug)]
pub struct MergeNode<O = ()> {
node: NodeID,
range: EdgeRange,
@@ -83,7 +84,7 @@ impl<O> PartialEq for MergeNode<O> {
}
impl<O> Eq for MergeNode<O> {}
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
pub struct MergeSegmentOrd(usize);
impl MergeSegmentOrd {
pub fn new(ord: usize) -> Self {
@@ -103,18 +104,18 @@ pub struct MergeIter<'a> {
}
impl<'a> MergeIter<'a> {
pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {
- let iters = iters
- .into_iter()
- .enumerate()
- .map(|(ord, iter)| {
- let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))
- as Box<dyn Iterator<Item = _>>;
-
- Reverse(file_store::Peekable::new(it))
- })
- .collect();
-
- Self { iters }
+ let mut heap = MinHeap::new();
+
+ for item in iters.into_iter().enumerate().map(|(ord, iter)| {
+ let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))
+ as Box<dyn Iterator<Item = _>>;
+
+ Reverse(file_store::Peekable::new(it))
+ }) {
+ heap.push(item);
+ }
+
+ Self { iters: heap }
}
pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {
@@ -159,15 +160,15 @@ pub struct EdgeMerger<'a, L = String> {
impl<'a, L> EdgeMerger<'a, L> {
pub fn new(iters: Vec<impl Iterator<Item = StoredEdge<L>> + 'a>) -> Self {
- let iters = iters
- .into_iter()
- .map(|iter| {
- let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;
- Reverse(file_store::Peekable::new(it))
- })
- .collect();
-
- Self { iters }
+ let mut heap = MinHeap::new();
+ for item in iters.into_iter().map(|iter| {
+ let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;
+ Reverse(file_store::Peekable::new(it))
+ }) {
+ heap.push(item);
+ }
+
+ Self { iters: heap }
}
}
@@ -222,6 +223,7 @@ impl<L> Hash for StoredEdge<L> {
#[cfg(test)]
mod tests {
use crate::{webgraph::NodeDatum, webpage::html::links::RelFlags};
+ use proptest::prelude::*;
use super::*;
@@ -267,7 +269,7 @@ mod tests {
}
#[test]
- fn test_merge_nodes_unequal_len() {
+ fn test_merge_nodes_unequal_len1() {
let a = vec![
MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),
MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),
@@ -298,6 +300,37 @@ mod tests {
assert!(!merger.advance(&mut buf));
}
+ #[test]
+ fn test_merge_nodes_unequal_len2() {
+ let a = vec![MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10)];
+ let b = vec![
+ MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),
+ MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),
+ MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),
+ ];
+
+ let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);
+ let mut buf = Vec::new();
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 1u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 2u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 4u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 5u64.into());
+
+ assert!(!merger.advance(&mut buf));
+ }
+
#[test]
fn test_datum_merge() {
let a = vec![
@@ -343,4 +376,65 @@ mod tests {
assert_eq!(merger.next().unwrap().other.host_rank(), 5);
assert!(merger.next().is_none());
}
+
+ #[test]
+ fn test_mergenode_heap_order() {
+ let mut heap = MinHeap::new();
+
+ heap.push(Reverse(MergeNode::new(
+ 15414505588350686497u64.into(),
+ EdgeRange::new(0..10, 1),
+ 0..10,
+ )));
+ heap.push(Reverse(MergeNode::new(
+ 470624832116206178u64.into(),
+ EdgeRange::new(0..10, 1),
+ 0..10,
+ )));
+ heap.push(Reverse(MergeNode::new(
+ 15414505588350686497u64.into(),
+ EdgeRange::new(0..10, 1),
+ 0..10,
+ )));
+
+ assert_eq!(
+ heap.into_iter()
+ .map(|x| x.0.id().as_u64())
+ .collect::<Vec<_>>(),
+ vec![
+ 470624832116206178,
+ 15414505588350686497,
+ 15414505588350686497
+ ]
+ );
+ }
+
+ proptest! {
+ #[test]
+ fn merge(mut ids: Vec<u64>) {
+ ids.sort();
+
+ let mut a: Vec<MergeNode> = Vec::new();
+ let mut b: Vec<MergeNode> = Vec::new();
+
+ for id in ids.clone() {
+ if rand::random() {
+ a.push(MergeNode::new(NodeID::from(id), EdgeRange::new(0..10, 4), 0..10));
+ } else {
+ b.push(MergeNode::new(NodeID::from(id), EdgeRange::new(0..10, 4), 0..10));
+ }
+ }
+
+ let mut merge = MergeIter::new(vec![a.into_iter() , b.into_iter()]);
+
+ let mut merged = Vec::new();
+ let mut buf = Vec::new();
+
+ while merge.advance(&mut buf) {
+ merged.extend(buf.iter().map(|n| n.id().as_u64()));
+ }
+
+ prop_assert_eq!(merged, ids);
+ }
+ }
}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 838aabcc..d66c8315 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -438,6 +438,7 @@ pub mod tests {
use crate::webpage::html::links::RelFlags;
use super::*;
+ use proptest::prelude::*;
pub fn test_edges() -> Vec<(Node, Node, String)> {
vec![
@@ -547,6 +548,13 @@ pub mod tests {
graph.distances(Node::from("A")).get(&Node::from("H")),
Some(&7)
);
+
+ assert_eq!(
+ graph
+ .reversed_distances(Node::from("H"))
+ .get(&Node::from("A")),
+ Some(&7)
+ );
}
#[test]
@@ -709,6 +717,27 @@ pub mod tests {
.len(),
4
);
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("B"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("C"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("D"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
}
#[test]
@@ -745,6 +774,162 @@ pub mod tests {
.len(),
4
);
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("D"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+ }
+
+ proptest! {
+ #[test]
+ fn prop_merge(
+ nodes in
+ proptest::collection::vec(
+ ("[a-z]", "[a-z]"), 0..100
+ )
+ ) {
+ let mut graphs = Vec::new();
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ for (from, to) in nodes.clone() {
+ wrt.insert(Node::new_for_test(from.as_str()), Node::new_for_test(to.as_str()), String::new(), RelFlags::default());
+
+ if rand::random::<usize>() % 10 == 0 {
+ graphs.push(wrt.finalize());
+ wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ }
+ }
+
+ if graphs.is_empty() {
+ return Ok(());
+ }
+
+ graphs.push(wrt.finalize());
+
+ let mut graph = graphs.pop().unwrap();
+
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.merge_all_segments(Compression::default()).unwrap();
+
+ for (from, to) in nodes {
+ if from == to {
+ continue;
+ }
+
+ let from = Node::new_for_test(from.as_str());
+ let to = Node::new_for_test(to.as_str());
+
+ let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);
+ let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);
+
+ prop_assert!(outgoing.iter().any(|e| e.to == to));
+ prop_assert!(ingoing.iter().any(|e| e.from == from));
+ }
+ }
+ }
+
+ fn proptest_case(nodes: &[(&str, &str)]) {
+ let mut graphs = Vec::new();
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+
+ for (i, (from, to)) in nodes.iter().enumerate() {
+ wrt.insert(
+ Node::new_for_test(from),
+ Node::new_for_test(to),
+ String::new(),
+ RelFlags::default(),
+ );
+
+ if i % 2 == 0 {
+ graphs.push(wrt.finalize());
+ wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ }
+ }
+
+ graphs.push(wrt.finalize());
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.merge_all_segments(Compression::default()).unwrap();
+
+ for (from, to) in nodes {
+ if from == to {
+ continue;
+ }
+
+ let from = Node::new_for_test(from);
+ let to = Node::new_for_test(to);
+
+ let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);
+ let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);
+
+ assert!(outgoing.iter().any(|e| e.to == to));
+ assert!(ingoing.iter().any(|e| e.from == from));
+ }
+ }
+
+ #[test]
+ fn merge_proptest_case1() {
+ let nodes = [("k", "d"), ("k", "t"), ("y", "m")];
+ proptest_case(&nodes);
+ }
+
+ #[test]
+ fn merge_proptest_case2() {
+ let nodes = [("i", "k"), ("k", "g"), ("y", "m"), ("q", "r"), ("e", "g")];
+
+ proptest_case(&nodes);
+ }
+
+ #[test]
+ fn merge_proptest_case3() {
+ let nodes = [("h", "c"), ("r", "r")];
+
+ proptest_case(&nodes);
}
#[test]
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index 63a5b22f..8e964bd3 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -40,6 +40,14 @@ impl NodeID {
pub fn as_u64(self) -> u64 {
self.0
}
+
+ pub fn from_be_bytes(bytes: [u8; 8]) -> Self {
+ NodeID(u64::from_be_bytes(bytes))
+ }
+
+ pub fn to_be_bytes(self) -> [u8; 8] {
+ self.0.to_be_bytes()
+ }
}
impl From<u128> for NodeID {
@@ -111,6 +119,13 @@ pub struct Node {
}
impl Node {
+ #[cfg(test)]
+ pub fn new_for_test(name: &str) -> Self {
+ Node {
+ name: name.to_string(),
+ }
+ }
+
pub fn into_host(self) -> Node {
let url = if self.name.contains("://") {
Url::parse(&self.name)
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 2c140643..0805ca97 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -54,8 +54,8 @@ impl HostDb {
pub fn insert(&mut self, node: &FullNodeID) {
let key = [
- node.host.as_u64().to_le_bytes(),
- node.id.as_u64().to_le_bytes(),
+ node.host.as_u64().to_be_bytes(),
+ node.id.as_u64().to_be_bytes(),
]
.concat();
@@ -63,14 +63,14 @@ impl HostDb {
}
fn get(&self, host: &NodeID) -> Vec<NodeID> {
- let host = host.as_u64().to_le_bytes().to_vec();
+ let host = host.as_u64().to_be_bytes().to_vec();
let query = speedy_kv::automaton::ExactMatch(&host).starts_with();
self.db
.search_raw(query)
.map(|(key, _)| {
- let id = u64::from_le_bytes(
+ let id = u64::from_be_bytes(
key.as_bytes()[u64::BITS as usize / 8..].try_into().unwrap(),
);
NodeID::from(id)
@@ -154,11 +154,13 @@ impl RangesDb {
}
fn merge_nodes(&self) -> impl Iterator<Item = MergeNode> + '_ {
- self.edges.iter_raw().zip_eq(self.labels.iter_raw()).map(
- move |((key_node, val), (key_label, labels))| {
+ self.edges
+ .sorted_iter_raw()
+ .zip_eq(self.labels.sorted_iter_raw())
+ .map(move |((key_node, val), (key_label, labels))| {
debug_assert_eq!(key_node, key_label);
- let node = NodeID::deserialize(key_node.as_bytes());
+ let node = NodeID::from_be_bytes(key_node.as_bytes().try_into().unwrap());
let range = EdgeRange::deserialize(val.as_bytes());
let labels = {
@@ -167,8 +169,7 @@ impl RangesDb {
};
MergeNode::new(node, range, labels)
- },
- )
+ })
}
}
@@ -201,13 +202,14 @@ impl ConstSerializable for NodeDatum {
const BYTES: usize = std::mem::size_of::<NodeDatum>();
fn serialize(&self, buf: &mut [u8]) {
- self.node().serialize(&mut buf[..NodeID::BYTES]);
- self.host_rank().serialize(&mut buf[NodeID::BYTES..]);
+ let node = self.node().to_be_bytes();
+ buf[..8].copy_from_slice(&node);
+ self.host_rank().serialize(&mut buf[8..]);
}
fn deserialize(buf: &[u8]) -> Self {
- let node = NodeID::deserialize(&buf[..NodeID::BYTES]);
- let host_rank = u64::deserialize(&buf[NodeID::BYTES..]);
+ let node = NodeID::from_be_bytes(buf[..8].try_into().unwrap());
+ let host_rank = u64::deserialize(&buf[8..]);
Self::new(node, host_rank)
}
@@ -260,7 +262,7 @@ impl LabelBlock {
}
pub fn compress(&self, compression: Compression) -> CompressedLabelBlock {
- let bytes = bincode::encode_to_vec(self, bincode::config::standard()).unwrap();
+ let bytes = bincode::encode_to_vec(self, common::bincode_config()).unwrap();
let compressed = compression.compress(&bytes);
CompressedLabelBlock {
@@ -279,7 +281,7 @@ pub struct CompressedLabelBlock {
impl CompressedLabelBlock {
pub fn decompress(&self) -> LabelBlock {
let bytes = self.compressions.decompress(&self.data);
- let (res, _) = bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();
+ let (res, _) = bincode::decode_from_slice(&bytes, common::bincode_config()).unwrap();
res
}
}
@@ -386,6 +388,7 @@ impl EdgeStore {
// write postings
let node_host_rank = buf[0].range().host_rank;
let node_id = buf[0].id();
+ debug_assert!(buf.iter().all(|node| node.id() == node_id));
let mut first_label_offset = None;
let mut last_label_offset = None;
let mut first_node_offset = None;
@@ -431,7 +434,7 @@ impl EdgeStore {
);
let node_range_bytes = node_range.serialize_to_vec();
- let node_bytes = node_id.serialize_to_vec();
+ let node_bytes = node_id.as_u64().to_be_bytes().to_vec();
ranges.insert_raw_node(node_bytes.clone(), node_range_bytes);
let label_range_bytes = label_range.serialize_to_vec();
@@ -484,7 +487,7 @@ impl EdgeStore {
}
pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {
- let node_bytes = node.as_u64().to_le_bytes();
+ let node_bytes = node.as_u64().to_be_bytes();
match (
self.ranges.edges.get_raw(&node_bytes),
@@ -492,11 +495,11 @@ impl EdgeStore {
) {
(Some(node_range_bytes), Some(edge_range_bytes)) => {
let node_range = EdgeRange::deserialize(node_range_bytes.as_bytes());
- let edge_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());
+ let label_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());
let labels = self
.edge_labels
- .slice(usize_range(edge_range))
+ .slice(usize_range(label_range))
.map(|r| r.decompress())
.flat_map(|block| block.labels.into_iter());
@@ -531,7 +534,7 @@ impl EdgeStore {
}
pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {
- let node_bytes = node.as_u64().to_le_bytes();
+ let node_bytes = node.as_u64().to_be_bytes();
match self.ranges.edges.get_raw(&node_bytes) {
Some(node_range_bytes) => {
@@ -572,7 +575,7 @@ impl EdgeStore {
pub fn iter_without_label(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {
self.ranges.edges.iter_raw().flat_map(move |(key, val)| {
- let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());
+ let node = u64::from_be_bytes((key.as_bytes()).try_into().unwrap());
let node = NodeID::from(node);
let edge_range = EdgeRange::deserialize(val.as_bytes());
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index b6588bcc..91e2a546 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -297,7 +297,7 @@ impl FinalEdgeStoreWriter {
}
self.hosts.insert(&node);
- let node_bytes = node.id.serialize_to_vec();
+ let node_bytes = node.id.as_u64().to_be_bytes().to_vec();
debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());
debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());
diff --git a/crates/core/src/webpage/safety_classifier.rs b/crates/core/src/webpage/safety_classifier.rs
index ceac0791..924f7ba8 100644
--- a/crates/core/src/webpage/safety_classifier.rs
+++ b/crates/core/src/webpage/safety_classifier.rs
@@ -187,7 +187,7 @@ impl Model {
.truncate(true)
.open(path)?;
- bincode::encode_into_std_write(&self, &mut file, bincode::config::standard())?;
+ bincode::encode_into_std_write(&self, &mut file, common::bincode_config())?;
file.flush()?;
Ok(())
@@ -197,7 +197,7 @@ impl Model {
let file = OpenOptions::new().read(true).open(path)?;
let mut reader = std::io::BufReader::new(file);
- let model = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;
+ let model = bincode::decode_from_std_read(&mut reader, common::bincode_config())?;
Ok(model)
}
diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.toml
index 961fe387..c883142b 100644
--- a/crates/file-store/Cargo.toml
+++ b/crates/file-store/Cargo.toml
@@ -9,5 +9,6 @@ version = "0.1.0"
[dependencies]
anyhow.workspace = true
bincode.workspace = true
+common.workspace = true
memmap2.workspace = true
ownedbytes.workspace = true
diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rs
index 4d343494..5f06828a 100644
--- a/crates/file-store/src/iterable.rs
+++ b/crates/file-store/src/iterable.rs
@@ -102,7 +102,7 @@ where
}
pub fn write(&mut self, item: &T) -> Result<WrittenOffset> {
- let serialized = bincode::encode_to_vec(item, bincode::config::standard())?;
+ let serialized = bincode::encode_to_vec(item, common::bincode_config())?;
let header = IterableHeader {
num_upcoming_bytes: serialized.len() as u64,
};
@@ -187,8 +187,7 @@ where
let serialized = &self.data[self.offset..self.offset + header.num_upcoming_bytes as usize];
self.offset += header.num_upcoming_bytes as usize;
- let (item, _) =
- bincode::decode_from_slice(serialized, bincode::config::standard()).unwrap();
+ let (item, _) = bincode::decode_from_slice(serialized, common::bincode_config()).unwrap();
Some(item)
}
diff --git a/crates/file-store/src/peekable.rs b/crates/file-store/src/peekable.rs
index 65aab11c..1509ef65 100644
--- a/crates/file-store/src/peekable.rs
+++ b/crates/file-store/src/peekable.rs
@@ -108,3 +108,15 @@ where
T: Eq,
{
}
+
+impl<I, T> std::fmt::Debug for Peekable<I>
+where
+ I: Iterator<Item = T>,
+ T: std::fmt::Debug,
+{
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Peekable")
+ .field("peeked", &self.peeked)
+ .finish()
+ }
+}
diff --git a/crates/leechy/src/conf.toml b/crates/leechy/src/conf.toml
index 1c492b01..a358d6b6 100644
--- a/crates/leechy/src/conf.toml
+++ b/crates/leechy/src/conf.toml
@@ -1,4 +1,4 @@
[[engines]]
name = "startpage"
-search = "https://www.startpage.com/search?q={query}&qsr=en_US"
+search = "https://www.startpage.com/sp/search?q={query}&qsr=en_US"
xpath = "./div[contains(@class, 'result')]/a"
diff --git a/crates/speedy-kv/Cargo.toml b/crates/speedy-kv/Cargo.toml
index 364195ea..01762f9c 100644
--- a/crates/speedy-kv/Cargo.toml
+++ b/crates/speedy-kv/Cargo.toml
@@ -1,16 +1,17 @@
[package]
-name = "speedy_kv"
-version = "0.1.0"
edition = "2021"
license = "AGPL-3.0"
+name = "speedy_kv"
+version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow.workspace = true
bincode.workspace = true
-bloom = { path = "../bloom" }
-file_store = { path = "../file-store" }
+bloom = {path = "../bloom"}
+common.workspace = true
+file_store = {path = "../file-store"}
fst.workspace = true
memmap2.workspace = true
serde.workspace = true
diff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rs
index 68cf8dd1..08a2c64e 100644
--- a/crates/speedy-kv/src/lib.rs
+++ b/crates/speedy-kv/src/lib.rs
@@ -446,11 +446,11 @@ where
V: bincode::Decode,
{
pub fn get(&self, key: &K) -> Result<Option<V>> {
- let key = bincode::encode_to_vec(key, bincode::config::standard())?;
+ let key = bincode::encode_to_vec(key, common::bincode_config())?;
match self.get_raw(key.as_slice()) {
Some(v) => {
- let (v, _) = bincode::decode_from_slice(v.as_bytes(), bincode::config::standard())?;
+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config())?;
Ok(Some(v))
}
None => Ok(None),
@@ -467,10 +467,8 @@ where
A: fst::Automaton + Clone + 'a,
{
self.search_raw(query).filter_map(|(k, v)| {
- let (k, _) =
- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;
- let (v, _) =
- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;
+ let (k, _) = bincode::decode_from_slice(k.as_bytes(), common::bincode_config()).ok()?;
+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config()).ok()?;
Some((k, v))
})
@@ -492,10 +490,8 @@ where
{
pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
self.iter_raw().filter_map(|(k, v)| {
- let (k, _) =
- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;
- let (v, _) =
- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;
+ let (k, _) = bincode::decode_from_slice(k.as_bytes(), common::bincode_config()).ok()?;
+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config()).ok()?;
Some((k, v))
})
@@ -503,10 +499,8 @@ where
pub fn sorted_iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
self.sorted_iter_raw().filter_map(|(k, v)| {
- let (k, _) =
- bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).ok()?;
- let (v, _) =
- bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).ok()?;
+ let (k, _) = bincode::decode_from_slice(k.as_bytes(), common::bincode_config()).ok()?;
+ let (v, _) = bincode::decode_from_slice(v.as_bytes(), common::bincode_config()).ok()?;
Some((k, v))
})
diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rs
index 458a55d2..196aacf3 100644
--- a/crates/speedy-kv/src/segment.rs
+++ b/crates/speedy-kv/src/segment.rs
@@ -79,7 +79,7 @@ where
self.writers.store.finish()?;
let mut wrt = BufWriter::new(self.writers.bloom);
- bincode::encode_into_std_write(self.bloom, &mut wrt, bincode::config::standard())?;
+ bincode::encode_into_std_write(self.bloom, &mut wrt, common::bincode_config())?;
wrt.flush()?;
Ok(())
@@ -117,7 +117,7 @@ impl<K, V> Segment<K, V> {
.read(true)
.open(folder.as_ref().join(Segment::<K, V>::bloom_file_name(uuid)))?;
- let bloom = bincode::decode_from_std_read(&mut bloom, bincode::config::standard())?;
+ let bloom = bincode::decode_from_std_read(&mut bloom, common::bincode_config())?;
Ok(Self {
uuid,
@@ -363,13 +363,17 @@ where
I: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,
{
pub fn new(segments: Vec<Peekable<I>>) -> Self {
- Self {
- segments: segments
- .into_iter()
- .enumerate()
- .map(|(segment_ord, iter)| SortedPeekable { segment_ord, iter })
- .collect(),
+ let mut heap = BinaryHeap::new();
+
+ for item in segments
+ .into_iter()
+ .enumerate()
+ .map(|(segment_ord, iter)| SortedPeekable { segment_ord, iter })
+ {
+ heap.push(item);
}
+
+ Self { segments: heap }
}
}
diff --git a/crates/speedy-kv/src/serialized.rs b/crates/speedy-kv/src/serialized.rs
index aef8e47f..38491db4 100644
--- a/crates/speedy-kv/src/serialized.rs
+++ b/crates/speedy-kv/src/serialized.rs
@@ -84,7 +84,7 @@ where
T: bincode::Encode,
{
pub fn new(value: &T) -> Result<Self> {
- let bytes = bincode::encode_to_vec(value, bincode::config::standard())?;
+ let bytes = bincode::encode_to_vec(value, common::bincode_config())?;
Ok(Self {
bytes,
_marker: std::marker::PhantomData,
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 26a27a30..a67d3659 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -13,10 +13,12 @@ param_grid = {
"verbosity": [-1],
"metric": ["ndcg"],
"ndcg_at": [[1, 2, 3, 5, 10]],
- "learning_rate": [0.025],
- "max_depth": [-1, 2, 4, 8],
- "num_leaves": [7, 15, 31],
- "lambda_l2": [0.1, 0.5, 1.0],
+ "learning_rate": [0.003],
+ # "max_depth": [-1, 2, 4, 8],
+ "max_depth": [-1],
+ # "num_leaves": [7, 15, 31],
+ "num_leaves": [31],
+ "lambda_l2": [1.0, 1.5, 2.0],
"linear_tree": [False],
}
@@ -149,10 +151,10 @@ for param in params:
scores = []
for metric, vals in res.items():
- if metric in ['cvbooster']:
+ if metric in ["cvbooster"]:
continue
- if 'stdv' in metric:
+ if "stdv" in metric:
continue
scores.append(vals[0])
@@ -188,7 +190,11 @@ print()
print("Feature importance:")
pprint(
sorted(
- [(id2feature[i], v) for i, v in enumerate(booster.feature_importance()) if v > 0],
+ [
+ (id2feature[i], v)
+ for i, v in enumerate(booster.feature_importance())
+ if v > 0
+ ],
key=lambda x: x[1],
reverse=True,
)
diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.py
index 6f38b471..cf975ffe 100644
--- a/ltr/leechy_annotate.py
+++ b/ltr/leechy_annotate.py
@@ -4,6 +4,7 @@ import stract
import numpy as np
from tqdm import tqdm
import time
+import sys
NUM_LABELS = 4
@@ -34,7 +35,16 @@ unannotated_queries = db.get_unannotated_queries()
eng = leechy.Engine()
for qid, query in tqdm(unannotated_queries.items()):
tqdm.write(query)
- leechy_results = eng.search(query)
+ leechy_results = []
+
+ for i in range(3):
+ leechy_results = eng.search(query)
+ if len(leechy_results) > 0:
+ break
+
+ if len(leechy_results) == 0:
+ tqdm.write(f"No results found for {query}")
+ sys.exit(-1)
for i, result in enumerate(leechy_results):
label = NUM_LABELS - int(np.log2(i + 1))
diff --git a/ltr/stract.py b/ltr/stract.py
index 25026f27..2617ffd8 100644
--- a/ltr/stract.py
+++ b/ltr/stract.py
@@ -4,6 +4,7 @@ import requests
API = "http://localhost:3000/beta/api/search"
NUM_RESULTS_PER_QUERY = 20
+
def simplify_snippet(snippet):
if "text" not in snippet:
return ""
@@ -15,7 +16,7 @@ def search(query, page=0):
payload = {
"query": query,
"numResults": NUM_RESULTS_PER_QUERY,
- 'page': page,
+ "page": page,
"returnRankingSignals": True,
}
@@ -29,4 +30,4 @@ def search(query, page=0):
for w in requests.post(API, json=payload).json()["webpages"][
:NUM_RESULTS_PER_QUERY
]
- ]
\ No newline at end of file
+ ]