Skip to content

web search done right

ability to convert some of the wander budget to scheduled urls

Mikkel Denkerefa6f9fab051
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 4bcd11ae..e0a6d564 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -19,6 +19,7 @@ use futures::StreamExt;
use indicatif::ProgressIterator;
use itertools::Itertools;
use rayon::{prelude::*, ThreadPoolBuilder};
+use rustc_hash::FxHashSet;
use std::cmp::Reverse;
use std::collections::BTreeMap;
use std::hash::{Hash, Hasher};
@@ -48,6 +49,7 @@ use super::Domain;
const MAX_UNCOMMITTED_INSERTS_PER_GROUP: usize = 50_000;
const NUM_GROUPS: usize = 1024;
const CONCURRENCY_LIMIT: usize = 32;
+const CONVERT_HIGH_WANDER_BUDGET: bool = false;
#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, Hash)]
struct StoredUrl(#[bincode(with_serde)] Url);
@@ -203,7 +205,7 @@ impl CrawlPlanner {
.map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default())
.sum();
- let wander_budget = ((wander_budget * host_centrality) / total_host_centralities)
+ let mut wander_budget = ((wander_budget * host_centrality) / total_host_centralities)
.max(1.0)
.round() as u64;
@@ -225,9 +227,56 @@ impl CrawlPlanner {
})
.collect();
- urls.sort_by(|a, b| b.weight.total_cmp(&a.weight));
+ urls.sort_by(|a, b| a.weight.total_cmp(&b.weight));
urls.reverse();
+ if CONVERT_HIGH_WANDER_BUDGET {
+ // convert some of the wander budget to scheduled urls if the wander budget is too large
+ if wander_budget > urls.len() as u64 {
+ // we should maybe convert the budget on a per-host basis
+ // instead of aggregating all hosts for the domain. some blog sites
+ // like wordpress/blogspot/github.io etc. will have lots of hosts,
+ // and if we simply convert the budget for the domain, we might end up
+ // with a lot of urls from the same host.
+
+ let mut bloom = bloom::U64BloomFilter::new(urls.len() as u64, 0.05);
+ let mut hosts = FxHashSet::default();
+ for url in &urls {
+ let node = Node::from(url.url.clone());
+ bloom.insert(node.id().as_u64());
+ hosts.insert(node.into_host().id());
+ }
+
+ let budget_to_convert = wander_budget - urls.len() as u64;
+
+ let hosts = hosts.into_iter().collect::<Vec<_>>();
+ let nodes: Vec<_> = crate::block_on(self.page_graph.pages_by_hosts(&hosts))
+ .unwrap_or_default()
+ .into_iter()
+ .filter(|node| !bloom.contains(node.as_u64()))
+ .take(budget_to_convert as usize)
+ .collect();
+
+ let nodes =
+ crate::block_on(self.page_graph.batch_get_node(&nodes)).unwrap_or_default();
+
+ let mut new_scheduled = 0;
+ for node in nodes.into_iter().flatten() {
+ if let Ok(url) = Url::parse(&format!("https://{}", node.as_str())) {
+ urls.push(WeightedUrl { url, weight: 0.0 });
+ new_scheduled += 1;
+ }
+ }
+
+ tracing::info!(
+ "converted {} wander budget to scheduled urls",
+ new_scheduled
+ );
+
+ wander_budget -= new_scheduled as u64;
+ }
+ }
+
let job = Job {
domain: domain.clone(),
urls: urls.into_iter().collect(),
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 26cdff16..27a1f47d 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -17,6 +17,7 @@
use std::net::SocketAddr;
use std::sync::Arc;
+use itertools::Itertools;
use tracing::info;
use utoipa::ToSchema;
@@ -56,11 +57,12 @@ sonic_service!(
RawIngoingEdges,
RawOutgoingEdges,
RawIngoingEdgesWithLabels,
- RawOutgoingEdgesWithLabels
+ RawOutgoingEdgesWithLabels,
+ PagesByHosts
]
);
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct GetNode {
pub node: NodeID,
}
@@ -73,7 +75,7 @@ impl Message<WebGraphService> for GetNode {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct IngoingEdges {
pub node: Node,
pub limit: EdgeLimit,
@@ -87,7 +89,7 @@ impl Message<WebGraphService> for IngoingEdges {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct OutgoingEdges {
pub node: Node,
pub limit: EdgeLimit,
@@ -101,7 +103,7 @@ impl Message<WebGraphService> for OutgoingEdges {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct RawIngoingEdges {
pub node: NodeID,
pub limit: EdgeLimit,
@@ -115,7 +117,7 @@ impl Message<WebGraphService> for RawIngoingEdges {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct RawOutgoingEdges {
pub node: NodeID,
pub limit: EdgeLimit,
@@ -129,7 +131,7 @@ impl Message<WebGraphService> for RawOutgoingEdges {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct RawIngoingEdgesWithLabels {
pub node: NodeID,
pub limit: EdgeLimit,
@@ -145,7 +147,7 @@ impl Message<WebGraphService> for RawIngoingEdgesWithLabels {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct RawOutgoingEdgesWithLabels {
pub node: NodeID,
pub limit: EdgeLimit,
@@ -161,6 +163,23 @@ impl Message<WebGraphService> for RawOutgoingEdgesWithLabels {
}
}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct PagesByHosts {
+ pub hosts: Vec<NodeID>,
+}
+
+impl Message<WebGraphService> for PagesByHosts {
+ type Response = Vec<NodeID>;
+
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
+ self.hosts
+ .iter()
+ .flat_map(|host| server.graph.pages_by_host(host))
+ .unique()
+ .collect()
+ }
+}
+
pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
let addr: SocketAddr = config.host;
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index aeb45518..ecc7b2eb 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -33,8 +33,8 @@ use crate::{
},
},
entrypoint::webgraph_server::{
- GetNode, IngoingEdges, OutgoingEdges, RawIngoingEdges, RawIngoingEdgesWithLabels,
- RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,
+ GetNode, IngoingEdges, OutgoingEdges, PagesByHosts, RawIngoingEdges,
+ RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,
},
Result,
};
@@ -394,4 +394,27 @@ impl RemoteWebgraph {
Ok(edges)
}
+
+ pub async fn pages_by_hosts(&self, hosts: &[NodeID]) -> Result<Vec<NodeID>> {
+ let res = self
+ .conn()
+ .await
+ .send(
+ PagesByHosts {
+ hosts: hosts.to_vec(),
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await?;
+
+ Ok(res
+ .into_iter()
+ .flat_map(|(_, reps)| {
+ debug_assert!(reps.len() <= 1);
+ reps.into_iter().flat_map(|(_, rep)| rep)
+ })
+ .unique()
+ .collect())
+ }
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 5bb44797..948f1e80 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -317,7 +317,7 @@ impl EdgeStore {
let edge_labels = store
.edge_labels
.slice(usize_range(node.labels()))
- .map(|r| r.unwrap().decompress())
+ .map(|r| r.decompress())
.flat_map(|block| block.labels.into_iter());
edges.push(
@@ -333,17 +333,17 @@ impl EdgeStore {
fn merge_postings<P: AsRef<Path>>(
stores: &[EdgeStore],
label_compression: Compression,
- path: P,
+ folder: P,
) -> Result<Self> {
let reversed = stores[0].reversed;
- let mut ranges = RangesDb::open(path.as_ref().join("ranges"));
+ let mut ranges = RangesDb::open(folder.as_ref().join("ranges"));
let edge_labels_file = File::options()
.read(true)
.create(true)
.truncate(false)
.write(true)
- .open(path.as_ref().join("labels"))
+ .open(folder.as_ref().join("labels"))
.unwrap();
let mut edge_labels = IterableStoreWriter::new(edge_labels_file);
@@ -352,7 +352,7 @@ impl EdgeStore {
.create(true)
.truncate(false)
.write(true)
- .open(path.as_ref().join("nodes"))
+ .open(folder.as_ref().join("nodes"))
.unwrap();
let mut edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);
@@ -440,7 +440,7 @@ impl EdgeStore {
edge_labels.flush().unwrap();
}
- Ok(Self::open(path, reversed))
+ Ok(Self::open(folder, reversed))
}
pub fn merge<P: AsRef<Path>>(
@@ -486,7 +486,7 @@ impl EdgeStore {
let edge_labels = self
.edge_labels
.slice(usize_range(edge_range))
- .map(|r| r.unwrap().decompress())
+ .map(|r| r.decompress())
.flat_map(|block| block.labels.into_iter());
let edge_labels = limit.apply(edge_labels);
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index 84f3d0d7..c84e7210 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -183,7 +183,7 @@ impl EdgeStoreWriter {
.iter()
.map(|p| IterableStoreReader::open(p).unwrap())
.collect();
- let file_reader = SortedIterableStoreReader::new(readers).map(|r| r.unwrap());
+ let file_reader = SortedIterableStoreReader::new(readers);
let edges = std::mem::take(&mut self.edges);
@@ -212,7 +212,9 @@ impl EdgeStoreWriter {
impl Drop for EdgeStoreWriter {
fn drop(&mut self) {
- std::fs::remove_dir_all(self.path.join("writer")).unwrap();
+ if self.path.join("writer").exists() {
+ std::fs::remove_dir_all(self.path.join("writer")).unwrap();
+ }
}
}
diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rs
index e0afde60..009f47e9 100644
--- a/crates/file-store/src/iterable.rs
+++ b/crates/file-store/src/iterable.rs
@@ -27,6 +27,7 @@
use crate::{owned_bytes::OwnedBytes, ConstSerializable, Result};
use std::{
+ cmp::Reverse,
io::{self, Write},
ops::Range,
path::Path,
@@ -167,7 +168,7 @@ impl<T> Iterator for IterableStoreReader<T>
where
T: bincode::Decode,
{
- type Item = Result<T>;
+ type Item = T;
fn next(&mut self) -> Option<Self::Item> {
if self.offset + IterableHeader::serialized_size() > self.data.len() {
@@ -185,13 +186,10 @@ where
let serialized = &self.data[self.offset..self.offset + header.num_upcoming_bytes as usize];
self.offset += header.num_upcoming_bytes as usize;
+ let (item, _) =
+ bincode::decode_from_slice(serialized, bincode::config::standard()).unwrap();
- Some(
- match bincode::decode_from_slice(serialized, bincode::config::standard()) {
- Ok((item, _)) => Ok(item),
- Err(err) => Err(err.into()),
- },
- )
+ Some(item)
}
}
@@ -213,11 +211,13 @@ impl<T> io::Seek for IterableStoreReader<T> {
}
}
+type MinHeap<T> = std::collections::BinaryHeap<Reverse<T>>;
+
pub struct SortedIterableStoreReader<T>
where
T: bincode::Decode,
{
- readers: Vec<Peekable<IterableStoreReader<T>>>,
+ readers: MinHeap<Peekable<IterableStoreReader<T>>>,
}
impl<T> SortedIterableStoreReader<T>
@@ -225,7 +225,11 @@ where
T: Ord + bincode::Decode,
{
pub fn new(readers: Vec<IterableStoreReader<T>>) -> Self {
- let readers = readers.into_iter().map(Peekable::new).collect::<Vec<_>>();
+ let readers = readers
+ .into_iter()
+ .map(Peekable::new)
+ .map(Reverse)
+ .collect();
Self { readers }
}
@@ -235,41 +239,10 @@ impl<T> Iterator for SortedIterableStoreReader<T>
where
T: Ord + bincode::Decode,
{
- type Item = Result<T>;
+ type Item = T;
fn next(&mut self) -> Option<Self::Item> {
- let mut min_index = None;
-
- let num_readers = self.readers.len();
- for i in 0..num_readers {
- let reader = &self.readers[i];
- if let Some(item) = reader.peek() {
- match item {
- Ok(item) => match min_index {
- Some(cur_min) => {
- let cur_min_reader: &Peekable<IterableStoreReader<T>> =
- &self.readers[cur_min];
-
- match cur_min_reader.peek().unwrap().as_ref() {
- Ok(cur_min_item) => {
- if item < cur_min_item {
- min_index = Some(i);
- }
- }
- Err(err) => return Some(Err(anyhow::anyhow!("{err}"))),
- }
- }
- None => min_index = Some(i),
- },
- Err(err) => return Some(Err(anyhow::anyhow!("{err}"))),
- }
- }
- }
-
- match min_index {
- Some(min_index) => Some(self.readers[min_index].next().unwrap()),
- None => None,
- }
+ self.readers.peek_mut().and_then(|mut item| item.0.next())
}
}
@@ -412,7 +385,7 @@ mod tests {
let reader = IterableStoreReader::from_bytes(writer);
- let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();
+ let items: Vec<i32> = reader.collect();
assert_eq!(items, vec![1, 2, 3]);
}
@@ -436,7 +409,7 @@ mod tests {
let reader = SortedIterableStoreReader::new(vec![reader1, reader2]);
- let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();
+ let items: Vec<i32> = reader.collect();
assert_eq!(items, vec![1, 2, 3, 4, 5, 6]);
}