web search done right
ability to convert some of the wander budget to scheduled urls
Mikkel Denkerefa6f9fab051
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 4bcd11ae..e0a6d564 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -19,6 +19,7 @@ use futures::StreamExt;use indicatif::ProgressIterator;use itertools::Itertools;use rayon::{prelude::*, ThreadPoolBuilder};+use rustc_hash::FxHashSet;use std::cmp::Reverse;use std::collections::BTreeMap;use std::hash::{Hash, Hasher};@@ -48,6 +49,7 @@ use super::Domain;const MAX_UNCOMMITTED_INSERTS_PER_GROUP: usize = 50_000;const NUM_GROUPS: usize = 1024;const CONCURRENCY_LIMIT: usize = 32;+const CONVERT_HIGH_WANDER_BUDGET: bool = false;#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, Hash)]struct StoredUrl(#[bincode(with_serde)] Url);@@ -203,7 +205,7 @@ impl CrawlPlanner {.map(|id| self.host_centrality.get(&id).unwrap().unwrap_or_default()).sum();- let wander_budget = ((wander_budget * host_centrality) / total_host_centralities)+ let mut wander_budget = ((wander_budget * host_centrality) / total_host_centralities).max(1.0).round() as u64;@@ -225,9 +227,56 @@ impl CrawlPlanner {}).collect();- urls.sort_by(|a, b| b.weight.total_cmp(&a.weight));+ urls.sort_by(|a, b| a.weight.total_cmp(&b.weight));urls.reverse();+ if CONVERT_HIGH_WANDER_BUDGET {+ // convert some of the wander budget to scheduled urls if the wander budget is too large+ if wander_budget > urls.len() as u64 {+ // we should maybe convert the budget on a per-host basis+ // instead of aggregating all hosts for the domain. some blog sites+ // like wordpress/blogspot/github.io etc. will have lots of hosts,+ // and if we simply convert the budget for the domain, we might end up+ // with a lot of urls from the same host.++ let mut bloom = bloom::U64BloomFilter::new(urls.len() as u64, 0.05);+ let mut hosts = FxHashSet::default();+ for url in &urls {+ let node = Node::from(url.url.clone());+ bloom.insert(node.id().as_u64());+ hosts.insert(node.into_host().id());+ }++ let budget_to_convert = wander_budget - urls.len() as u64;++ let hosts = hosts.into_iter().collect::<Vec<_>>();+ let nodes: Vec<_> = crate::block_on(self.page_graph.pages_by_hosts(&hosts))+ .unwrap_or_default()+ .into_iter()+ .filter(|node| !bloom.contains(node.as_u64()))+ .take(budget_to_convert as usize)+ .collect();++ let nodes =+ crate::block_on(self.page_graph.batch_get_node(&nodes)).unwrap_or_default();++ let mut new_scheduled = 0;+ for node in nodes.into_iter().flatten() {+ if let Ok(url) = Url::parse(&format!("https://{}", node.as_str())) {+ urls.push(WeightedUrl { url, weight: 0.0 });+ new_scheduled += 1;+ }+ }++ tracing::info!(+ "converted {} wander budget to scheduled urls",+ new_scheduled+ );++ wander_budget -= new_scheduled as u64;+ }+ }+let job = Job {domain: domain.clone(),urls: urls.into_iter().collect(),diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 26cdff16..27a1f47d 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -17,6 +17,7 @@use std::net::SocketAddr;use std::sync::Arc;+use itertools::Itertools;use tracing::info;use utoipa::ToSchema;@@ -56,11 +57,12 @@ sonic_service!(RawIngoingEdges,RawOutgoingEdges,RawIngoingEdgesWithLabels,- RawOutgoingEdgesWithLabels+ RawOutgoingEdgesWithLabels,+ PagesByHosts]);-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct GetNode {pub node: NodeID,}@@ -73,7 +75,7 @@ impl Message<WebGraphService> for GetNode {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct IngoingEdges {pub node: Node,pub limit: EdgeLimit,@@ -87,7 +89,7 @@ impl Message<WebGraphService> for IngoingEdges {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct OutgoingEdges {pub node: Node,pub limit: EdgeLimit,@@ -101,7 +103,7 @@ impl Message<WebGraphService> for OutgoingEdges {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct RawIngoingEdges {pub node: NodeID,pub limit: EdgeLimit,@@ -115,7 +117,7 @@ impl Message<WebGraphService> for RawIngoingEdges {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct RawOutgoingEdges {pub node: NodeID,pub limit: EdgeLimit,@@ -129,7 +131,7 @@ impl Message<WebGraphService> for RawOutgoingEdges {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct RawIngoingEdgesWithLabels {pub node: NodeID,pub limit: EdgeLimit,@@ -145,7 +147,7 @@ impl Message<WebGraphService> for RawIngoingEdgesWithLabels {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct RawOutgoingEdgesWithLabels {pub node: NodeID,pub limit: EdgeLimit,@@ -161,6 +163,23 @@ impl Message<WebGraphService> for RawOutgoingEdgesWithLabels {}}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct PagesByHosts {+ pub hosts: Vec<NodeID>,+}++impl Message<WebGraphService> for PagesByHosts {+ type Response = Vec<NodeID>;++ async fn handle(self, server: &WebGraphService) -> Self::Response {+ self.hosts+ .iter()+ .flat_map(|host| server.graph.pages_by_host(host))+ .unique()+ .collect()+ }+}+pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {let addr: SocketAddr = config.host;diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex aeb45518..ecc7b2eb 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -33,8 +33,8 @@ use crate::{},},entrypoint::webgraph_server::{- GetNode, IngoingEdges, OutgoingEdges, RawIngoingEdges, RawIngoingEdgesWithLabels,- RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,+ GetNode, IngoingEdges, OutgoingEdges, PagesByHosts, RawIngoingEdges,+ RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,},Result,};@@ -394,4 +394,27 @@ impl RemoteWebgraph {Ok(edges)}++ pub async fn pages_by_hosts(&self, hosts: &[NodeID]) -> Result<Vec<NodeID>> {+ let res = self+ .conn()+ .await+ .send(+ PagesByHosts {+ hosts: hosts.to_vec(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .unique()+ .collect())+ }}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 5bb44797..948f1e80 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -317,7 +317,7 @@ impl EdgeStore {let edge_labels = store.edge_labels.slice(usize_range(node.labels()))- .map(|r| r.unwrap().decompress())+ .map(|r| r.decompress()).flat_map(|block| block.labels.into_iter());edges.push(@@ -333,17 +333,17 @@ impl EdgeStore {fn merge_postings<P: AsRef<Path>>(stores: &[EdgeStore],label_compression: Compression,- path: P,+ folder: P,) -> Result<Self> {let reversed = stores[0].reversed;- let mut ranges = RangesDb::open(path.as_ref().join("ranges"));+ let mut ranges = RangesDb::open(folder.as_ref().join("ranges"));let edge_labels_file = File::options().read(true).create(true).truncate(false).write(true)- .open(path.as_ref().join("labels"))+ .open(folder.as_ref().join("labels")).unwrap();let mut edge_labels = IterableStoreWriter::new(edge_labels_file);@@ -352,7 +352,7 @@ impl EdgeStore {.create(true).truncate(false).write(true)- .open(path.as_ref().join("nodes"))+ .open(folder.as_ref().join("nodes")).unwrap();let mut edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);@@ -440,7 +440,7 @@ impl EdgeStore {edge_labels.flush().unwrap();}- Ok(Self::open(path, reversed))+ Ok(Self::open(folder, reversed))}pub fn merge<P: AsRef<Path>>(@@ -486,7 +486,7 @@ impl EdgeStore {let edge_labels = self.edge_labels.slice(usize_range(edge_range))- .map(|r| r.unwrap().decompress())+ .map(|r| r.decompress()).flat_map(|block| block.labels.into_iter());let edge_labels = limit.apply(edge_labels);diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex 84f3d0d7..c84e7210 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -183,7 +183,7 @@ impl EdgeStoreWriter {.iter().map(|p| IterableStoreReader::open(p).unwrap()).collect();- let file_reader = SortedIterableStoreReader::new(readers).map(|r| r.unwrap());+ let file_reader = SortedIterableStoreReader::new(readers);let edges = std::mem::take(&mut self.edges);@@ -212,7 +212,9 @@ impl EdgeStoreWriter {impl Drop for EdgeStoreWriter {fn drop(&mut self) {- std::fs::remove_dir_all(self.path.join("writer")).unwrap();+ if self.path.join("writer").exists() {+ std::fs::remove_dir_all(self.path.join("writer")).unwrap();+ }}}diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rsindex e0afde60..009f47e9 100644--- a/crates/file-store/src/iterable.rs+++ b/crates/file-store/src/iterable.rs@@ -27,6 +27,7 @@use crate::{owned_bytes::OwnedBytes, ConstSerializable, Result};use std::{+ cmp::Reverse,io::{self, Write},ops::Range,path::Path,@@ -167,7 +168,7 @@ impl<T> Iterator for IterableStoreReader<T>whereT: bincode::Decode,{- type Item = Result<T>;+ type Item = T;fn next(&mut self) -> Option<Self::Item> {if self.offset + IterableHeader::serialized_size() > self.data.len() {@@ -185,13 +186,10 @@ wherelet serialized = &self.data[self.offset..self.offset + header.num_upcoming_bytes as usize];self.offset += header.num_upcoming_bytes as usize;+ let (item, _) =+ bincode::decode_from_slice(serialized, bincode::config::standard()).unwrap();- Some(- match bincode::decode_from_slice(serialized, bincode::config::standard()) {- Ok((item, _)) => Ok(item),- Err(err) => Err(err.into()),- },- )+ Some(item)}}@@ -213,11 +211,13 @@ impl<T> io::Seek for IterableStoreReader<T> {}}+type MinHeap<T> = std::collections::BinaryHeap<Reverse<T>>;+pub struct SortedIterableStoreReader<T>whereT: bincode::Decode,{- readers: Vec<Peekable<IterableStoreReader<T>>>,+ readers: MinHeap<Peekable<IterableStoreReader<T>>>,}impl<T> SortedIterableStoreReader<T>@@ -225,7 +225,11 @@ whereT: Ord + bincode::Decode,{pub fn new(readers: Vec<IterableStoreReader<T>>) -> Self {- let readers = readers.into_iter().map(Peekable::new).collect::<Vec<_>>();+ let readers = readers+ .into_iter()+ .map(Peekable::new)+ .map(Reverse)+ .collect();Self { readers }}@@ -235,41 +239,10 @@ impl<T> Iterator for SortedIterableStoreReader<T>whereT: Ord + bincode::Decode,{- type Item = Result<T>;+ type Item = T;fn next(&mut self) -> Option<Self::Item> {- let mut min_index = None;-- let num_readers = self.readers.len();- for i in 0..num_readers {- let reader = &self.readers[i];- if let Some(item) = reader.peek() {- match item {- Ok(item) => match min_index {- Some(cur_min) => {- let cur_min_reader: &Peekable<IterableStoreReader<T>> =- &self.readers[cur_min];-- match cur_min_reader.peek().unwrap().as_ref() {- Ok(cur_min_item) => {- if item < cur_min_item {- min_index = Some(i);- }- }- Err(err) => return Some(Err(anyhow::anyhow!("{err}"))),- }- }- None => min_index = Some(i),- },- Err(err) => return Some(Err(anyhow::anyhow!("{err}"))),- }- }- }-- match min_index {- Some(min_index) => Some(self.readers[min_index].next().unwrap()),- None => None,- }+ self.readers.peek_mut().and_then(|mut item| item.0.next())}}@@ -412,7 +385,7 @@ mod tests {let reader = IterableStoreReader::from_bytes(writer);- let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();+ let items: Vec<i32> = reader.collect();assert_eq!(items, vec![1, 2, 3]);}@@ -436,7 +409,7 @@ mod tests {let reader = SortedIterableStoreReader::new(vec![reader1, reader2]);- let items: Vec<i32> = reader.map(|item| item.unwrap()).collect();+ let items: Vec<i32> = reader.collect();assert_eq!(items, vec![1, 2, 3, 4, 5, 6]);}