Skip to content

web search done right

forgot some files

Mikkel Denkerd07d113ad19c
diff --git a/configs/crawler/planner.toml b/configs/crawler/planner.toml
new file mode 100644
index 00000000..cf2824be
--- /dev/null
+++ b/configs/crawler/planner.toml
@@ -0,0 +1,11 @@
+host_graph_path = "data/webgraph_host"
+host_harmonic_path = "data/centrality/harmonic"
+output_path = "data/crawlplan"
+page_graph_path = "data/webgraph_page"
+page_harmonic_path = "data/centrality_page/derived_harmonic"
+
+num_job_queues = 5
+
+crawl_budget = 1_000
+top_host_fraction = 0.01
+wander_fraction = 0.2
diff --git a/core/src/crawler/file_queue.rs b/core/src/crawler/file_queue.rs
new file mode 100644
index 00000000..92883dae
--- /dev/null
+++ b/core/src/crawler/file_queue.rs
@@ -0,0 +1,220 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::Result;
+use memmap2::Mmap;
+use std::{
+ fs::File,
+ io::{BufWriter, Read, Seek, Write},
+ path::PathBuf,
+};
+
+const POINTER_KEY: &str = "pointer";
+const DATA_KEY: &str = "data";
+
+#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)]
+struct Header {
+ body_size: usize,
+}
+
+pub struct FileQueueWriter<T> {
+ path: PathBuf,
+ writer: BufWriter<File>,
+ _marker: std::marker::PhantomData<T>,
+}
+
+impl<T> FileQueueWriter<T>
+where
+ T: serde::Serialize + serde::de::DeserializeOwned,
+{
+ pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(path.as_ref())?;
+ }
+
+ let file = std::fs::OpenOptions::new()
+ .create(true)
+ .write(true)
+ .read(true)
+ .open(path.as_ref().join(DATA_KEY))?;
+
+ Ok(Self {
+ path: path.as_ref().to_path_buf(),
+ writer: BufWriter::new(file),
+ _marker: std::marker::PhantomData,
+ })
+ }
+
+ pub fn push(&mut self, item: T) -> Result<()> {
+ let body = bincode::serialize(&item)?;
+ let header = Header {
+ body_size: body.len(),
+ };
+
+ let header_bytes = bincode::serialize(&header)?;
+
+ self.writer.write_all(&header_bytes)?;
+ self.writer.write_all(&body)?;
+
+ Ok(())
+ }
+
+ pub fn flush(&mut self) -> Result<()> {
+ self.writer.flush()?;
+
+ Ok(())
+ }
+
+ pub fn finalize(mut self) -> Result<FileQueue<T>> {
+ self.flush()?;
+
+ let file = self.writer.into_inner()?;
+
+ Ok(FileQueue {
+ pointer: FilePointer::new(self.path)?,
+ file: unsafe { Mmap::map(&file)? },
+ _marker: std::marker::PhantomData,
+ })
+ }
+}
+
+struct FilePointer {
+ file: File,
+}
+
+impl FilePointer {
+ fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(path.as_ref())?;
+ }
+
+ let file = std::fs::OpenOptions::new()
+ .create(true)
+ .write(true)
+ .read(true)
+ .open(path.as_ref().join(POINTER_KEY))?;
+
+ Ok(Self { file })
+ }
+
+ fn get(&mut self) -> usize {
+ self.file.seek(std::io::SeekFrom::Start(0)).unwrap();
+ let mut buf = [0u8; 8];
+
+ if self.file.read_exact(&mut buf).is_err() {
+ return 0;
+ }
+
+ usize::from_be_bytes(buf)
+ }
+
+ fn set(&mut self, pointer: usize) -> Result<()> {
+ self.file.seek(std::io::SeekFrom::Start(0))?;
+ self.file.write_all(&pointer.to_be_bytes())?;
+ self.file.flush()?;
+
+ Ok(())
+ }
+}
+
+pub struct FileQueue<T> {
+ pointer: FilePointer,
+ file: Mmap,
+ _marker: std::marker::PhantomData<T>,
+}
+
+impl<T> FileQueue<T>
+where
+ T: serde::Serialize + serde::de::DeserializeOwned,
+{
+ pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(path.as_ref())?;
+ }
+
+ let file = File::open(path.as_ref().join(DATA_KEY))?;
+ let file = unsafe { Mmap::map(&file)? };
+
+ Ok(Self {
+ pointer: FilePointer::new(path)?,
+ file,
+ _marker: std::marker::PhantomData,
+ })
+ }
+
+ pub fn pop(&mut self) -> Result<Option<T>> {
+ let cur_pointer = self.pointer.get();
+
+ if cur_pointer >= self.file.len() {
+ return Ok(None);
+ }
+
+ let header_size = std::mem::size_of::<Header>();
+ let header: Header =
+ bincode::deserialize(&self.file[cur_pointer..cur_pointer + header_size])?;
+
+ let body =
+ &self.file[cur_pointer + header_size..cur_pointer + header_size + header.body_size];
+ let item = bincode::deserialize(body)?;
+
+ self.pointer
+ .set(cur_pointer + header_size + header.body_size)?;
+
+ Ok(Some(item))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use proptest::prelude::*;
+
+ #[test]
+ fn simple() {
+ let mut writer = FileQueueWriter::new(crate::gen_temp_path()).unwrap();
+
+ writer.push("Hello".to_string()).unwrap();
+ writer.push("World".to_string()).unwrap();
+
+ let mut queue = writer.finalize().unwrap();
+
+ assert_eq!(queue.pop().unwrap().unwrap(), "Hello");
+ assert_eq!(queue.pop().unwrap().unwrap(), "World");
+ assert_eq!(queue.pop().unwrap(), None);
+ }
+
+ proptest! {
+ #[test]
+ fn prop(data: Vec<String>) {
+ let expected = data.clone();
+
+ let mut writer = FileQueueWriter::new(crate::gen_temp_path()).unwrap();
+
+ for item in data {
+ writer.push(item).unwrap();
+ }
+
+ let mut queue = writer.finalize().unwrap();
+ let mut actual = Vec::new();
+
+ while let Some(item) = queue.pop().unwrap() {
+ actual.push(item);
+ }
+
+ prop_assert_eq!(actual, expected);
+ }
+ }
+}
diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rs
new file mode 100644
index 00000000..1a5c49f5
--- /dev/null
+++ b/core/src/crawler/planner.rs
@@ -0,0 +1,230 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use anyhow::{anyhow, Result};
+use rayon::prelude::*;
+use std::collections::HashMap;
+use std::{
+ collections::VecDeque,
+ path::Path,
+ sync::{atomic::AtomicUsize, Mutex},
+};
+use url::Url;
+
+use crate::{
+ config::CrawlPlannerConfig,
+ crawler::{file_queue::FileQueueWriter, Job},
+ kv::{rocksdb_store::RocksDbStore, Kv},
+ webgraph::{NodeID, Webgraph},
+};
+
+use super::Domain;
+
+fn top_hosts(host_centrality: &RocksDbStore<NodeID, f64>, fraction: f64) -> Vec<NodeID> {
+ let mut hosts = host_centrality
+ .iter()
+ .map(|(id, centrality)| (id, centrality))
+ .map(|(id, centrality)| {
+ if !centrality.is_finite() {
+ (id, 0.0)
+ } else {
+ (id, centrality)
+ }
+ })
+ .collect::<Vec<_>>();
+
+ hosts.sort_by(|(_, a), (_, b)| b.total_cmp(a));
+
+ let num_hosts = (hosts.len() as f64 * fraction) as usize;
+
+ hosts
+ .into_iter()
+ .take(num_hosts)
+ .map(|(id, _)| id)
+ .collect()
+}
+
+fn all_pages(
+ page_centrality: &RocksDbStore<NodeID, f64>,
+ page_graph: &Webgraph,
+ host: NodeID,
+) -> Vec<(NodeID, f64)> {
+ page_graph
+ .raw_ingoing_edges_by_host(&host)
+ .into_iter()
+ .map(|e| e.to)
+ .map(|id| (id, page_centrality.get(&id).unwrap_or_default()))
+ .collect::<Vec<_>>()
+}
+
+fn group_domain(hosts: Vec<NodeID>, host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {
+ let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();
+
+ for host in hosts {
+ let node = host_graph.id2node(&host).unwrap();
+ if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {
+ let domain = Domain::from(url);
+ domains.entry(domain).or_default().push(host);
+ }
+ }
+
+ domains
+}
+
+fn check_config(config: &CrawlPlannerConfig) -> Result<()> {
+ if !(0.0..=1.0).contains(&config.wander_fraction) {
+ return Err(anyhow::anyhow!(
+ "top_host_fraction must be in range [0.0, 1.0]"
+ ));
+ }
+
+ if !(0.0..=1.0).contains(&config.top_host_fraction) {
+ return Err(anyhow::anyhow!(
+ "top_host_fraction must be in range [0.0, 1.0]"
+ ));
+ }
+
+ Ok(())
+}
+
+pub fn make_crawl_plan<P: AsRef<Path>>(
+ host_centrality: RocksDbStore<NodeID, f64>,
+ page_centrality: RocksDbStore<NodeID, f64>,
+ host_graph: Webgraph,
+ page_graph: Webgraph,
+ config: CrawlPlannerConfig,
+ output: P,
+) -> Result<()> {
+ check_config(&config)?;
+
+ if output.as_ref().exists() {
+ return Err(anyhow!("output path already exists"));
+ }
+
+ let queue_path = output.as_ref().join("job_queue");
+ std::fs::create_dir_all(&queue_path)?;
+
+ let hosts = top_hosts(&host_centrality, config.top_host_fraction);
+ tracing::debug!("generating for {} hosts", hosts.len());
+ let grouped = group_domain(hosts, &host_graph);
+
+ let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)
+ .map(|i| {
+ let path = queue_path.join(format!("{}.queue", i));
+ FileQueueWriter::new(path)
+ })
+ .collect::<Result<Vec<_>>>()?
+ .into_iter()
+ .map(Mutex::new)
+ .collect();
+
+ let stats = Mutex::new(Vec::new());
+
+ let next_queue = AtomicUsize::new(0);
+
+ grouped.into_par_iter().for_each(|(domain, hosts)| {
+ let mut total_wander_budget = 0;
+ let mut total_schedule_budget = 0;
+ let mut total_scheduled_urls = 0;
+ let mut urls = VecDeque::new();
+
+ for host in hosts {
+ let mut pages = all_pages(&page_centrality, &page_graph, host);
+ pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));
+ tracing::debug!("num pages: {}", pages.len());
+ let host_centrality = host_centrality.get(&host).unwrap_or_default();
+
+ let host_budget = (config.crawl_budget as f64 * host_centrality).max(1.0) as u64;
+ tracing::debug!("host_budget: {host_budget}");
+ let schedule_budget =
+ (host_budget as f64 * (1.0 - config.wander_fraction)).max(1.0) as u64;
+ tracing::debug!("schedule_budget: {schedule_budget}");
+ let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0) as u64;
+ tracing::debug!("wander_budget: {wander_budget}");
+
+ total_wander_budget += wander_budget;
+ total_schedule_budget += schedule_budget;
+
+ let before = urls.len();
+ urls.extend(
+ pages
+ .into_iter()
+ .map(|(id, _)| id)
+ .filter_map(|id| page_graph.id2node(&id))
+ .map(|n| n.name)
+ .filter_map(|n| Url::parse(&format!("http://{n}")).ok())
+ .take(schedule_budget as usize),
+ );
+
+ total_scheduled_urls += urls.len() as u64 - before as u64;
+ }
+
+ let job = Job {
+ domain: domain.clone(),
+ urls,
+ wandering_urls: total_wander_budget,
+ };
+
+ let domain_stats = DomainStats {
+ domain,
+ schedule_budget: total_schedule_budget,
+ wander_budget: total_wander_budget,
+ scheduled_urls: total_scheduled_urls,
+ };
+ stats
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(domain_stats);
+
+ // `fetch_add` wraps around on overflow
+ let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ job_queues[queue_index % job_queues.len()]
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .push(job)
+ .unwrap();
+ });
+
+ for queue in job_queues {
+ queue
+ .into_inner()
+ .unwrap_or_else(|e| e.into_inner())
+ .finalize()?;
+ }
+
+ let metadata = Metadata {
+ stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),
+ };
+
+ let metadata_path = output.as_ref().join("metadata.json");
+
+ let metadata_file = std::fs::File::create(metadata_path)?;
+ serde_json::to_writer_pretty(metadata_file, &metadata)?;
+
+ Ok(())
+}
+
+#[derive(serde::Serialize)]
+struct DomainStats {
+ domain: Domain,
+ schedule_budget: u64,
+ scheduled_urls: u64,
+ wander_budget: u64,
+}
+
+#[derive(serde::Serialize)]
+struct Metadata {
+ stats: Vec<DomainStats>,
+}
diff --git a/core/src/crawler/site_graph.rs b/core/src/crawler/site_graph.rs
new file mode 100644
index 00000000..b13a1cff
--- /dev/null
+++ b/core/src/crawler/site_graph.rs
@@ -0,0 +1,158 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! In-memory graph that the worker constructs for the site during crawl.
+
+use hashbrown::{HashMap, HashSet};
+use url::Url;
+
+use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct Node {
+ url: Url,
+}
+
+impl From<Url> for Node {
+ fn from(url: Url) -> Self {
+ Self { url }
+ }
+}
+
+impl From<Node> for Url {
+ fn from(node: Node) -> Self {
+ node.url
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct NodeRef(usize);
+
+#[derive(Default)]
+pub struct SiteGraph {
+ nodes: Vec<Node>,
+ node_refs: HashMap<Node, NodeRef>,
+ edges: Vec<(NodeRef, NodeRef)>,
+}
+
+impl SiteGraph {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn add_node(&mut self, node: Node) -> NodeRef {
+ if let Some(node_ref) = self.node_refs.get(&node) {
+ return *node_ref;
+ }
+
+ let node_ref = NodeRef(self.nodes.len());
+ self.nodes.push(node.clone());
+ self.node_refs.insert(node, node_ref);
+
+ node_ref
+ }
+
+ pub fn add_edge(&mut self, from: NodeRef, to: NodeRef) {
+ self.edges.push((from, to));
+ }
+
+ pub fn get_node(&self, node_ref: NodeRef) -> Option<&Node> {
+ self.nodes.get(node_ref.0)
+ }
+
+ pub fn compute_centralities(&self) -> HashMap<NodeRef, f64> {
+ let mut counters: HashMap<_, _> = self
+ .node_refs
+ .iter()
+ .map(|(_, node_ref)| {
+ let mut counter: HyperLogLog<64> = HyperLogLog::default();
+ counter.add(node_ref.0 as u64);
+ (*node_ref, counter)
+ })
+ .collect();
+
+ let mut centralities: HashMap<_, _> = self
+ .node_refs
+ .iter()
+ .map(|(_, node_ref)| {
+ let kahan = KahanSum::default();
+
+ (*node_ref, kahan)
+ })
+ .collect();
+
+ let mut changed_nodes: HashSet<_> = self.node_refs.values().copied().collect();
+ let mut t = 0;
+
+ loop {
+ if changed_nodes.is_empty() {
+ break;
+ }
+
+ let mut new_counters = counters.clone();
+ let mut new_changed_nodes = HashSet::new();
+
+ for (from, to) in &self.edges {
+ if changed_nodes.contains(from) {
+ let counter_to = new_counters.get_mut(to).unwrap();
+ let counter_from = &counters[from];
+
+ if counter_to
+ .registers()
+ .iter()
+ .zip(counter_from.registers().iter())
+ .any(|(to, from)| *from > *to)
+ {
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(*to);
+ }
+ }
+ }
+
+ centralities.iter_mut().for_each(|(node, score)| {
+ *score += new_counters
+ .get(node)
+ .map(|counter| counter.size())
+ .unwrap_or_default()
+ .checked_sub(
+ counters
+ .get(node)
+ .map(|counter| counter.size())
+ .unwrap_or_default(),
+ )
+ .unwrap_or_default() as f64
+ / (t + 1) as f64;
+ });
+
+ counters = new_counters;
+ changed_nodes = new_changed_nodes;
+ t += 1;
+ }
+
+ centralities
+ .into_iter()
+ .map(|(node_ref, kahan)| (node_ref, f64::from(kahan)))
+ .filter(|(_, centrality)| *centrality > 0.0)
+ .map(|(node_ref, centrality)| {
+ if !centrality.is_finite() {
+ (node_ref, 0.0)
+ } else {
+ (node_ref, centrality)
+ }
+ })
+ .collect()
+ }
+}