web search done right
forgot some files
Mikkel Denkerd07d113ad19c
diff --git a/configs/crawler/planner.toml b/configs/crawler/planner.tomlnew file mode 100644index 00000000..cf2824be--- /dev/null+++ b/configs/crawler/planner.toml@@ -0,0 +1,11 @@+host_graph_path = "data/webgraph_host"+host_harmonic_path = "data/centrality/harmonic"+output_path = "data/crawlplan"+page_graph_path = "data/webgraph_page"+page_harmonic_path = "data/centrality_page/derived_harmonic"++num_job_queues = 5++crawl_budget = 1_000+top_host_fraction = 0.01+wander_fraction = 0.2diff --git a/core/src/crawler/file_queue.rs b/core/src/crawler/file_queue.rsnew file mode 100644index 00000000..92883dae--- /dev/null+++ b/core/src/crawler/file_queue.rs@@ -0,0 +1,220 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::Result;+use memmap2::Mmap;+use std::{+ fs::File,+ io::{BufWriter, Read, Seek, Write},+ path::PathBuf,+};++const POINTER_KEY: &str = "pointer";+const DATA_KEY: &str = "data";++#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)]+struct Header {+ body_size: usize,+}++pub struct FileQueueWriter<T> {+ path: PathBuf,+ writer: BufWriter<File>,+ _marker: std::marker::PhantomData<T>,+}++impl<T> FileQueueWriter<T>+where+ T: serde::Serialize + serde::de::DeserializeOwned,+{+ pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let file = std::fs::OpenOptions::new()+ .create(true)+ .write(true)+ .read(true)+ .open(path.as_ref().join(DATA_KEY))?;++ Ok(Self {+ path: path.as_ref().to_path_buf(),+ writer: BufWriter::new(file),+ _marker: std::marker::PhantomData,+ })+ }++ pub fn push(&mut self, item: T) -> Result<()> {+ let body = bincode::serialize(&item)?;+ let header = Header {+ body_size: body.len(),+ };++ let header_bytes = bincode::serialize(&header)?;++ self.writer.write_all(&header_bytes)?;+ self.writer.write_all(&body)?;++ Ok(())+ }++ pub fn flush(&mut self) -> Result<()> {+ self.writer.flush()?;++ Ok(())+ }++ pub fn finalize(mut self) -> Result<FileQueue<T>> {+ self.flush()?;++ let file = self.writer.into_inner()?;++ Ok(FileQueue {+ pointer: FilePointer::new(self.path)?,+ file: unsafe { Mmap::map(&file)? },+ _marker: std::marker::PhantomData,+ })+ }+}++struct FilePointer {+ file: File,+}++impl FilePointer {+ fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let file = std::fs::OpenOptions::new()+ .create(true)+ .write(true)+ .read(true)+ .open(path.as_ref().join(POINTER_KEY))?;++ Ok(Self { file })+ }++ fn get(&mut self) -> usize {+ self.file.seek(std::io::SeekFrom::Start(0)).unwrap();+ let mut buf = [0u8; 8];++ if self.file.read_exact(&mut buf).is_err() {+ return 0;+ }++ usize::from_be_bytes(buf)+ }++ fn set(&mut self, pointer: usize) -> Result<()> {+ self.file.seek(std::io::SeekFrom::Start(0))?;+ self.file.write_all(&pointer.to_be_bytes())?;+ self.file.flush()?;++ Ok(())+ }+}++pub struct FileQueue<T> {+ pointer: FilePointer,+ file: Mmap,+ _marker: std::marker::PhantomData<T>,+}++impl<T> FileQueue<T>+where+ T: serde::Serialize + serde::de::DeserializeOwned,+{+ pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let file = File::open(path.as_ref().join(DATA_KEY))?;+ let file = unsafe { Mmap::map(&file)? };++ Ok(Self {+ pointer: FilePointer::new(path)?,+ file,+ _marker: std::marker::PhantomData,+ })+ }++ pub fn pop(&mut self) -> Result<Option<T>> {+ let cur_pointer = self.pointer.get();++ if cur_pointer >= self.file.len() {+ return Ok(None);+ }++ let header_size = std::mem::size_of::<Header>();+ let header: Header =+ bincode::deserialize(&self.file[cur_pointer..cur_pointer + header_size])?;++ let body =+ &self.file[cur_pointer + header_size..cur_pointer + header_size + header.body_size];+ let item = bincode::deserialize(body)?;++ self.pointer+ .set(cur_pointer + header_size + header.body_size)?;++ Ok(Some(item))+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use proptest::prelude::*;++ #[test]+ fn simple() {+ let mut writer = FileQueueWriter::new(crate::gen_temp_path()).unwrap();++ writer.push("Hello".to_string()).unwrap();+ writer.push("World".to_string()).unwrap();++ let mut queue = writer.finalize().unwrap();++ assert_eq!(queue.pop().unwrap().unwrap(), "Hello");+ assert_eq!(queue.pop().unwrap().unwrap(), "World");+ assert_eq!(queue.pop().unwrap(), None);+ }++ proptest! {+ #[test]+ fn prop(data: Vec<String>) {+ let expected = data.clone();++ let mut writer = FileQueueWriter::new(crate::gen_temp_path()).unwrap();++ for item in data {+ writer.push(item).unwrap();+ }++ let mut queue = writer.finalize().unwrap();+ let mut actual = Vec::new();++ while let Some(item) = queue.pop().unwrap() {+ actual.push(item);+ }++ prop_assert_eq!(actual, expected);+ }+ }+}diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rsnew file mode 100644index 00000000..1a5c49f5--- /dev/null+++ b/core/src/crawler/planner.rs@@ -0,0 +1,230 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use anyhow::{anyhow, Result};+use rayon::prelude::*;+use std::collections::HashMap;+use std::{+ collections::VecDeque,+ path::Path,+ sync::{atomic::AtomicUsize, Mutex},+};+use url::Url;++use crate::{+ config::CrawlPlannerConfig,+ crawler::{file_queue::FileQueueWriter, Job},+ kv::{rocksdb_store::RocksDbStore, Kv},+ webgraph::{NodeID, Webgraph},+};++use super::Domain;++fn top_hosts(host_centrality: &RocksDbStore<NodeID, f64>, fraction: f64) -> Vec<NodeID> {+ let mut hosts = host_centrality+ .iter()+ .map(|(id, centrality)| (id, centrality))+ .map(|(id, centrality)| {+ if !centrality.is_finite() {+ (id, 0.0)+ } else {+ (id, centrality)+ }+ })+ .collect::<Vec<_>>();++ hosts.sort_by(|(_, a), (_, b)| b.total_cmp(a));++ let num_hosts = (hosts.len() as f64 * fraction) as usize;++ hosts+ .into_iter()+ .take(num_hosts)+ .map(|(id, _)| id)+ .collect()+}++fn all_pages(+ page_centrality: &RocksDbStore<NodeID, f64>,+ page_graph: &Webgraph,+ host: NodeID,+) -> Vec<(NodeID, f64)> {+ page_graph+ .raw_ingoing_edges_by_host(&host)+ .into_iter()+ .map(|e| e.to)+ .map(|id| (id, page_centrality.get(&id).unwrap_or_default()))+ .collect::<Vec<_>>()+}++fn group_domain(hosts: Vec<NodeID>, host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {+ let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();++ for host in hosts {+ let node = host_graph.id2node(&host).unwrap();+ if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {+ let domain = Domain::from(url);+ domains.entry(domain).or_default().push(host);+ }+ }++ domains+}++fn check_config(config: &CrawlPlannerConfig) -> Result<()> {+ if !(0.0..=1.0).contains(&config.wander_fraction) {+ return Err(anyhow::anyhow!(+ "top_host_fraction must be in range [0.0, 1.0]"+ ));+ }++ if !(0.0..=1.0).contains(&config.top_host_fraction) {+ return Err(anyhow::anyhow!(+ "top_host_fraction must be in range [0.0, 1.0]"+ ));+ }++ Ok(())+}++pub fn make_crawl_plan<P: AsRef<Path>>(+ host_centrality: RocksDbStore<NodeID, f64>,+ page_centrality: RocksDbStore<NodeID, f64>,+ host_graph: Webgraph,+ page_graph: Webgraph,+ config: CrawlPlannerConfig,+ output: P,+) -> Result<()> {+ check_config(&config)?;++ if output.as_ref().exists() {+ return Err(anyhow!("output path already exists"));+ }++ let queue_path = output.as_ref().join("job_queue");+ std::fs::create_dir_all(&queue_path)?;++ let hosts = top_hosts(&host_centrality, config.top_host_fraction);+ tracing::debug!("generating for {} hosts", hosts.len());+ let grouped = group_domain(hosts, &host_graph);++ let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)+ .map(|i| {+ let path = queue_path.join(format!("{}.queue", i));+ FileQueueWriter::new(path)+ })+ .collect::<Result<Vec<_>>>()?+ .into_iter()+ .map(Mutex::new)+ .collect();++ let stats = Mutex::new(Vec::new());++ let next_queue = AtomicUsize::new(0);++ grouped.into_par_iter().for_each(|(domain, hosts)| {+ let mut total_wander_budget = 0;+ let mut total_schedule_budget = 0;+ let mut total_scheduled_urls = 0;+ let mut urls = VecDeque::new();++ for host in hosts {+ let mut pages = all_pages(&page_centrality, &page_graph, host);+ pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));+ tracing::debug!("num pages: {}", pages.len());+ let host_centrality = host_centrality.get(&host).unwrap_or_default();++ let host_budget = (config.crawl_budget as f64 * host_centrality).max(1.0) as u64;+ tracing::debug!("host_budget: {host_budget}");+ let schedule_budget =+ (host_budget as f64 * (1.0 - config.wander_fraction)).max(1.0) as u64;+ tracing::debug!("schedule_budget: {schedule_budget}");+ let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0) as u64;+ tracing::debug!("wander_budget: {wander_budget}");++ total_wander_budget += wander_budget;+ total_schedule_budget += schedule_budget;++ let before = urls.len();+ urls.extend(+ pages+ .into_iter()+ .map(|(id, _)| id)+ .filter_map(|id| page_graph.id2node(&id))+ .map(|n| n.name)+ .filter_map(|n| Url::parse(&format!("http://{n}")).ok())+ .take(schedule_budget as usize),+ );++ total_scheduled_urls += urls.len() as u64 - before as u64;+ }++ let job = Job {+ domain: domain.clone(),+ urls,+ wandering_urls: total_wander_budget,+ };++ let domain_stats = DomainStats {+ domain,+ schedule_budget: total_schedule_budget,+ wander_budget: total_wander_budget,+ scheduled_urls: total_scheduled_urls,+ };+ stats+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(domain_stats);++ // `fetch_add` wraps around on overflow+ let queue_index = next_queue.fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ job_queues[queue_index % job_queues.len()]+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(job)+ .unwrap();+ });++ for queue in job_queues {+ queue+ .into_inner()+ .unwrap_or_else(|e| e.into_inner())+ .finalize()?;+ }++ let metadata = Metadata {+ stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),+ };++ let metadata_path = output.as_ref().join("metadata.json");++ let metadata_file = std::fs::File::create(metadata_path)?;+ serde_json::to_writer_pretty(metadata_file, &metadata)?;++ Ok(())+}++#[derive(serde::Serialize)]+struct DomainStats {+ domain: Domain,+ schedule_budget: u64,+ scheduled_urls: u64,+ wander_budget: u64,+}++#[derive(serde::Serialize)]+struct Metadata {+ stats: Vec<DomainStats>,+}diff --git a/core/src/crawler/site_graph.rs b/core/src/crawler/site_graph.rsnew file mode 100644index 00000000..b13a1cff--- /dev/null+++ b/core/src/crawler/site_graph.rs@@ -0,0 +1,158 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! In-memory graph that the worker constructs for the site during crawl.++use hashbrown::{HashMap, HashSet};+use url::Url;++use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};++#[derive(Clone, Debug, PartialEq, Eq, Hash)]+pub struct Node {+ url: Url,+}++impl From<Url> for Node {+ fn from(url: Url) -> Self {+ Self { url }+ }+}++impl From<Node> for Url {+ fn from(node: Node) -> Self {+ node.url+ }+}++#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+pub struct NodeRef(usize);++#[derive(Default)]+pub struct SiteGraph {+ nodes: Vec<Node>,+ node_refs: HashMap<Node, NodeRef>,+ edges: Vec<(NodeRef, NodeRef)>,+}++impl SiteGraph {+ pub fn new() -> Self {+ Self::default()+ }++ pub fn add_node(&mut self, node: Node) -> NodeRef {+ if let Some(node_ref) = self.node_refs.get(&node) {+ return *node_ref;+ }++ let node_ref = NodeRef(self.nodes.len());+ self.nodes.push(node.clone());+ self.node_refs.insert(node, node_ref);++ node_ref+ }++ pub fn add_edge(&mut self, from: NodeRef, to: NodeRef) {+ self.edges.push((from, to));+ }++ pub fn get_node(&self, node_ref: NodeRef) -> Option<&Node> {+ self.nodes.get(node_ref.0)+ }++ pub fn compute_centralities(&self) -> HashMap<NodeRef, f64> {+ let mut counters: HashMap<_, _> = self+ .node_refs+ .iter()+ .map(|(_, node_ref)| {+ let mut counter: HyperLogLog<64> = HyperLogLog::default();+ counter.add(node_ref.0 as u64);+ (*node_ref, counter)+ })+ .collect();++ let mut centralities: HashMap<_, _> = self+ .node_refs+ .iter()+ .map(|(_, node_ref)| {+ let kahan = KahanSum::default();++ (*node_ref, kahan)+ })+ .collect();++ let mut changed_nodes: HashSet<_> = self.node_refs.values().copied().collect();+ let mut t = 0;++ loop {+ if changed_nodes.is_empty() {+ break;+ }++ let mut new_counters = counters.clone();+ let mut new_changed_nodes = HashSet::new();++ for (from, to) in &self.edges {+ if changed_nodes.contains(from) {+ let counter_to = new_counters.get_mut(to).unwrap();+ let counter_from = &counters[from];++ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ new_changed_nodes.insert(*to);+ }+ }+ }++ centralities.iter_mut().for_each(|(node, score)| {+ *score += new_counters+ .get(node)+ .map(|counter| counter.size())+ .unwrap_or_default()+ .checked_sub(+ counters+ .get(node)+ .map(|counter| counter.size())+ .unwrap_or_default(),+ )+ .unwrap_or_default() as f64+ / (t + 1) as f64;+ });++ counters = new_counters;+ changed_nodes = new_changed_nodes;+ t += 1;+ }++ centralities+ .into_iter()+ .map(|(node_ref, kahan)| (node_ref, f64::from(kahan)))+ .filter(|(_, centrality)| *centrality > 0.0)+ .map(|(node_ref, centrality)| {+ if !centrality.is_finite() {+ (node_ref, 0.0)+ } else {+ (node_ref, centrality)+ }+ })+ .collect()+ }+}