web search done right
canonical index to lookup canonical version of urls (if they are the same host) for a higher quality web graph
Mikkel Denker2a17ca1d5b95
diff --git a/configs/canonical_index.toml b/configs/canonical_index.tomlnew file mode 100644index 00000000..91828f2c--- /dev/null+++ b/configs/canonical_index.toml@@ -0,0 +1,6 @@+output_path = "data/canonical"++[warc_source]+folder = "./data"+names = ["sample.warc.gz"]+type = "Local"diff --git a/configs/webgraph/create.toml b/configs/webgraph/create.tomlindex ee3b4726..d3e651cd 100644--- a/configs/webgraph/create.toml+++ b/configs/webgraph/create.toml@@ -2,6 +2,7 @@ limit_warc_files = 5host_graph_base_path = "data/webgraph"page_graph_base_path = "data/webgraph_page"batch_size = 1+canonical_index_path = "data/canonical"[warc_source]folder = "./data"diff --git a/crates/core/src/canon_index.rs b/crates/core/src/canon_index.rsnew file mode 100644index 00000000..d696fbf0--- /dev/null+++ b/crates/core/src/canon_index.rs@@ -0,0 +1,70 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++//! Index used to find the canonical URL for a given URL.+//! (https://en.wikipedia.org/wiki/Canonical_link_element)++use std::path::Path;++use url::Url;++use crate::{speedy_kv, webpage::url_ext::UrlExt, Result};++#[derive(bincode::Decode, bincode::Encode)]+struct StoredUrl(#[bincode(with_serde)] Url);++pub struct CanonicalIndex {+ inner: speedy_kv::Db<StoredUrl, StoredUrl>,+}++impl CanonicalIndex {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ Ok(Self {+ inner: speedy_kv::Db::open_or_create(path)?,+ })+ }++ pub fn insert(&mut self, original: Url, canonical: Url) -> Result<()> {+ if original == canonical || original.root_domain() != canonical.root_domain() {+ return Ok(());+ }++ let original = StoredUrl(original);+ let canonical = StoredUrl(canonical);++ self.inner.insert(original, canonical)+ }++ pub fn get(&self, url: &Url) -> Result<Option<Url>> {+ Ok(self+ .inner+ .get(&StoredUrl(url.clone()))?+ .map(|stored_url| stored_url.0))+ }++ pub fn commit(&mut self) -> Result<()> {+ self.inner.commit()+ }++ pub fn optimize_read(&mut self) -> Result<()> {+ self.inner.merge_all_segments()+ }++ pub fn merge(&mut self, other: CanonicalIndex) -> Result<()> {+ self.inner.merge(other.inner)?;+ self.optimize_read()+ }+}diff --git a/crates/core/src/collector.rs b/crates/core/src/collector.rsindex 86dd26ef..3edd6794 100644--- a/crates/core/src/collector.rs+++ b/crates/core/src/collector.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License asdiff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex c486f1df..fdf5b5da 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -62,6 +62,7 @@ pub struct WebgraphConstructConfig {pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,pub batch_size: Option<usize>,+ pub canonical_index_path: Option<String>,}#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]@@ -556,3 +557,11 @@ pub struct HarmonicWorkerConfig {pub graph_path: String,pub host: SocketAddr,}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct CanonicalIndexConfig {+ pub output_path: String,+ pub warc_source: WarcSource,+ pub limit_warc_files: Option<usize>,+ pub skip_warc_files: Option<usize>,+}diff --git a/crates/core/src/entrypoint/canonical.rs b/crates/core/src/entrypoint/canonical.rsnew file mode 100644index 00000000..f9efbded--- /dev/null+++ b/crates/core/src/entrypoint/canonical.rs@@ -0,0 +1,121 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::path::Path;++use tracing::info;+use url::Url;++use crate::webpage::Html;+use crate::Result;++use crate::{+ canon_index::CanonicalIndex,+ config::{CanonicalIndexConfig, WarcSource},+ warc::WarcFile,+};++#[derive(Debug, bincode::Encode, bincode::Decode, Clone)]+pub struct Job {+ pub warc_source: WarcSource,+ pub warc_path: String,+}++pub struct Worker {+ pub index: CanonicalIndex,+}+impl Worker {+ pub fn process_job(&mut self, job: &Job) -> Result<()> {+ let name = job.warc_path.split('/').last().unwrap();+ info!("processing {}", name);++ let warc_file = WarcFile::download(&job.warc_source, &job.warc_path)?;++ for record in warc_file.records().flatten() {+ let webpage = match Html::parse_without_text(&record.response.body, &record.request.url)+ {+ Ok(webpage) => webpage,+ Err(err) => {+ tracing::error!("error parsing webpage: {}", err);+ continue;+ }+ };++ if let Some(canonical_url) = webpage.canonical_url() {+ let url = Url::parse(&record.request.url)?;+ self.index.insert(url, canonical_url)?;+ }+ }++ self.index.commit()?;++ info!("{} done", name);++ Ok(())+ }+}++pub fn create(config: CanonicalIndexConfig) -> Result<()> {+ let jobs: Vec<_> = config+ .warc_source+ .paths()?+ .into_iter()+ .skip(config.skip_warc_files.unwrap_or(0))+ .take(config.limit_warc_files.unwrap_or(usize::MAX))+ .map(|warc_path| Job {+ warc_source: config.warc_source.clone(),+ warc_path,+ })+ .collect();++ let num_workers = num_cpus::get();+ let mut handles = Vec::new();++ for i in 0..num_workers {+ let path = Path::new(&config.output_path).join(format!("worker_{}", i));+ let index = CanonicalIndex::open(path)?;++ let mut worker = Worker { index };+ let jobs = jobs.clone();++ let handle = std::thread::spawn(move || {+ for job in jobs.into_iter().skip(i).step_by(num_workers) {+ worker.process_job(&job).unwrap();+ }++ worker.index.optimize_read().unwrap();+ worker.index+ });++ handles.push(handle);+ }++ let mut indexes = Vec::new();+ for handler in handles {+ let index = handler.join().unwrap();+ indexes.push(index);+ }++ if !indexes.is_empty() {+ let mut first = indexes.remove(0);+ for index in indexes {+ first.merge(index)?;+ }+ first.optimize_read()?;+ }++ Ok(())+}diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rsindex aecc9629..2695f181 100644--- a/crates/core/src/entrypoint/feed_indexer.rs+++ b/crates/core/src/entrypoint/feed_indexer.rs@@ -28,9 +28,8 @@ use url::Url;use crate::{config::{FeedIndexingConfig, WarcSource},- entrypoint::download_all_warc_files,feed::{index::FeedIndex, Feed, FeedKind},- warc::PayloadType,+ warc::{PayloadType, WarcFile},};#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]@@ -57,11 +56,7 @@ impl IndexingWorker {let mut index = FeedIndex::open(Path::new(&job.base_path).join(&name)).unwrap();- let w = &[job.warc_path.clone()];- let warc_files = download_all_warc_files(w, &job.source_config);- tokio::pin!(warc_files);-- for file in warc_files.by_ref() {+ if let Ok(file) = WarcFile::download(&job.source_config, &job.warc_path) {for record infile.records().flatten()@@ -96,8 +91,8 @@ impl IndexingWorker {}index.commit().unwrap();+ index.merge_into_max_segments(1).unwrap();}- index.merge_into_max_segments(1).unwrap();info!("{} done", name);diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rsindex d98533c7..e2f6e73a 100644--- a/crates/core/src/entrypoint/mod.rs+++ b/crates/core/src/entrypoint/mod.rs@@ -18,6 +18,7 @@pub mod ampc;pub mod api;pub mod autosuggest_scrape;+pub mod canonical;mod centrality;#[cfg(feature = "dev")]pub mod configure;diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex a6744e0f..102d87cc 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -14,16 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- config::WarcSource,- config::{self, WebgraphConstructConfig},+ canon_index::CanonicalIndex,+ config::{self, WarcSource, WebgraphConstructConfig},entrypoint::download_all_warc_files,webgraph::{self, Node, WebgraphWriter},webpage::{url_ext::UrlExt, Html},Result,};use itertools::Itertools;+use url::Url;-use std::{fs, path::Path};+use std::{path::Path, sync::Arc};use tokio::pin;use tracing::{info, trace};@@ -81,9 +82,18 @@ pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWrit)}+fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {+ if let Some(url) = index.get(&url).unwrap() {+ url+ } else {+ url+ }+}+pub struct WebgraphWorker {pub host_graph: webgraph::WebgraphWriter,pub page_graph: webgraph::WebgraphWriter,+ pub canonical_index: Option<Arc<CanonicalIndex>>,}impl WebgraphWorker {@@ -113,8 +123,14 @@ impl WebgraphWorker {.into_iter().filter(|link| matches!(link.destination.scheme(), "http" | "https")){- let source = link.source.clone();- let destination = link.destination.clone();+ let mut source = link.source.clone();+ let mut destination = link.destination.clone();++ if let Some(index) = &self.canonical_index {+ source = canonical_or_self(index, source);+ destination = canonical_or_self(index, destination);+ }+link.text = link.text.chars().take(128).collect();trace!("inserting link {:?}", link);@@ -167,6 +183,12 @@ impl Webgraph {}).collect_vec();+ let canonical_index = if let Some(index_path) = &config.canonical_index_path {+ Some(Arc::new(CanonicalIndex::open(index_path)?))+ } else {+ None+ };+let num_workers = num_cpus::get();let mut handlers = Vec::new();@@ -192,6 +214,7 @@ impl Webgraph {let mut worker = WebgraphWorker {host_graph: open_host_graph_writer(host_path),page_graph: open_page_graph_writer(page_path),+ canonical_index: canonical_index.clone(),};let jobs = jobs.clone();@@ -216,14 +239,8 @@ impl Webgraph {let (mut host_graph, mut page_graph) = graphs.pop().unwrap();for (other_host, other_page) in graphs {- let other_host_path = other_host.path.clone();- let other_page_path = other_page.path.clone();-- host_graph.merge(other_host);- page_graph.merge(other_page);-- fs::remove_dir_all(other_host_path)?;- fs::remove_dir_all(other_page_path)?;+ host_graph.merge(other_host)?;+ page_graph.merge(other_page)?;}Ok(())diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex d452cc81..2e459259 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -41,6 +41,7 @@ pub mod autosuggest;pub mod bangs;mod bincode_utils;pub mod bloom;+pub mod canon_index;mod collector;pub mod config;pub mod crawler;diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 362a3d8c..785e9958 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -20,6 +20,7 @@ use std::fs;use std::path::Path;use stract::config;use stract::entrypoint::autosuggest_scrape::{self, Gl};+use tracing::level_filters::LevelFilter;#[cfg(feature = "dev")]use stract::entrypoint::configure;@@ -219,7 +220,9 @@ enum WebgraphOptions {#[derive(Subcommand)]enum IndexingOptions {/// Create the search index.- Search { config_path: String },+ Search {+ config_path: String,+ },/// Merge multiple search indexes into a single index.MergeSearch {@@ -234,7 +237,14 @@ enum IndexingOptions {},/// Create the feed index. Used to find feeds to put into the live index.- Feed { config_path: String },+ Feed {+ config_path: String,+ },++ // Create an index of canonical urls.+ Canonical {+ config_path: String,+ },}fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {@@ -250,7 +260,12 @@ fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {fn main() -> Result<()> {tracing_subscriber::fmt().with_max_level(tracing::Level::INFO)- .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())+ .with_env_filter(+ tracing_subscriber::EnvFilter::builder()+ .with_default_directive(LevelFilter::OFF.into())+ .from_env_lossy()+ .add_directive("stract=info".parse().unwrap()),+ ).without_time().with_target(false).finish()@@ -279,6 +294,10 @@ fn main() -> Result<()> {.collect::<Vec<_>>();entrypoint::indexer::merge(pointers)?;}+ IndexingOptions::Canonical { config_path } => {+ let config: config::CanonicalIndexConfig = load_toml_config(config_path);+ entrypoint::canonical::create(config)?;+ }},Commands::Centrality { mode } => {match mode {@@ -308,7 +327,7 @@ fn main() -> Result<()> {for other_path in paths {let other = WebgraphBuilder::new(&other_path).single_threaded().open();- webgraph.merge(other);+ webgraph.merge(other)?;std::fs::remove_dir_all(other_path).unwrap();}}diff --git a/crates/core/src/speedy_kv/mod.rs b/crates/core/src/speedy_kv/mod.rsindex bc8703b3..69b744d0 100644--- a/crates/core/src/speedy_kv/mod.rs+++ b/crates/core/src/speedy_kv/mod.rs@@ -370,6 +370,21 @@ impl<K, V> Db<K, V> {Ok(())}++ pub fn merge(&mut self, other: Self) -> Result<()> {+ let other_folder = other.folder().to_path_buf();+ for segment in other.segments {+ segment.move_to(self.folder())?;+ self.segments.push(segment);+ }++ self.meta.segments = self.segments.iter().map(|s| s.uuid()).collect();+ self.save_meta()?;++ std::fs::remove_dir_all(other_folder)?;++ Ok(())+ }}impl<K, V> Db<K, V>@@ -559,4 +574,40 @@ mod tests {assert_eq!(db.len(), 2);}++ #[test]+ fn test_merge_db() {+ let mut db1 = Db::open_or_create(gen_temp_path()).unwrap();+ let mut db2 = Db::open_or_create(gen_temp_path()).unwrap();++ dbg!(db1.folder());+ dbg!(db2.folder());++ db1.insert(1, 2).unwrap();+ db1.insert(2, 3).unwrap();++ db1.commit().unwrap();++ db2.insert(3, 4).unwrap();+ db2.insert(4, 5).unwrap();++ db2.commit().unwrap();++ db1.merge(db2).unwrap();++ assert_eq!(db1.get(&1).unwrap(), Some(2));+ assert_eq!(db1.get(&2).unwrap(), Some(3));+ assert_eq!(db1.get(&3).unwrap(), Some(4));+ assert_eq!(db1.get(&4).unwrap(), Some(5));++ let path = db1.folder().to_path_buf();+ drop(db1);++ let db = Db::open_or_create(path).unwrap();++ assert_eq!(db.get(&1).unwrap(), Some(2));+ assert_eq!(db.get(&2).unwrap(), Some(3));+ assert_eq!(db.get(&3).unwrap(), Some(4));+ assert_eq!(db.get(&4).unwrap(), Some(5));+ }}diff --git a/crates/core/src/speedy_kv/segment.rs b/crates/core/src/speedy_kv/segment.rsindex 25f26b26..2f8188a0 100644--- a/crates/core/src/speedy_kv/segment.rs+++ b/crates/core/src/speedy_kv/segment.rs@@ -264,6 +264,37 @@ impl<K, V> Segment<K, V> {fn bloom_path(&self) -> PathBuf {self.folder.join(Self::bloom_file_name(self.uuid))}++ pub fn move_to<P: AsRef<Path>>(&self, new_folder: P) -> Result<()> {+ if !new_folder.as_ref().exists() {+ std::fs::create_dir_all(&new_folder)?;+ }++ std::fs::rename(+ self.blob_index.path(),+ new_folder+ .as_ref()+ .join(self.blob_index.path().file_name().unwrap()),+ )?;+ std::fs::rename(+ self.id_index.path(),+ new_folder+ .as_ref()+ .join(self.id_index.path().file_name().unwrap()),+ )?;+ std::fs::rename(+ self.store.path(),+ new_folder+ .as_ref()+ .join(self.store.path().file_name().unwrap()),+ )?;+ std::fs::rename(+ self.bloom_path(),+ new_folder.as_ref().join(Self::bloom_file_name(self.uuid)),+ )?;++ Ok(())+ }}struct Peekable<I>diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex e430ecd2..925629e3 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -13,11 +13,11 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::fs;use std::fs::File;use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;use std::sync::Arc;+use std::{fs, io};use itertools::Itertools;use rand::seq::SliceRandom;@@ -86,7 +86,7 @@ impl Meta {}pub struct Webgraph {- pub path: String,+ path: String,segments: Vec<Segment>,executor: Arc<Executor>,id2node: Id2NodeDb,@@ -134,7 +134,8 @@ impl Webgraph {}}- pub fn merge(&mut self, other: Webgraph) {+ pub fn merge(&mut self, other: Webgraph) -> io::Result<()> {+ let other_folder = other.path.clone();self.id2node.batch_put(other.id2node.iter());for segment in other.segments {@@ -148,8 +149,12 @@ impl Webgraph {.push(Segment::open(new_path, id, self.compression));}+ fs::remove_dir_all(other_folder)?;+self.save_metadata();self.id2node.flush();++ Ok(())}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {@@ -409,7 +414,7 @@ pub mod tests {let mut graph = graphs.pop().unwrap();for other in graphs {- graph.merge(other);+ graph.merge(other).unwrap();}assert_eq!(@@ -438,7 +443,7 @@ pub mod tests {let mut graph = graphs.pop().unwrap();for other in graphs {- graph.merge(other);+ graph.merge(other).unwrap();}assert_eq!(diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex be9692d2..4166a53e 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -14,6 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub const MAX_BATCH_SIZE: usize = 5_000_000;+use std::{collections::BTreeSet,fs::File,@@ -59,8 +61,6 @@ impl<L: EdgeLabel> PartialEq for SortableEdge<L> {impl<L: EdgeLabel> Eq for SortableEdge<L> {}-pub const MAX_BATCH_SIZE: usize = 1_000_000;-struct SortedEdgeIterator<M, D>whereM: Iterator<Item = SortableEdge<String>>,