Skip to content

web search done right

canonical index to lookup canonical version of urls (if they are the same host) for a higher quality web graph

Mikkel Denker2a17ca1d5b95
diff --git a/configs/canonical_index.toml b/configs/canonical_index.toml
new file mode 100644
index 00000000..91828f2c
--- /dev/null
+++ b/configs/canonical_index.toml
@@ -0,0 +1,6 @@
+output_path = "data/canonical"
+
+[warc_source]
+folder = "./data"
+names = ["sample.warc.gz"]
+type = "Local"
diff --git a/configs/webgraph/create.toml b/configs/webgraph/create.toml
index ee3b4726..d3e651cd 100644
--- a/configs/webgraph/create.toml
+++ b/configs/webgraph/create.toml
@@ -2,6 +2,7 @@ limit_warc_files = 5
host_graph_base_path = "data/webgraph"
page_graph_base_path = "data/webgraph_page"
batch_size = 1
+canonical_index_path = "data/canonical"
[warc_source]
folder = "./data"
diff --git a/crates/core/src/canon_index.rs b/crates/core/src/canon_index.rs
new file mode 100644
index 00000000..d696fbf0
--- /dev/null
+++ b/crates/core/src/canon_index.rs
@@ -0,0 +1,70 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+//! Index used to find the canonical URL for a given URL.
+//! (https://en.wikipedia.org/wiki/Canonical_link_element)
+
+use std::path::Path;
+
+use url::Url;
+
+use crate::{speedy_kv, webpage::url_ext::UrlExt, Result};
+
+#[derive(bincode::Decode, bincode::Encode)]
+struct StoredUrl(#[bincode(with_serde)] Url);
+
+pub struct CanonicalIndex {
+ inner: speedy_kv::Db<StoredUrl, StoredUrl>,
+}
+
+impl CanonicalIndex {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ Ok(Self {
+ inner: speedy_kv::Db::open_or_create(path)?,
+ })
+ }
+
+ pub fn insert(&mut self, original: Url, canonical: Url) -> Result<()> {
+ if original == canonical || original.root_domain() != canonical.root_domain() {
+ return Ok(());
+ }
+
+ let original = StoredUrl(original);
+ let canonical = StoredUrl(canonical);
+
+ self.inner.insert(original, canonical)
+ }
+
+ pub fn get(&self, url: &Url) -> Result<Option<Url>> {
+ Ok(self
+ .inner
+ .get(&StoredUrl(url.clone()))?
+ .map(|stored_url| stored_url.0))
+ }
+
+ pub fn commit(&mut self) -> Result<()> {
+ self.inner.commit()
+ }
+
+ pub fn optimize_read(&mut self) -> Result<()> {
+ self.inner.merge_all_segments()
+ }
+
+ pub fn merge(&mut self, other: CanonicalIndex) -> Result<()> {
+ self.inner.merge(other.inner)?;
+ self.optimize_read()
+ }
+}
diff --git a/crates/core/src/collector.rs b/crates/core/src/collector.rs
index 86dd26ef..3edd6794 100644
--- a/crates/core/src/collector.rs
+++ b/crates/core/src/collector.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index c486f1df..fdf5b5da 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -62,6 +62,7 @@ pub struct WebgraphConstructConfig {
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
pub batch_size: Option<usize>,
+ pub canonical_index_path: Option<String>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
@@ -556,3 +557,11 @@ pub struct HarmonicWorkerConfig {
pub graph_path: String,
pub host: SocketAddr,
}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub struct CanonicalIndexConfig {
+ pub output_path: String,
+ pub warc_source: WarcSource,
+ pub limit_warc_files: Option<usize>,
+ pub skip_warc_files: Option<usize>,
+}
diff --git a/crates/core/src/entrypoint/canonical.rs b/crates/core/src/entrypoint/canonical.rs
new file mode 100644
index 00000000..f9efbded
--- /dev/null
+++ b/crates/core/src/entrypoint/canonical.rs
@@ -0,0 +1,121 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use std::path::Path;
+
+use tracing::info;
+use url::Url;
+
+use crate::webpage::Html;
+use crate::Result;
+
+use crate::{
+ canon_index::CanonicalIndex,
+ config::{CanonicalIndexConfig, WarcSource},
+ warc::WarcFile,
+};
+
+#[derive(Debug, bincode::Encode, bincode::Decode, Clone)]
+pub struct Job {
+ pub warc_source: WarcSource,
+ pub warc_path: String,
+}
+
+pub struct Worker {
+ pub index: CanonicalIndex,
+}
+impl Worker {
+ pub fn process_job(&mut self, job: &Job) -> Result<()> {
+ let name = job.warc_path.split('/').last().unwrap();
+ info!("processing {}", name);
+
+ let warc_file = WarcFile::download(&job.warc_source, &job.warc_path)?;
+
+ for record in warc_file.records().flatten() {
+ let webpage = match Html::parse_without_text(&record.response.body, &record.request.url)
+ {
+ Ok(webpage) => webpage,
+ Err(err) => {
+ tracing::error!("error parsing webpage: {}", err);
+ continue;
+ }
+ };
+
+ if let Some(canonical_url) = webpage.canonical_url() {
+ let url = Url::parse(&record.request.url)?;
+ self.index.insert(url, canonical_url)?;
+ }
+ }
+
+ self.index.commit()?;
+
+ info!("{} done", name);
+
+ Ok(())
+ }
+}
+
+pub fn create(config: CanonicalIndexConfig) -> Result<()> {
+ let jobs: Vec<_> = config
+ .warc_source
+ .paths()?
+ .into_iter()
+ .skip(config.skip_warc_files.unwrap_or(0))
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
+ .map(|warc_path| Job {
+ warc_source: config.warc_source.clone(),
+ warc_path,
+ })
+ .collect();
+
+ let num_workers = num_cpus::get();
+ let mut handles = Vec::new();
+
+ for i in 0..num_workers {
+ let path = Path::new(&config.output_path).join(format!("worker_{}", i));
+ let index = CanonicalIndex::open(path)?;
+
+ let mut worker = Worker { index };
+ let jobs = jobs.clone();
+
+ let handle = std::thread::spawn(move || {
+ for job in jobs.into_iter().skip(i).step_by(num_workers) {
+ worker.process_job(&job).unwrap();
+ }
+
+ worker.index.optimize_read().unwrap();
+ worker.index
+ });
+
+ handles.push(handle);
+ }
+
+ let mut indexes = Vec::new();
+ for handler in handles {
+ let index = handler.join().unwrap();
+ indexes.push(index);
+ }
+
+ if !indexes.is_empty() {
+ let mut first = indexes.remove(0);
+ for index in indexes {
+ first.merge(index)?;
+ }
+ first.optimize_read()?;
+ }
+
+ Ok(())
+}
diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rs
index aecc9629..2695f181 100644
--- a/crates/core/src/entrypoint/feed_indexer.rs
+++ b/crates/core/src/entrypoint/feed_indexer.rs
@@ -28,9 +28,8 @@ use url::Url;
use crate::{
config::{FeedIndexingConfig, WarcSource},
- entrypoint::download_all_warc_files,
feed::{index::FeedIndex, Feed, FeedKind},
- warc::PayloadType,
+ warc::{PayloadType, WarcFile},
};
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
@@ -57,11 +56,7 @@ impl IndexingWorker {
let mut index = FeedIndex::open(Path::new(&job.base_path).join(&name)).unwrap();
- let w = &[job.warc_path.clone()];
- let warc_files = download_all_warc_files(w, &job.source_config);
- tokio::pin!(warc_files);
-
- for file in warc_files.by_ref() {
+ if let Ok(file) = WarcFile::download(&job.source_config, &job.warc_path) {
for record in
file.records()
.flatten()
@@ -96,8 +91,8 @@ impl IndexingWorker {
}
index.commit().unwrap();
+ index.merge_into_max_segments(1).unwrap();
}
- index.merge_into_max_segments(1).unwrap();
info!("{} done", name);
diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rs
index d98533c7..e2f6e73a 100644
--- a/crates/core/src/entrypoint/mod.rs
+++ b/crates/core/src/entrypoint/mod.rs
@@ -18,6 +18,7 @@
pub mod ampc;
pub mod api;
pub mod autosuggest_scrape;
+pub mod canonical;
mod centrality;
#[cfg(feature = "dev")]
pub mod configure;
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index a6744e0f..102d87cc 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -14,16 +14,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- config::WarcSource,
- config::{self, WebgraphConstructConfig},
+ canon_index::CanonicalIndex,
+ config::{self, WarcSource, WebgraphConstructConfig},
entrypoint::download_all_warc_files,
webgraph::{self, Node, WebgraphWriter},
webpage::{url_ext::UrlExt, Html},
Result,
};
use itertools::Itertools;
+use url::Url;
-use std::{fs, path::Path};
+use std::{path::Path, sync::Arc};
use tokio::pin;
use tracing::{info, trace};
@@ -81,9 +82,18 @@ pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWrit
)
}
+fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {
+ if let Some(url) = index.get(&url).unwrap() {
+ url
+ } else {
+ url
+ }
+}
+
pub struct WebgraphWorker {
pub host_graph: webgraph::WebgraphWriter,
pub page_graph: webgraph::WebgraphWriter,
+ pub canonical_index: Option<Arc<CanonicalIndex>>,
}
impl WebgraphWorker {
@@ -113,8 +123,14 @@ impl WebgraphWorker {
.into_iter()
.filter(|link| matches!(link.destination.scheme(), "http" | "https"))
{
- let source = link.source.clone();
- let destination = link.destination.clone();
+ let mut source = link.source.clone();
+ let mut destination = link.destination.clone();
+
+ if let Some(index) = &self.canonical_index {
+ source = canonical_or_self(index, source);
+ destination = canonical_or_self(index, destination);
+ }
+
link.text = link.text.chars().take(128).collect();
trace!("inserting link {:?}", link);
@@ -167,6 +183,12 @@ impl Webgraph {
})
.collect_vec();
+ let canonical_index = if let Some(index_path) = &config.canonical_index_path {
+ Some(Arc::new(CanonicalIndex::open(index_path)?))
+ } else {
+ None
+ };
+
let num_workers = num_cpus::get();
let mut handlers = Vec::new();
@@ -192,6 +214,7 @@ impl Webgraph {
let mut worker = WebgraphWorker {
host_graph: open_host_graph_writer(host_path),
page_graph: open_page_graph_writer(page_path),
+ canonical_index: canonical_index.clone(),
};
let jobs = jobs.clone();
@@ -216,14 +239,8 @@ impl Webgraph {
let (mut host_graph, mut page_graph) = graphs.pop().unwrap();
for (other_host, other_page) in graphs {
- let other_host_path = other_host.path.clone();
- let other_page_path = other_page.path.clone();
-
- host_graph.merge(other_host);
- page_graph.merge(other_page);
-
- fs::remove_dir_all(other_host_path)?;
- fs::remove_dir_all(other_page_path)?;
+ host_graph.merge(other_host)?;
+ page_graph.merge(other_page)?;
}
Ok(())
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index d452cc81..2e459259 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -41,6 +41,7 @@ pub mod autosuggest;
pub mod bangs;
mod bincode_utils;
pub mod bloom;
+pub mod canon_index;
mod collector;
pub mod config;
pub mod crawler;
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 362a3d8c..785e9958 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -20,6 +20,7 @@ use std::fs;
use std::path::Path;
use stract::config;
use stract::entrypoint::autosuggest_scrape::{self, Gl};
+use tracing::level_filters::LevelFilter;
#[cfg(feature = "dev")]
use stract::entrypoint::configure;
@@ -219,7 +220,9 @@ enum WebgraphOptions {
#[derive(Subcommand)]
enum IndexingOptions {
/// Create the search index.
- Search { config_path: String },
+ Search {
+ config_path: String,
+ },
/// Merge multiple search indexes into a single index.
MergeSearch {
@@ -234,7 +237,14 @@ enum IndexingOptions {
},
/// Create the feed index. Used to find feeds to put into the live index.
- Feed { config_path: String },
+ Feed {
+ config_path: String,
+ },
+
+ // Create an index of canonical urls.
+ Canonical {
+ config_path: String,
+ },
}
fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
@@ -250,7 +260,12 @@ fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_max_level(tracing::Level::INFO)
- .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+ .with_env_filter(
+ tracing_subscriber::EnvFilter::builder()
+ .with_default_directive(LevelFilter::OFF.into())
+ .from_env_lossy()
+ .add_directive("stract=info".parse().unwrap()),
+ )
.without_time()
.with_target(false)
.finish()
@@ -279,6 +294,10 @@ fn main() -> Result<()> {
.collect::<Vec<_>>();
entrypoint::indexer::merge(pointers)?;
}
+ IndexingOptions::Canonical { config_path } => {
+ let config: config::CanonicalIndexConfig = load_toml_config(config_path);
+ entrypoint::canonical::create(config)?;
+ }
},
Commands::Centrality { mode } => {
match mode {
@@ -308,7 +327,7 @@ fn main() -> Result<()> {
for other_path in paths {
let other = WebgraphBuilder::new(&other_path).single_threaded().open();
- webgraph.merge(other);
+ webgraph.merge(other)?;
std::fs::remove_dir_all(other_path).unwrap();
}
}
diff --git a/crates/core/src/speedy_kv/mod.rs b/crates/core/src/speedy_kv/mod.rs
index bc8703b3..69b744d0 100644
--- a/crates/core/src/speedy_kv/mod.rs
+++ b/crates/core/src/speedy_kv/mod.rs
@@ -370,6 +370,21 @@ impl<K, V> Db<K, V> {
Ok(())
}
+
+ pub fn merge(&mut self, other: Self) -> Result<()> {
+ let other_folder = other.folder().to_path_buf();
+ for segment in other.segments {
+ segment.move_to(self.folder())?;
+ self.segments.push(segment);
+ }
+
+ self.meta.segments = self.segments.iter().map(|s| s.uuid()).collect();
+ self.save_meta()?;
+
+ std::fs::remove_dir_all(other_folder)?;
+
+ Ok(())
+ }
}
impl<K, V> Db<K, V>
@@ -559,4 +574,40 @@ mod tests {
assert_eq!(db.len(), 2);
}
+
+ #[test]
+ fn test_merge_db() {
+ let mut db1 = Db::open_or_create(gen_temp_path()).unwrap();
+ let mut db2 = Db::open_or_create(gen_temp_path()).unwrap();
+
+ dbg!(db1.folder());
+ dbg!(db2.folder());
+
+ db1.insert(1, 2).unwrap();
+ db1.insert(2, 3).unwrap();
+
+ db1.commit().unwrap();
+
+ db2.insert(3, 4).unwrap();
+ db2.insert(4, 5).unwrap();
+
+ db2.commit().unwrap();
+
+ db1.merge(db2).unwrap();
+
+ assert_eq!(db1.get(&1).unwrap(), Some(2));
+ assert_eq!(db1.get(&2).unwrap(), Some(3));
+ assert_eq!(db1.get(&3).unwrap(), Some(4));
+ assert_eq!(db1.get(&4).unwrap(), Some(5));
+
+ let path = db1.folder().to_path_buf();
+ drop(db1);
+
+ let db = Db::open_or_create(path).unwrap();
+
+ assert_eq!(db.get(&1).unwrap(), Some(2));
+ assert_eq!(db.get(&2).unwrap(), Some(3));
+ assert_eq!(db.get(&3).unwrap(), Some(4));
+ assert_eq!(db.get(&4).unwrap(), Some(5));
+ }
}
diff --git a/crates/core/src/speedy_kv/segment.rs b/crates/core/src/speedy_kv/segment.rs
index 25f26b26..2f8188a0 100644
--- a/crates/core/src/speedy_kv/segment.rs
+++ b/crates/core/src/speedy_kv/segment.rs
@@ -264,6 +264,37 @@ impl<K, V> Segment<K, V> {
fn bloom_path(&self) -> PathBuf {
self.folder.join(Self::bloom_file_name(self.uuid))
}
+
+ pub fn move_to<P: AsRef<Path>>(&self, new_folder: P) -> Result<()> {
+ if !new_folder.as_ref().exists() {
+ std::fs::create_dir_all(&new_folder)?;
+ }
+
+ std::fs::rename(
+ self.blob_index.path(),
+ new_folder
+ .as_ref()
+ .join(self.blob_index.path().file_name().unwrap()),
+ )?;
+ std::fs::rename(
+ self.id_index.path(),
+ new_folder
+ .as_ref()
+ .join(self.id_index.path().file_name().unwrap()),
+ )?;
+ std::fs::rename(
+ self.store.path(),
+ new_folder
+ .as_ref()
+ .join(self.store.path().file_name().unwrap()),
+ )?;
+ std::fs::rename(
+ self.bloom_path(),
+ new_folder.as_ref().join(Self::bloom_file_name(self.uuid)),
+ )?;
+
+ Ok(())
+ }
}
struct Peekable<I>
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index e430ecd2..925629e3 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -13,11 +13,11 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::fs;
use std::fs::File;
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;
use std::sync::Arc;
+use std::{fs, io};
use itertools::Itertools;
use rand::seq::SliceRandom;
@@ -86,7 +86,7 @@ impl Meta {
}
pub struct Webgraph {
- pub path: String,
+ path: String,
segments: Vec<Segment>,
executor: Arc<Executor>,
id2node: Id2NodeDb,
@@ -134,7 +134,8 @@ impl Webgraph {
}
}
- pub fn merge(&mut self, other: Webgraph) {
+ pub fn merge(&mut self, other: Webgraph) -> io::Result<()> {
+ let other_folder = other.path.clone();
self.id2node.batch_put(other.id2node.iter());
for segment in other.segments {
@@ -148,8 +149,12 @@ impl Webgraph {
.push(Segment::open(new_path, id, self.compression));
}
+ fs::remove_dir_all(other_folder)?;
+
self.save_metadata();
self.id2node.flush();
+
+ Ok(())
}
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
@@ -409,7 +414,7 @@ pub mod tests {
let mut graph = graphs.pop().unwrap();
for other in graphs {
- graph.merge(other);
+ graph.merge(other).unwrap();
}
assert_eq!(
@@ -438,7 +443,7 @@ pub mod tests {
let mut graph = graphs.pop().unwrap();
for other in graphs {
- graph.merge(other);
+ graph.merge(other).unwrap();
}
assert_eq!(
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index be9692d2..4166a53e 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -14,6 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub const MAX_BATCH_SIZE: usize = 5_000_000;
+
use std::{
collections::BTreeSet,
fs::File,
@@ -59,8 +61,6 @@ impl<L: EdgeLabel> PartialEq for SortableEdge<L> {
impl<L: EdgeLabel> Eq for SortableEdge<L> {}
-pub const MAX_BATCH_SIZE: usize = 1_000_000;
-
struct SortedEdgeIterator<M, D>
where
M: Iterator<Item = SortableEdge<String>>,