Skip to content

web search done right

count 'www' and no-subdomain as the same in webgraph. Also other small bugfixes in webgraph

Mikkel Denker1a99076eae23
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
new file mode 100644
index 00000000..d3e2d932
--- /dev/null
+++ b/configs/webgraph/local.toml
@@ -0,0 +1,10 @@
+limit_warc_files = 3
+graph_base_path = "data/webgraph"
+
+[warc_source]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "warc.paths"
+# type = "Local"
+# folder = "./data/warc_files"
+# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
\ No newline at end of file
diff --git a/configs/webgraph/local/local.toml b/configs/webgraph/local/local.toml
deleted file mode 100644
index 6b23e4aa..00000000
--- a/configs/webgraph/local/local.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-limit_warc_files = 10
-graph_base_path = "data/webgraph"
-
-[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "warc.paths"
\ No newline at end of file
diff --git a/configs/webgraph/local/master.toml b/configs/webgraph/local/master.toml
deleted file mode 100644
index a7192ced..00000000
--- a/configs/webgraph/local/master.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-workers = ["0.0.0.0:1338"]
-
-[warc_source]
-type = "local"
-folder = "./warc_files"
-names = ["small1.warc.gz", "small2.warc.gz"]
diff --git a/configs/webgraph/local/profile.toml b/configs/webgraph/profile.toml
similarity index 100%
rename from configs/webgraph/local/profile.toml
rename to configs/webgraph/profile.toml
diff --git a/justfile b/justfile
index 889a405d..926b5c17 100644
--- a/justfile
+++ b/justfile
@@ -7,10 +7,14 @@
cargo flamegraph --root -- indexer local configs/indexer/local.toml
@worker:
- cargo run --release -- configs/webgraph/worker.toml
+ cargo run --release -- webgraph worker configs/webgraph/worker.toml
@master:
- cargo run --release -- configs/webgraph/master.toml
+ cargo run --release -- webgraph master configs/webgraph/master.toml
+
+@webgraph:
+ rm -rf data/webgraph
+ cargo run --release -- webgraph local configs/webgraph/local.toml
@frontend:
cargo watch -x 'run -- frontend data/index data/queries_us.csv data/entity'
diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rs
index baad8828..d3e19278 100644
--- a/src/entrypoint/centrality.rs
+++ b/src/entrypoint/centrality.rs
@@ -14,23 +14,73 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::Path;
+use std::{collections::HashMap, fs::File, path::Path};
-use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};
+use crate::{
+ ranking::centrality_store::CentralityStore,
+ webgraph::{Webgraph, WebgraphBuilder},
+};
pub struct Centrality {}
impl Centrality {
- pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
- let graph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();
+ fn host_centrality(graph: &Webgraph) -> HashMap<String, f64> {
+ graph
+ .host_harmonic_centrality()
+ .into_iter()
+ .map(|(node, centrality)| (node.name, centrality))
+ .collect()
+ }
+
+ fn full_centrality(graph: &Webgraph) -> HashMap<String, f64> {
+ graph
+ .harmonic_centrality()
+ .into_iter()
+ .map(|(node, centrality)| (node.name, centrality))
+ .collect()
+ }
+
+ fn save<P: AsRef<Path>>(centrality: HashMap<String, f64>, output_path: P) {
+ let mut centrality_store = CentralityStore::new(output_path.as_ref());
+
+ centrality_store.append(centrality.clone().into_iter());
+
+ let mut centralities: Vec<_> = centrality.into_iter().collect();
+ centralities
+ .sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
+
+ let csv_file = File::options()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(output_path.as_ref().join("data.csv"))
+ .unwrap();
+ let mut wtr = csv::Writer::from_writer(csv_file);
- let mut centrality_store = CentralityStore::new(output_path);
+ for (host, centrality) in centralities {
+ wtr.write_record(&[host, centrality.to_string()]).unwrap();
+ }
+
+ wtr.flush().unwrap();
+ }
+
+ fn host<P: AsRef<Path>>(graph: &Webgraph, output_path: P) {
+ let centrality = Self::host_centrality(graph);
+ Self::save(centrality, output_path);
+ }
+
+ fn full<P: AsRef<Path>>(graph: &Webgraph, output_path: P) {
+ let centrality = Self::full_centrality(graph);
+ Self::save(centrality, output_path);
+ }
+
+ pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ let graph = WebgraphBuilder::new(webgraph_path)
+ .with_host_graph()
+ .with_full_graph()
+ .open();
- centrality_store.append(
- graph
- .host_harmonic_centrality()
- .into_iter()
- .map(|(node, centrality)| (node.name, centrality)),
- );
+ Self::host(&graph, output_path.as_ref().join("host"));
+ Self::full(&graph, output_path.as_ref().join("full"));
}
}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 9cca690d..8a2e7309 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -85,7 +85,10 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
{
let html = Html::parse(&record.response.body, &record.request.url);
let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph
- let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();
+ let centrality = worker
+ .centrality_store
+ .get(html.url().host_without_specific_subdomains())
+ .unwrap_or_default();
let fetch_time_ms = record.metadata.fetch_time_ms as u64;
trace!("inserting webpage: {:?}", html.url());
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index e43ff525..aeef16ba 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -48,6 +48,7 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {
let mut graph = WebgraphBuilder::new(Path::new(&self.graph_base_path).join(name))
.with_host_graph()
+ .with_full_graph()
.open();
let source = match self.config {
diff --git a/src/index.rs b/src/index.rs
index 94f9c99f..3028c1a1 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -136,10 +136,10 @@ impl Index {
}
fn maybe_insert_favicon(&mut self, webpage: &Webpage) {
- if !webpage.html.is_homepage()
+ if !webpage.html.url().is_homepage()
|| self
.favicon_store
- .contains(&webpage.html.domain().to_string())
+ .contains(&webpage.html.url().domain().to_string())
{
return;
}
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index 31032212..60069883 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -21,6 +21,7 @@ use std::cell::RefCell;
use std::collections::{BinaryHeap, HashMap};
use std::path::Path;
use std::{cmp, fs};
+use tracing::info;
use graph_store::GraphStore;
@@ -45,8 +46,12 @@ pub struct Node {
impl Node {
fn into_host(self) -> Node {
+ let url = Url::from(self.name);
+
+ let host = url.host_without_specific_subdomains();
+
Node {
- name: Url::from(self.name).host().to_string(),
+ name: host.to_string(),
}
}
}
@@ -374,7 +379,7 @@ impl<S: Store> Webgraph<S> {
F: Fn(Node) -> HashMap<NodeID, usize>,
{
let nodes: Vec<_> = graph.nodes().collect();
- dbg!(&nodes);
+ info!("Found {} nodes in the graph", nodes.len());
let pb = ProgressBar::new(nodes.len() as u64);
pb.set_style(
ProgressStyle::default_bar()
@@ -403,6 +408,7 @@ impl<S: Store> Webgraph<S> {
(node, centrality)
})
+ .filter(|(_, centrality)| *centrality > 0.0)
.collect()
}
@@ -567,7 +573,7 @@ mod test {
let centrality = graph.harmonic_centrality();
assert_eq!(centrality.get(&Node::from("C")).unwrap(), &1.0);
- assert_eq!(centrality.get(&Node::from("D")).unwrap(), &0.0);
+ assert_eq!(centrality.get(&Node::from("D")), None);
assert_eq!(
(*centrality.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,
0.67
@@ -611,10 +617,27 @@ mod test {
let host_centrality = graph.host_harmonic_centrality();
assert!(
host_centrality.get(&Node::from("B.com")).unwrap()
- > host_centrality.get(&Node::from("A.com")).unwrap()
+ > host_centrality.get(&Node::from("A.com")).unwrap_or(&0.0)
);
}
+ #[test]
+ fn www_subdomain_ignored() {
+ let mut graph = WebgraphBuilder::new_memory()
+ .with_full_graph()
+ .with_host_graph()
+ .open();
+
+ graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());
+ graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());
+
+ graph.flush();
+
+ let centrality = graph.host_harmonic_centrality();
+ assert_eq!(centrality.get(&Node::from("A.com")), Some(&1.0));
+ assert_eq!(centrality.get(&Node::from("www.A.com")), None);
+ }
+
#[test]
fn merge() {
let mut graph1 = WebgraphBuilder::new_memory()
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 778a8e3e..d4f83f68 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -358,18 +358,6 @@ impl Html {
&self.url
}
- pub fn host(&self) -> &str {
- self.url.host()
- }
-
- pub fn domain(&self) -> &str {
- self.url.domain()
- }
-
- pub fn is_homepage(&self) -> bool {
- self.url.is_homepage()
- }
-
pub fn metadata(&self) -> Vec<Meta> {
let mut metas = Vec::new();
@@ -422,17 +410,17 @@ impl Html {
doc.add_text(tantivy_field, self.description().unwrap_or_default());
}
Field::Url => doc.add_text(tantivy_field, self.url()),
- Field::Host => doc.add_text(tantivy_field, self.host()),
- Field::Domain => doc.add_text(tantivy_field, self.domain()),
+ Field::Host => doc.add_text(tantivy_field, self.url().host()),
+ Field::Domain => doc.add_text(tantivy_field, self.url().domain()),
Field::DomainIfHomepage => {
- if self.is_homepage() {
- doc.add_text(tantivy_field, self.domain());
+ if self.url().is_homepage() {
+ doc.add_text(tantivy_field, self.url().domain());
} else {
doc.add_text(tantivy_field, "");
}
}
Field::IsHomepage => {
- doc.add_u64(tantivy_field, if self.is_homepage() { 1 } else { 0 });
+ doc.add_u64(tantivy_field, if self.url().is_homepage() { 1 } else { 0 });
}
Field::LastUpdated => doc.add_u64(
tantivy_field,
@@ -526,7 +514,7 @@ impl Html {
links
.into_iter()
.filter(|link| !link.host().is_empty())
- .filter(|link| link.host() != self.host())
+ .filter(|link| link.host() != self.url().host())
.unique_by(|link| link.host().to_string())
.collect()
}
@@ -681,8 +669,8 @@ mod tests {
webpage.url().to_string().as_str(),
"https://www.example.com/whatever"
);
- assert_eq!(webpage.host(), "www.example.com");
- assert_eq!(webpage.domain(), "example.com");
+ assert_eq!(webpage.url().host(), "www.example.com");
+ assert_eq!(webpage.url().domain(), "example.com");
}
#[test]
@@ -783,31 +771,31 @@ mod tests {
let raw = "";
let webpage = Html::parse(raw, "https://www.domain.co.uk");
- assert_eq!(webpage.domain(), "domain.co.uk");
+ assert_eq!(webpage.url().domain(), "domain.co.uk");
}
#[test]
fn is_homepage() {
let webpage = Html::parse("", "https://www.example.com");
- assert!(webpage.is_homepage());
+ assert!(webpage.url().is_homepage());
let webpage = Html::parse("", "https://www.example.com/");
- assert!(webpage.is_homepage());
+ assert!(webpage.url().is_homepage());
let webpage = Html::parse("", "https://www.example.com/test");
- assert!(!webpage.is_homepage());
+ assert!(!webpage.url().is_homepage());
let webpage = Html::parse("", "https://example.com/test");
- assert!(!webpage.is_homepage());
+ assert!(!webpage.url().is_homepage());
let webpage = Html::parse("", "https://example.com/");
- assert!(webpage.is_homepage());
+ assert!(webpage.url().is_homepage());
let webpage = Html::parse("", "https://example.com");
- assert!(webpage.is_homepage());
+ assert!(webpage.url().is_homepage());
let webpage = Html::parse("", "http://example.com");
- assert!(webpage.is_homepage());
+ assert!(webpage.url().is_homepage());
}
#[test]
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index 1e686b7c..b0932de9 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -89,6 +89,18 @@ impl Url {
}
}
+ pub fn subdomain(&self) -> Option<&str> {
+ if let Some(subdomain) = self.host().strip_suffix(self.domain()) {
+ if subdomain.is_empty() || subdomain == "." {
+ None
+ } else {
+ Some(&subdomain[..subdomain.len() - 1])
+ }
+ } else {
+ None
+ }
+ }
+
pub fn is_homepage(&self) -> bool {
let url = self.strip_protocol();
match url.find('/') {
@@ -170,6 +182,18 @@ impl Url {
pub fn is_valid_uri(&self) -> bool {
self.full().as_str().parse::<http::Uri>().is_ok()
}
+
+ pub(crate) fn host_without_specific_subdomains(&self) -> &str {
+ if let Some(subdomain) = self.subdomain() {
+ if subdomain == "www" {
+ self.domain()
+ } else {
+ self.host()
+ }
+ } else {
+ self.host()
+ }
+ }
}
#[cfg(test)]
@@ -239,4 +263,16 @@ mod tests {
let url: Url = "da<>ilymail.co.uk".to_string().into();
assert!(!url.is_valid_uri());
}
+
+ #[test]
+ fn subdomain() {
+ let url: Url = "https://test.example.com".to_string().into();
+ assert_eq!(url.subdomain(), Some("test"));
+
+ let url: Url = "https://test1.test2.example.com".to_string().into();
+ assert_eq!(url.subdomain(), Some("test1.test2"));
+
+ let url: Url = "https://example.com".to_string().into();
+ assert_eq!(url.subdomain(), None);
+ }
}