web search done right
count 'www' and no-subdomain as the same in webgraph. Also other small bugfixes in webgraph
Mikkel Denker1a99076eae23
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.tomlnew file mode 100644index 00000000..d3e2d932--- /dev/null+++ b/configs/webgraph/local.toml@@ -0,0 +1,10 @@+limit_warc_files = 3+graph_base_path = "data/webgraph"++[warc_source]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "warc.paths"+# type = "Local"+# folder = "./data/warc_files"+# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]\ No newline at end of filediff --git a/configs/webgraph/local/local.toml b/configs/webgraph/local/local.tomldeleted file mode 100644index 6b23e4aa..00000000--- a/configs/webgraph/local/local.toml+++ /dev/null@@ -1,7 +0,0 @@-limit_warc_files = 10-graph_base_path = "data/webgraph"--[warc_source]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"-warc_paths_file = "warc.paths"\ No newline at end of filediff --git a/configs/webgraph/local/master.toml b/configs/webgraph/local/master.tomldeleted file mode 100644index a7192ced..00000000--- a/configs/webgraph/local/master.toml+++ /dev/null@@ -1,6 +0,0 @@-workers = ["0.0.0.0:1338"]--[warc_source]-type = "local"-folder = "./warc_files"-names = ["small1.warc.gz", "small2.warc.gz"]diff --git a/configs/webgraph/local/profile.toml b/configs/webgraph/profile.tomlsimilarity index 100%rename from configs/webgraph/local/profile.tomlrename to configs/webgraph/profile.tomldiff --git a/justfile b/justfileindex 889a405d..926b5c17 100644--- a/justfile+++ b/justfile@@ -7,10 +7,14 @@cargo flamegraph --root -- indexer local configs/indexer/local.toml@worker:- cargo run --release -- configs/webgraph/worker.toml+ cargo run --release -- webgraph worker configs/webgraph/worker.toml@master:- cargo run --release -- configs/webgraph/master.toml+ cargo run --release -- webgraph master configs/webgraph/master.toml++@webgraph:+ rm -rf data/webgraph+ cargo run --release -- webgraph local configs/webgraph/local.toml@frontend:cargo watch -x 'run -- frontend data/index data/queries_us.csv data/entity'diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rsindex baad8828..d3e19278 100644--- a/src/entrypoint/centrality.rs+++ b/src/entrypoint/centrality.rs@@ -14,23 +14,73 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;+use std::{collections::HashMap, fs::File, path::Path};-use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};+use crate::{+ ranking::centrality_store::CentralityStore,+ webgraph::{Webgraph, WebgraphBuilder},+};pub struct Centrality {}impl Centrality {- pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {- let graph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();+ fn host_centrality(graph: &Webgraph) -> HashMap<String, f64> {+ graph+ .host_harmonic_centrality()+ .into_iter()+ .map(|(node, centrality)| (node.name, centrality))+ .collect()+ }++ fn full_centrality(graph: &Webgraph) -> HashMap<String, f64> {+ graph+ .harmonic_centrality()+ .into_iter()+ .map(|(node, centrality)| (node.name, centrality))+ .collect()+ }++ fn save<P: AsRef<Path>>(centrality: HashMap<String, f64>, output_path: P) {+ let mut centrality_store = CentralityStore::new(output_path.as_ref());++ centrality_store.append(centrality.clone().into_iter());++ let mut centralities: Vec<_> = centrality.into_iter().collect();+ centralities+ .sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));++ let csv_file = File::options()+ .write(true)+ .create(true)+ .truncate(true)+ .open(output_path.as_ref().join("data.csv"))+ .unwrap();+ let mut wtr = csv::Writer::from_writer(csv_file);- let mut centrality_store = CentralityStore::new(output_path);+ for (host, centrality) in centralities {+ wtr.write_record(&[host, centrality.to_string()]).unwrap();+ }++ wtr.flush().unwrap();+ }++ fn host<P: AsRef<Path>>(graph: &Webgraph, output_path: P) {+ let centrality = Self::host_centrality(graph);+ Self::save(centrality, output_path);+ }++ fn full<P: AsRef<Path>>(graph: &Webgraph, output_path: P) {+ let centrality = Self::full_centrality(graph);+ Self::save(centrality, output_path);+ }++ pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ let graph = WebgraphBuilder::new(webgraph_path)+ .with_host_graph()+ .with_full_graph()+ .open();- centrality_store.append(- graph- .host_harmonic_centrality()- .into_iter()- .map(|(node, centrality)| (node.name, centrality)),- );+ Self::host(&graph, output_path.as_ref().join("host"));+ Self::full(&graph, output_path.as_ref().join("full"));}}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 9cca690d..8a2e7309 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -85,7 +85,10 @@ impl Map<IndexingWorker, FrozenIndex> for Job {{let html = Html::parse(&record.response.body, &record.request.url);let backlinks: Vec<Link> = Vec::new(); // TODO: lookup backlinks in full webgraph- let centrality = worker.centrality_store.get(html.host()).unwrap_or_default();+ let centrality = worker+ .centrality_store+ .get(html.url().host_without_specific_subdomains())+ .unwrap_or_default();let fetch_time_ms = record.metadata.fetch_time_ms as u64;trace!("inserting webpage: {:?}", html.url());diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex e43ff525..aeef16ba 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -48,6 +48,7 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {let mut graph = WebgraphBuilder::new(Path::new(&self.graph_base_path).join(name)).with_host_graph()+ .with_full_graph().open();let source = match self.config {diff --git a/src/index.rs b/src/index.rsindex 94f9c99f..3028c1a1 100644--- a/src/index.rs+++ b/src/index.rs@@ -136,10 +136,10 @@ impl Index {}fn maybe_insert_favicon(&mut self, webpage: &Webpage) {- if !webpage.html.is_homepage()+ if !webpage.html.url().is_homepage()|| self.favicon_store- .contains(&webpage.html.domain().to_string())+ .contains(&webpage.html.url().domain().to_string()){return;}diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 31032212..60069883 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -21,6 +21,7 @@ use std::cell::RefCell;use std::collections::{BinaryHeap, HashMap};use std::path::Path;use std::{cmp, fs};+use tracing::info;use graph_store::GraphStore;@@ -45,8 +46,12 @@ pub struct Node {impl Node {fn into_host(self) -> Node {+ let url = Url::from(self.name);++ let host = url.host_without_specific_subdomains();+Node {- name: Url::from(self.name).host().to_string(),+ name: host.to_string(),}}}@@ -374,7 +379,7 @@ impl<S: Store> Webgraph<S> {F: Fn(Node) -> HashMap<NodeID, usize>,{let nodes: Vec<_> = graph.nodes().collect();- dbg!(&nodes);+ info!("Found {} nodes in the graph", nodes.len());let pb = ProgressBar::new(nodes.len() as u64);pb.set_style(ProgressStyle::default_bar()@@ -403,6 +408,7 @@ impl<S: Store> Webgraph<S> {(node, centrality)})+ .filter(|(_, centrality)| *centrality > 0.0).collect()}@@ -567,7 +573,7 @@ mod test {let centrality = graph.harmonic_centrality();assert_eq!(centrality.get(&Node::from("C")).unwrap(), &1.0);- assert_eq!(centrality.get(&Node::from("D")).unwrap(), &0.0);+ assert_eq!(centrality.get(&Node::from("D")), None);assert_eq!((*centrality.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,0.67@@ -611,10 +617,27 @@ mod test {let host_centrality = graph.host_harmonic_centrality();assert!(host_centrality.get(&Node::from("B.com")).unwrap()- > host_centrality.get(&Node::from("A.com")).unwrap()+ > host_centrality.get(&Node::from("A.com")).unwrap_or(&0.0));}+ #[test]+ fn www_subdomain_ignored() {+ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());+ graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());++ graph.flush();++ let centrality = graph.host_harmonic_centrality();+ assert_eq!(centrality.get(&Node::from("A.com")), Some(&1.0));+ assert_eq!(centrality.get(&Node::from("www.A.com")), None);+ }+#[test]fn merge() {let mut graph1 = WebgraphBuilder::new_memory()diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 778a8e3e..d4f83f68 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -358,18 +358,6 @@ impl Html {&self.url}- pub fn host(&self) -> &str {- self.url.host()- }-- pub fn domain(&self) -> &str {- self.url.domain()- }-- pub fn is_homepage(&self) -> bool {- self.url.is_homepage()- }-pub fn metadata(&self) -> Vec<Meta> {let mut metas = Vec::new();@@ -422,17 +410,17 @@ impl Html {doc.add_text(tantivy_field, self.description().unwrap_or_default());}Field::Url => doc.add_text(tantivy_field, self.url()),- Field::Host => doc.add_text(tantivy_field, self.host()),- Field::Domain => doc.add_text(tantivy_field, self.domain()),+ Field::Host => doc.add_text(tantivy_field, self.url().host()),+ Field::Domain => doc.add_text(tantivy_field, self.url().domain()),Field::DomainIfHomepage => {- if self.is_homepage() {- doc.add_text(tantivy_field, self.domain());+ if self.url().is_homepage() {+ doc.add_text(tantivy_field, self.url().domain());} else {doc.add_text(tantivy_field, "");}}Field::IsHomepage => {- doc.add_u64(tantivy_field, if self.is_homepage() { 1 } else { 0 });+ doc.add_u64(tantivy_field, if self.url().is_homepage() { 1 } else { 0 });}Field::LastUpdated => doc.add_u64(tantivy_field,@@ -526,7 +514,7 @@ impl Html {links.into_iter().filter(|link| !link.host().is_empty())- .filter(|link| link.host() != self.host())+ .filter(|link| link.host() != self.url().host()).unique_by(|link| link.host().to_string()).collect()}@@ -681,8 +669,8 @@ mod tests {webpage.url().to_string().as_str(),"https://www.example.com/whatever");- assert_eq!(webpage.host(), "www.example.com");- assert_eq!(webpage.domain(), "example.com");+ assert_eq!(webpage.url().host(), "www.example.com");+ assert_eq!(webpage.url().domain(), "example.com");}#[test]@@ -783,31 +771,31 @@ mod tests {let raw = "";let webpage = Html::parse(raw, "https://www.domain.co.uk");- assert_eq!(webpage.domain(), "domain.co.uk");+ assert_eq!(webpage.url().domain(), "domain.co.uk");}#[test]fn is_homepage() {let webpage = Html::parse("", "https://www.example.com");- assert!(webpage.is_homepage());+ assert!(webpage.url().is_homepage());let webpage = Html::parse("", "https://www.example.com/");- assert!(webpage.is_homepage());+ assert!(webpage.url().is_homepage());let webpage = Html::parse("", "https://www.example.com/test");- assert!(!webpage.is_homepage());+ assert!(!webpage.url().is_homepage());let webpage = Html::parse("", "https://example.com/test");- assert!(!webpage.is_homepage());+ assert!(!webpage.url().is_homepage());let webpage = Html::parse("", "https://example.com/");- assert!(webpage.is_homepage());+ assert!(webpage.url().is_homepage());let webpage = Html::parse("", "https://example.com");- assert!(webpage.is_homepage());+ assert!(webpage.url().is_homepage());let webpage = Html::parse("", "http://example.com");- assert!(webpage.is_homepage());+ assert!(webpage.url().is_homepage());}#[test]diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex 1e686b7c..b0932de9 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -89,6 +89,18 @@ impl Url {}}+ pub fn subdomain(&self) -> Option<&str> {+ if let Some(subdomain) = self.host().strip_suffix(self.domain()) {+ if subdomain.is_empty() || subdomain == "." {+ None+ } else {+ Some(&subdomain[..subdomain.len() - 1])+ }+ } else {+ None+ }+ }+pub fn is_homepage(&self) -> bool {let url = self.strip_protocol();match url.find('/') {@@ -170,6 +182,18 @@ impl Url {pub fn is_valid_uri(&self) -> bool {self.full().as_str().parse::<http::Uri>().is_ok()}++ pub(crate) fn host_without_specific_subdomains(&self) -> &str {+ if let Some(subdomain) = self.subdomain() {+ if subdomain == "www" {+ self.domain()+ } else {+ self.host()+ }+ } else {+ self.host()+ }+ }}#[cfg(test)]@@ -239,4 +263,16 @@ mod tests {let url: Url = "da<>ilymail.co.uk".to_string().into();assert!(!url.is_valid_uri());}++ #[test]+ fn subdomain() {+ let url: Url = "https://test.example.com".to_string().into();+ assert_eq!(url.subdomain(), Some("test"));++ let url: Url = "https://test1.test2.example.com".to_string().into();+ assert_eq!(url.subdomain(), Some("test1.test2"));++ let url: Url = "https://example.com".to_string().into();+ assert_eq!(url.subdomain(), None);+ }}