web search done right
cleanup host graph nodes
Mikkel Denkerd89149da2cf1
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 5bac3be0..271d5a38 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -108,7 +108,12 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {.centrality_store.harmonic.host- .get(&html.url().host_without_specific_subdomains().to_string())+ .get(+ &html+ .url()+ .host_without_specific_subdomains_and_query()+ .to_string(),+ ).unwrap_or_default();if let Some(host_centrality_threshold) = job.host_centrality_threshold {diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 06f42f68..a31ca8e7 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -83,6 +83,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {.into_iter().filter(|link| matches!(link.destination.protocol(), "http" | "https")).filter(|link| link.source.domain() != link.destination.domain())+ .filter(|link| !link.matches_url_regex()){trace!("inserting link {:?}", link);graph.insert(diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex a3f93e91..44b8b2da 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -48,7 +48,7 @@ impl Node {pub fn into_host(self) -> Node {let url = Url::from(self.name);- let host = url.host_without_specific_subdomains();+ let host = url.host_without_specific_subdomains_and_query();Node {name: host.to_lowercase(),@@ -621,4 +621,10 @@ mod test {let n = Node::from("TEST".to_string());assert_eq!(&n.name, "test");}++ #[test]+ fn host_node_cleanup() {+ let n = Node::from("https://www.example.com?test").into_host();+ assert_eq!(&n.name, "example.com");+ }}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 39c91d47..561a9d03 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -45,7 +45,7 @@ use self::{region::Region,};-static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {+pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {Regex::new(r#"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b"#).unwrap()});@@ -956,6 +956,12 @@ pub struct Link {pub text: String,}+impl Link {+ pub fn matches_url_regex(&self) -> bool {+ self.destination.matches_url_regex() && self.source.matches_url_regex()+ }+}+pub type Meta = HashMap<String, String>;#[cfg(test)]diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex a661400d..a597a62f 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -19,6 +19,8 @@ use std::{fmt::Display, time::Duration};use serde::{Deserialize, Serialize};use tracing::debug;+use super::URL_REGEX;+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]pub struct Url(String);@@ -182,8 +184,8 @@ impl Url {self.full().as_str().parse::<http::Uri>().is_ok()}- pub(crate) fn host_without_specific_subdomains(&self) -> &str {- if let Some(subdomain) = self.subdomain() {+ pub(crate) fn host_without_specific_subdomains_and_query(&self) -> &str {+ let res = if let Some(subdomain) = self.subdomain() {if subdomain == "www" {self.domain()} else {@@ -191,6 +193,12 @@ impl Url {}} else {self.site()+ };++ if let Some(query_begin) = res.find('?') {+ &res[..query_begin]+ } else {+ res}}@@ -210,6 +218,10 @@ impl Url {full}}++ pub fn matches_url_regex(&self) -> bool {+ URL_REGEX.is_match(&self.full())+ }}#[cfg(test)]@@ -232,12 +244,14 @@ mod tests {assert_eq!(url.domain(), "dailymail.co.uk");assert_eq!(url.site(), "dailymail.co.uk");assert_eq!(url.full().as_str(), "https://dailymail.co.uk");+ assert!(url.matches_url_regex());}#[test]fn full() {let url: Url = "https://example.com".to_string().into();assert_eq!(url.full().as_str(), "https://example.com");+ assert!(url.matches_url_regex());let url: Url = "http://example.com".to_string().into();assert_eq!(url.full().as_str(), "http://example.com");