Skip to content

web search done right

cleanup host graph nodes

Mikkel Denkerd89149da2cf1
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 5bac3be0..271d5a38 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -108,7 +108,12 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
.centrality_store
.harmonic
.host
- .get(&html.url().host_without_specific_subdomains().to_string())
+ .get(
+ &html
+ .url()
+ .host_without_specific_subdomains_and_query()
+ .to_string(),
+ )
.unwrap_or_default();
if let Some(host_centrality_threshold) = job.host_centrality_threshold {
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 06f42f68..a31ca8e7 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -83,6 +83,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
.into_iter()
.filter(|link| matches!(link.destination.protocol(), "http" | "https"))
.filter(|link| link.source.domain() != link.destination.domain())
+ .filter(|link| !link.matches_url_regex())
{
trace!("inserting link {:?}", link);
graph.insert(
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index a3f93e91..44b8b2da 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -48,7 +48,7 @@ impl Node {
pub fn into_host(self) -> Node {
let url = Url::from(self.name);
- let host = url.host_without_specific_subdomains();
+ let host = url.host_without_specific_subdomains_and_query();
Node {
name: host.to_lowercase(),
@@ -621,4 +621,10 @@ mod test {
let n = Node::from("TEST".to_string());
assert_eq!(&n.name, "test");
}
+
+ #[test]
+ fn host_node_cleanup() {
+ let n = Node::from("https://www.example.com?test").into_host();
+ assert_eq!(&n.name, "example.com");
+ }
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 39c91d47..561a9d03 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -45,7 +45,7 @@ use self::{
region::Region,
};
-static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
+pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
Regex::new(r#"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b"#).unwrap()
});
@@ -956,6 +956,12 @@ pub struct Link {
pub text: String,
}
+impl Link {
+ pub fn matches_url_regex(&self) -> bool {
+ self.destination.matches_url_regex() && self.source.matches_url_regex()
+ }
+}
+
pub type Meta = HashMap<String, String>;
#[cfg(test)]
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index a661400d..a597a62f 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -19,6 +19,8 @@ use std::{fmt::Display, time::Duration};
use serde::{Deserialize, Serialize};
use tracing::debug;
+use super::URL_REGEX;
+
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
pub struct Url(String);
@@ -182,8 +184,8 @@ impl Url {
self.full().as_str().parse::<http::Uri>().is_ok()
}
- pub(crate) fn host_without_specific_subdomains(&self) -> &str {
- if let Some(subdomain) = self.subdomain() {
+ pub(crate) fn host_without_specific_subdomains_and_query(&self) -> &str {
+ let res = if let Some(subdomain) = self.subdomain() {
if subdomain == "www" {
self.domain()
} else {
@@ -191,6 +193,12 @@ impl Url {
}
} else {
self.site()
+ };
+
+ if let Some(query_begin) = res.find('?') {
+ &res[..query_begin]
+ } else {
+ res
}
}
@@ -210,6 +218,10 @@ impl Url {
full
}
}
+
+ pub fn matches_url_regex(&self) -> bool {
+ URL_REGEX.is_match(&self.full())
+ }
}
#[cfg(test)]
@@ -232,12 +244,14 @@ mod tests {
assert_eq!(url.domain(), "dailymail.co.uk");
assert_eq!(url.site(), "dailymail.co.uk");
assert_eq!(url.full().as_str(), "https://dailymail.co.uk");
+ assert!(url.matches_url_regex());
}
#[test]
fn full() {
let url: Url = "https://example.com".to_string().into();
assert_eq!(url.full().as_str(), "https://example.com");
+ assert!(url.matches_url_regex());
let url: Url = "http://example.com".to_string().into();
assert_eq!(url.full().as_str(), "http://example.com");