web search done right
remove fragments from page graph nodes
Mikkel Denker90653c51cc5d
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 9cddf89a..855c2dc4 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -157,7 +157,9 @@ pub mod host {).await{- Ok(Some(node)) => Ok(Json(KnowsHost::Known { host: node.name })),+ Ok(Some(node)) => Ok(Json(KnowsHost::Known {+ host: node.as_str().to_string(),+ })),Err(err) => {tracing::error!("Failed to send request to webgraph: {}", err);Ok(Json(KnowsHost::Unknown))diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 08d434d2..94416ce6 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -83,7 +83,7 @@ impl CrawlPlanner {for host in hosts {let node = self.host_graph.id2node(host).unwrap();- if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {+ if let Ok(url) = Url::parse(&format!("http://{}", node.as_str())) {let domain = Domain::from(url);domains.entry(domain).or_default().push(*host);}@@ -156,14 +156,14 @@ impl CrawlPlanner {total_wander_budget += wander_budget;total_schedule_budget += schedule_budget;- let host_name = self.host_graph.id2node(host).unwrap().name.clone();+ let host_name = self.host_graph.id2node(host).unwrap().as_str().to_string();let before = urls.len();urls.extend(pages.into_iter().filter_map(|(id, score)| self.page_graph.id2node(&id).map(|n| (n, score)))- .map(|(n, score)| (n.name, score))+ .map(|(n, score)| (n.as_str().to_string(), score)).filter_map(|(n, score)| {Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))})diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rsindex b9e3a893..d3f7bdd0 100644--- a/crates/core/src/entrypoint/centrality.rs+++ b/crates/core/src/entrypoint/centrality.rs@@ -40,7 +40,7 @@ fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {data.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));let mut wtr = csv::Writer::from_writer(csv_file);for (node, centrality) in data {- wtr.write_record(&[node.name, centrality.to_string()])+ wtr.write_record(&[node.as_str().to_string(), centrality.to_string()]).unwrap();}wtr.flush().unwrap();diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex d9a17cd4..db87f7c0 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -92,7 +92,7 @@ impl Message<WebGraphService> for SimilarHosts {let urls = similar_hosts.iter()- .filter_map(|s| Url::parse(&("http://".to_string() + s.node.name.as_str())).ok())+ .filter_map(|s| Url::parse(&("http://".to_string() + s.node.as_str())).ok()).collect_vec();let descriptions = server.searcher.get_homepage_descriptions(&urls).await;@@ -100,12 +100,12 @@ impl Message<WebGraphService> for SimilarHosts {let similar_hosts = similar_hosts.into_iter().map(|site| {- let description = Url::parse(&("http://".to_string() + site.node.name.as_str()))+ let description = Url::parse(&("http://".to_string() + site.node.as_str())).ok().and_then(|url| descriptions.get(&url).cloned());ScoredHost {- host: site.node.name,+ host: site.node.as_str().to_string(),score: site.score,description,}diff --git a/crates/core/src/feed/scheduler.rs b/crates/core/src/feed/scheduler.rsindex 6100da9a..d3598bae 100644--- a/crates/core/src/feed/scheduler.rs+++ b/crates/core/src/feed/scheduler.rs@@ -115,7 +115,7 @@ pub fn schedule(for host in top_hosts {let host = host_graph.id2node(&host).unwrap();- let url = Url::parse(&format!("http://{}", host.name));+ let url = Url::parse(&format!("http://{}", host.as_str()));if url.is_err() {continue;@@ -127,7 +127,7 @@ pub fn schedule(all_feeds.entry(domain).or_insert(HashSet::new())- .extend(index.search(&host.name).unwrap().into_iter());+ .extend(index.search(host.as_str()).unwrap().into_iter());}let mut splits = Vec::new();diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex 54797472..a5d46ad0 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -89,7 +89,7 @@ impl SimilarHostsFinder {let domains = nodes.iter().filter_map(|node| {- Url::parse(&format!("http://{}", &node.name))+ Url::parse(&format!("http://{}", &node.as_str())).ok().and_then(|url| url.root_domain().map(|d| d.to_string()))})@@ -161,7 +161,7 @@ impl SimilarHostsFinder {}).filter_map(|ScoredNodeID { node_id, score }| {let node = self.webgraph.id2node(&node_id).unwrap();- match Url::parse(&format!("http://{}", &node.name))+ match Url::parse(&format!("http://{}", &node.as_str())).ok().and_then(|url| url.root_domain().map(|s| s.to_string())){diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 7cc19220..25526f93 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -171,7 +171,7 @@ pub struct FullEdge {)]#[serde(rename_all = "camelCase")]pub struct Node {- pub name: String,+ name: String,}impl Node {@@ -193,6 +193,10 @@ impl Node {}}+ pub fn as_str(&self) -> &str {+ self.name.as_str()+ }+pub fn id(&self) -> NodeID {let digest = md5::compute(self.name.as_bytes());u128::from_le_bytes(*digest).into()@@ -232,29 +236,7 @@ impl From<Url> for Node {pub fn normalize_url(url: &Url) -> String {let mut url = url.clone();- let allowed_queries: Vec<_> = url- .query_pairs()- .filter(|(key, _)| {- !key.starts_with("utm_")- && !key.starts_with("fbclid")- && !key.starts_with("gclid")- && !key.starts_with("msclkid")- })- .map(|(key, value)| (key.to_string(), value.to_string()))- .collect();-- {- let mut queries = url.query_pairs_mut();- queries.clear();-- if !allowed_queries.is_empty() {- queries.extend_pairs(allowed_queries);- }- }-- if url.query().unwrap_or_default().is_empty() {- url.set_query(None);- }+ url.normalize();let scheme = url.scheme();let mut normalized = url@@ -1137,4 +1119,13 @@ mod test {]);}++ #[test]+ fn test_node_normalized() {+ let n = Node::from("http://www.example.com/abc");+ assert_eq!(n.as_str(), "example.com/abc");++ let n = Node::from("http://www.example.com/abc#123");+ assert_eq!(n.as_str(), "example.com/abc");+ }}diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 59175ad4..0b93b31e 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -65,8 +65,7 @@ impl Html {pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {let root = kuchiki::parse_html().one(html);- let mut url = Url::parse(url)?;- url.set_fragment(None); // remove fragment (e.g. #comments+ let url = Url::parse(url)?;let mut res = Self {root,@@ -77,31 +76,13 @@ impl Html {robots: None,};- let queries: Vec<_> = res- .url- .query_pairs()- .filter(|(key, _)| !key.starts_with("utm_"))- .map(|(key, value)| (key.to_string(), value.to_string()))- .collect();-- {- let mut query_mut = res.url.query_pairs_mut();- query_mut.clear();- if !queries.is_empty() {- query_mut.extend_pairs(queries);- }- }-if let Some(canonical) = res.canonical_url() {if canonical.root_domain() == res.url.root_domain() {res.url = canonical;}}- if res.url.query().unwrap_or_default().is_empty() {- res.url.set_query(None);- }-+ res.url.normalize();res.robots = res.parse_robots_meta();Ok(res)@@ -1126,7 +1107,7 @@ mod tests {<title data-rh="true">Russian Ships Near Data Cables Are Too Close for U.S. Comfort - The New York Times</title><meta data-rh="true" name="robots" content="noarchive, max-image-preview:large"/><meta data-rh="true" name="description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="og:url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Russian Ships Near Data Cables Are Too Close for U.S. Comfort (Published 2015)"/><meta data-rh="true" property="og:image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-facebookJumbo.jpg?year=2015&h=550&w=1050&s=ceeb5555ae80dd2e2fe05553973d3948298bcba76c8d62a24907ae21e61140a7&k=ZQJBKqZ0VN"/><meta data-rh="true" property="og:image:alt" content="Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year."/><meta data-rh="true" property="og:description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="twitter:url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="twitter:title" content="Russian Ships Near Data Cables Are Too Close for U.S. Comfort (Published 2015)"/><meta data-rh="true" property="twitter:description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="twitter:image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-articleLarge.jpg?year=2015&h=400&w=600&s=1c25983f231db27a32cbc8063f067135d143de5f7c2d671b5798f0b76507c68f&k=ZQJBKqZ0VN&tw=1"/><meta data-rh="true" property="twitter:image:alt" content="Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year."/><meta data-rh="true" property="twitter:card" content="summary_large_image"/> <link data-rh="true" rel="canonical" href="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><link data-rh="true" rel="alternate" href="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><link data-rh="true" rel="alternate" type="application/json+oembed" href="https://www.nytimes.com/svc/oembed/json/?url=https%3A%2F%2Fwww.nytimes.com%2F2015%2F10%2F26%2Fworld%2Feurope%2Frussian-presence-near-undersea-cables-concerns-us.html" title="Russian Ships Near Data Cables Are Too Close for U.S. Comfort"/> <script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"NewsArticle","description":"Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S.","image":{"@context":"http://schema.org","@type":"ImageObject","url":"https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-superJumbo.jpg","height":1365,"width":2048,"contentUrl":"https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-superJumbo.jpg","caption":"Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year.","creditText":"Brendan Smialowski/Agence France-Presse — Getty Images"},"mainEntityOfPage":"https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html","url":"https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html","author":[{"@context":"http://schema.org","@type":"Person","url":"http://www.nytimes.com/by/david-e-sanger","name":"David E. Sanger"},{"@context":"http://schema.org","@type":"Person","url":"http://www.nytimes.com/by/eric-schmitt","name":"Eric Schmitt"}],"dateModified":"2015-10-25T17:02:34.000Z","datePublished":"2015-10-25T17:02:35.000Z","headline":"Russian Ships Near Data Cables Are Too Close for U.S. Comfort","alternativeHeadline":"Russian Ships Near Data Cables Are Too Close for U.S. Comfort","publisher":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"hasPart":{"@type":"WebPageElement","isAccessibleForFree":false,"cssSelector":".meteredContent"},"copyrightHolder":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"sourceOrganization":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"copyrightYear":2024,"isAccessibleForFree":false,"isPartOf":{"@type":["CreativeWork","Product"],"name":"The New York Times","productID":"nytimes.com:basic"}}</script><script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"NewsMediaOrganization","name":"The New York Times","logo":{"@context":"http://schema.org","@type":"ImageObject","url":"https://static01.nyt.com/images/icons/t_logo_291_black.png","height":291,"width":291,"contentUrl":"https://static01.nyt.com/images/icons/t_logo_291_black.png","creditText":"The New York Times"},"url":"https://www.nytimes.com/","@id":"https://www.nytimes.com/#publisher","diversityPolicy":"https://www.nytco.com/company/diversity-and-inclusion/","ethicsPolicy":"https://www.nytco.com/company/standards-ethics/","masthead":"https://www.nytimes.com/interactive/2023/01/28/admin/the-new-york-times-masthead.html","foundingDate":"1851-09-18","sameAs":"https://en.wikipedia.org/wiki/The_New_York_Times"}</script><script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"BreadcrumbList","itemListElement":[{"@context":"http://schema.org","@type":"ListItem","name":"World","position":1,"item":"https://www.nytimes.com/section/world"},{"@context":"http://schema.org","@type":"ListItem","name":"Europe","position":2,"item":"https://www.nytimes.com/section/world/europe"}]}</script><meta data-rh="true" property="article:published_time" content="2015-10-25T17:02:35.000Z"/><meta data-rh="true" property="article:modified_time" content="2015-10-25T17:02:34.000Z"/><meta data-rh="true" http-equiv="Content-Language" content="en"/><meta data-rh="true" name="articleid" content="100000003995797"/><meta data-rh="true" name="nyt_uri" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" name="pubp_event_id" content="pubp://event/7219283749a6486e8ec2777e0840ab41"/><meta data-rh="true" name="image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-facebookJumbo.jpg?year=2015&h=550&w=1050&s=ceeb5555ae80dd2e2fe05553973d3948298bcba76c8d62a24907ae21e61140a7&k=ZQJBKqZ0VN"/><meta data-rh="true" name="byl" content="By David E. Sanger and Eric Schmitt"/><meta data-rh="true" name="news_keywords" content=",Russia,,Submarines and Submersibles,Wire and Cable,Computers and the Internet,Tech Industry,US,Homeland Security,Spying and Intelligence Agencies,US Military,Military,Mark Ferguson"/><meta data-rh="true" name="pdate" content="20151025"/><meta data-rh="true" property="article:section" content="World"/><meta data-rh="true" property="article:author" content="http://www.nytimes.com/by/david-e-sanger"/><meta data-rh="true" property="article:author" content="http://www.nytimes.com/by/eric-schmitt"/><meta data-rh="true" property="article:tag" content="Schmitt, Eric"/><meta data-rh="true" property="article:tag" content="Russia"/><meta data-rh="true" property="article:tag" content="Sanger, David E"/><meta data-rh="true" property="article:tag" content="Submarines and Submersibles"/><meta data-rh="true" property="article:tag" content="Wire and Cable"/><meta data-rh="true" property="article:tag" content="Computers and the Internet"/><meta data-rh="true" property="article:tag" content="United States"/><meta data-rh="true" property="article:tag" content="Homeland Security Department"/><meta data-rh="true" property="article:tag" content="Espionage and Intelligence Services"/><meta data-rh="true" property="article:tag" content="United States Defense and Military Forces"/><meta data-rh="true" property="article:tag" content="Defense and Military Forces"/><meta data-rh="true" property="article:tag" content="Ferguson, Mark E III"/><meta data-rh="true" property="article:opinion" content="false"/><meta data-rh="true" property="article:content_tier" content="metered"/><meta data-rh="true" name="CG" content="world"/><meta data-rh="true" name="SCG" content="europe"/><meta data-rh="true" name="CN" content=""/><meta data-rh="true" name="CT" content=""/><meta data-rh="true" name="PT" content="article"/><meta data-rh="true" name="PST" content="News"/><meta data-rh="true" name="url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" name="msapplication-starturl" content="https://www.nytimes.com"/><meta data-rh="true" property="al:android:url" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" property="al:android:package" content="com.nytimes.android"/><meta data-rh="true" property="al:android:app_name" content="NYTimes"/><meta data-rh="true" name="twitter:app:name:googleplay" content="NYTimes"/><meta data-rh="true" name="twitter:app:id:googleplay" content="com.nytimes.android"/><meta data-rh="true" name="twitter:app:url:googleplay" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" property="al:iphone:url" content="nytimes://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="al:iphone:app_store_id" content="284862083"/><meta data-rh="true" property="al:iphone:app_name" content="NYTimes"/><meta data-rh="true" property="al:ipad:url" content="nytimes://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="al:ipad:app_store_id" content="357066198"/><meta data-rh="true" property="al:ipad:app_name" content="NYTimes"/>-+</head><body>TESTdiff --git a/crates/core/src/webpage/url_ext.rs b/crates/core/src/webpage/url_ext.rsindex 48bd5034..83c2281c 100644--- a/crates/core/src/webpage/url_ext.rs+++ b/crates/core/src/webpage/url_ext.rs@@ -32,6 +32,7 @@ pub trait UrlExt {fn icann_domain(&self) -> Option<&str>;fn root_domain(&self) -> Option<&str>;fn normalized_host(&self) -> Option<&str>;+ fn normalize(&mut self);fn subdomain(&self) -> Option<&str>;fn is_homepage(&self) -> bool;fn tld(&self) -> Option<&str>;@@ -55,6 +56,33 @@ impl UrlExt for url::Url {self.host_str().map(|host| host.trim_start_matches("www."))}+ fn normalize(&mut self) {+ self.set_fragment(None); // remove fragment (e.g. #comments++ let queries: Vec<_> = self+ .query_pairs()+ .filter(|(key, _)| {+ !key.starts_with("utm_")+ && !key.starts_with("fbclid")+ && !key.starts_with("gclid")+ && !key.starts_with("msclkid")+ })+ .map(|(key, value)| (key.to_string(), value.to_string()))+ .collect();++ {+ let mut query_mut = self.query_pairs_mut();+ query_mut.clear();+ if !queries.is_empty() {+ query_mut.extend_pairs(queries);+ }+ }++ if self.query().unwrap_or_default().is_empty() {+ self.set_query(None);+ }+ }+fn subdomain(&self) -> Option<&str> {let domain = self.root_domain()?;let host = self.host_str()?;