web search done right
minor refactoring of url functions into a shared struct
Mikkel Denkerfaddb613aa1e
diff --git a/justfile b/justfileindex 3657d8d8..b22e9d7e 100644--- a/justfile+++ b/justfile@@ -13,7 +13,6 @@@local:rm -rf data/index/- cargo build --release- ./target/release/cuely indexer local configs/indexer/local.toml+ cargo run --release -- indexer local configs/indexer/local.tomlmv data/index/CC-MAIN-20220116093137-20220116123137-00049.warc.gz/* data/index/- ./target/release/cuely frontend data/index/ queries_us.csv\ No newline at end of file+ just frontenddiff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 7493c3f9..b509d2a7 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -17,7 +17,7 @@ use crate::{mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},warc::WarcFile,webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},- webpage::{self, Html},+ webpage::Html,HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,};use serde::{Deserialize, Serialize};@@ -61,11 +61,8 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {for link in webpage.links().into_iter()- .filter(|link| {- link.destination.starts_with("http://")- || link.destination.starts_with("https://")- })- .filter(|link| webpage::domain(&link.source) != webpage::domain(&link.destination))+ .filter(|link| matches!(link.destination.protocol(), "http" | "https"))+ .filter(|link| link.source.domain() != link.destination.domain()){trace!("inserting link {:?}", link);graph.insert(diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex c9bbdc34..0d133cc6 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -18,7 +18,7 @@ use axum::Extension;use crate::{index::RetrievedWebpage,- webpage::{strip_protocol, strip_query},+ webpage::{strip_protocol, strip_query, Url},};use std::collections::HashMap;use std::sync::Arc;@@ -34,17 +34,6 @@ pub fn html_escape(s: &str) -> String {.collect()}-pub fn find_protocol(url: &str) -> &'_ str {- let mut start_host = 0;- if url.starts_with("http://") || url.starts_with("https://") {- start_host = url- .find(':')- .expect("It was checked that url starts with protocol");- }-- &url[..start_host]-}-pub struct DisplayedWebpage {pub title: String,pub url: String,@@ -64,7 +53,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();}- let protocol = find_protocol(&pretty_url).to_string() + "://";+ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";pretty_url = strip_protocol(&pretty_url).replace('/', " › ");pretty_url = protocol + &pretty_url;@@ -121,14 +110,3 @@ pub async fn route(};HtmlTemplate(template)}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_find_protocol() {- assert_eq!(find_protocol("https://example.com"), "https");- assert_eq!(find_protocol("http://example.com"), "http");- }-}diff --git a/src/index.rs b/src/index.rsindex 3a08e12a..186ff713 100644--- a/src/index.rs+++ b/src/index.rs@@ -473,8 +473,8 @@ mod tests {),"https://www.b.com",vec![Link {- source: "https://www.a.com".to_string(),- destination: "https://www.b.com".to_string(),+ source: "https://www.a.com".to_string().into(),+ destination: "https://www.b.com".to_string().into(),text: "B site is great".to_string(),}],1.0,diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 699f869d..12b12652 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -83,8 +83,8 @@ mod tests {),"https://www.b.com",vec![Link {- source: "https://www.a.com".to_string(),- destination: "https://www.b.com".to_string(),+ source: "https://www.a.com".to_string().into(),+ destination: "https://www.b.com".to_string().into(),text: "B site is great".to_string(),}],5.0,diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 3a08f6d2..8a3d89dc 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{Error, Result};use logos::{Lexer, Logos};-use std::collections::HashMap;+use std::{collections::HashMap, fmt::Display};mod just_text;mod lexer;@@ -24,6 +24,17 @@ use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};use self::{just_text::JustText, lexer::Token};+pub fn find_protocol(url: &str) -> &'_ str {+ let mut start_host = 0;+ if url.starts_with("http://") || url.starts_with("https://") {+ start_host = url+ .find(':')+ .expect("It was checked that url starts with protocol");+ }++ &url[..start_host]+}+pub fn strip_protocol(url: &str) -> &'_ str {let mut start_host = 0;if url.starts_with("http://") || url.starts_with("https://") {@@ -177,7 +188,7 @@ impl<'a> Webpage<'a> {pub struct Html<'a> {raw: &'a str,tokens: Lexer<'a, Token<'a>>,- url: String,+ url: Url,}impl<'a> Html<'a> {@@ -187,7 +198,7 @@ impl<'a> Html<'a> {Self {raw: html,tokens,- url: url.to_string(),+ url: Url(url.to_string()),}}@@ -212,7 +223,7 @@ impl<'a> Html<'a> {if let Some(dest) = attributes.get("href") {links.push(Link {source: self.url.clone(),- destination: dest.to_string(),+ destination: dest.to_string().into(),text,})}@@ -228,7 +239,7 @@ impl<'a> Html<'a> {if let Some(dest) = tag.attributes().get("href") {links.push(Link {source: self.url.clone(),- destination: dest.to_string(),+ destination: dest.to_string().into(),text: String::new(),})}@@ -298,20 +309,20 @@ impl<'a> Html<'a> {title}- pub fn url(&self) -> &str {+ pub fn url(&self) -> &Url {&self.url}pub fn host(&self) -> &str {- host(self.url())+ self.url.host()}pub fn domain(&self) -> &str {- domain(self.url())+ self.url.domain()}pub fn is_homepage(&self) -> bool {- is_homepage(self.url())+ self.url.is_homepage()}pub fn metadata(&self) -> Vec<Meta> {@@ -402,10 +413,43 @@ impl<'a> Html<'a> {}}+#[derive(Debug, PartialEq, Clone)]+pub struct Url(String);++impl Display for Url {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "{}", &self.0)+ }+}++impl From<String> for Url {+ fn from(url: String) -> Self {+ Url(url)+ }+}++impl Url {+ pub fn host(&self) -> &str {+ host(&self.0)+ }++ pub fn domain(&self) -> &str {+ domain(&self.0)+ }++ pub fn is_homepage(&self) -> bool {+ is_homepage(&self.0)+ }++ pub fn protocol(&self) -> &str {+ find_protocol(&self.0)+ }+}+#[derive(Debug, PartialEq)]pub struct Link {- pub source: String,- pub destination: String,+ pub source: Url,+ pub destination: Url,pub text: String,}@@ -443,8 +487,8 @@ mod tests {assert_eq!(webpage.links(),vec![Link {- source: "https://www.example.com/whatever".to_string(),- destination: "example.com".to_string(),+ source: "https://www.example.com/whatever".to_string().into(),+ destination: "example.com".to_string().into(),text: "Link to example".to_string()}]);@@ -455,7 +499,10 @@ mod tests {expected_meta.insert("content".to_string(), "value".to_string());assert_eq!(webpage.metadata(), vec![expected_meta]);- assert_eq!(webpage.url(), "https://www.example.com/whatever");+ assert_eq!(+ webpage.url().to_string().as_str(),+ "https://www.example.com/whatever"+ );assert_eq!(webpage.host(), "www.example.com");assert_eq!(webpage.domain(), "example.com");}@@ -595,4 +642,10 @@ mod tests {assert!(webpage.text().is_some());assert!(!webpage.text().unwrap().is_empty());}++ #[test]+ fn test_find_protocol() {+ assert_eq!(find_protocol("https://example.com"), "https");+ assert_eq!(find_protocol("http://example.com"), "http");+ }}diff --git a/templates/base.html b/templates/base.htmlindex 38112bf2..613fb66b 100644--- a/templates/base.html+++ b/templates/base.html@@ -19,7 +19,6 @@body {font-family: Montserrat, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;- color: #333;box-sizing: border-box;padding: 0;margin: 0;