Skip to content

web search done right

minor refactoring of url functions into a shared struct

Mikkel Denkerfaddb613aa1e
diff --git a/justfile b/justfile
index 3657d8d8..b22e9d7e 100644
--- a/justfile
+++ b/justfile
@@ -13,7 +13,6 @@
@local:
rm -rf data/index/
- cargo build --release
- ./target/release/cuely indexer local configs/indexer/local.toml
+ cargo run --release -- indexer local configs/indexer/local.toml
mv data/index/CC-MAIN-20220116093137-20220116123137-00049.warc.gz/* data/index/
- ./target/release/cuely frontend data/index/ queries_us.csv
\ No newline at end of file
+ just frontend
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 7493c3f9..b509d2a7 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -17,7 +17,7 @@ use crate::{
mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},
warc::WarcFile,
webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},
- webpage::{self, Html},
+ webpage::Html,
HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
};
use serde::{Deserialize, Serialize};
@@ -61,11 +61,8 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {
for link in webpage
.links()
.into_iter()
- .filter(|link| {
- link.destination.starts_with("http://")
- || link.destination.starts_with("https://")
- })
- .filter(|link| webpage::domain(&link.source) != webpage::domain(&link.destination))
+ .filter(|link| matches!(link.destination.protocol(), "http" | "https"))
+ .filter(|link| link.source.domain() != link.destination.domain())
{
trace!("inserting link {:?}", link);
graph.insert(
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index c9bbdc34..0d133cc6 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -18,7 +18,7 @@ use axum::Extension;
use crate::{
index::RetrievedWebpage,
- webpage::{strip_protocol, strip_query},
+ webpage::{strip_protocol, strip_query, Url},
};
use std::collections::HashMap;
use std::sync::Arc;
@@ -34,17 +34,6 @@ pub fn html_escape(s: &str) -> String {
.collect()
}
-pub fn find_protocol(url: &str) -> &'_ str {
- let mut start_host = 0;
- if url.starts_with("http://") || url.starts_with("https://") {
- start_host = url
- .find(':')
- .expect("It was checked that url starts with protocol");
- }
-
- &url[..start_host]
-}
-
pub struct DisplayedWebpage {
pub title: String,
pub url: String,
@@ -64,7 +53,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
}
- let protocol = find_protocol(&pretty_url).to_string() + "://";
+ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
pretty_url = strip_protocol(&pretty_url).replace('/', " › ");
pretty_url = protocol + &pretty_url;
@@ -121,14 +110,3 @@ pub async fn route(
};
HtmlTemplate(template)
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_find_protocol() {
- assert_eq!(find_protocol("https://example.com"), "https");
- assert_eq!(find_protocol("http://example.com"), "http");
- }
-}
diff --git a/src/index.rs b/src/index.rs
index 3a08e12a..186ff713 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -473,8 +473,8 @@ mod tests {
),
"https://www.b.com",
vec![Link {
- source: "https://www.a.com".to_string(),
- destination: "https://www.b.com".to_string(),
+ source: "https://www.a.com".to_string().into(),
+ destination: "https://www.b.com".to_string().into(),
text: "B site is great".to_string(),
}],
1.0,
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 699f869d..12b12652 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -83,8 +83,8 @@ mod tests {
),
"https://www.b.com",
vec![Link {
- source: "https://www.a.com".to_string(),
- destination: "https://www.b.com".to_string(),
+ source: "https://www.a.com".to_string().into(),
+ destination: "https://www.b.com".to_string().into(),
text: "B site is great".to_string(),
}],
5.0,
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 3a08f6d2..8a3d89dc 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{Error, Result};
use logos::{Lexer, Logos};
-use std::collections::HashMap;
+use std::{collections::HashMap, fmt::Display};
mod just_text;
mod lexer;
@@ -24,6 +24,17 @@ use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};
use self::{just_text::JustText, lexer::Token};
+pub fn find_protocol(url: &str) -> &'_ str {
+ let mut start_host = 0;
+ if url.starts_with("http://") || url.starts_with("https://") {
+ start_host = url
+ .find(':')
+ .expect("It was checked that url starts with protocol");
+ }
+
+ &url[..start_host]
+}
+
pub fn strip_protocol(url: &str) -> &'_ str {
let mut start_host = 0;
if url.starts_with("http://") || url.starts_with("https://") {
@@ -177,7 +188,7 @@ impl<'a> Webpage<'a> {
pub struct Html<'a> {
raw: &'a str,
tokens: Lexer<'a, Token<'a>>,
- url: String,
+ url: Url,
}
impl<'a> Html<'a> {
@@ -187,7 +198,7 @@ impl<'a> Html<'a> {
Self {
raw: html,
tokens,
- url: url.to_string(),
+ url: Url(url.to_string()),
}
}
@@ -212,7 +223,7 @@ impl<'a> Html<'a> {
if let Some(dest) = attributes.get("href") {
links.push(Link {
source: self.url.clone(),
- destination: dest.to_string(),
+ destination: dest.to_string().into(),
text,
})
}
@@ -228,7 +239,7 @@ impl<'a> Html<'a> {
if let Some(dest) = tag.attributes().get("href") {
links.push(Link {
source: self.url.clone(),
- destination: dest.to_string(),
+ destination: dest.to_string().into(),
text: String::new(),
})
}
@@ -298,20 +309,20 @@ impl<'a> Html<'a> {
title
}
- pub fn url(&self) -> &str {
+ pub fn url(&self) -> &Url {
&self.url
}
pub fn host(&self) -> &str {
- host(self.url())
+ self.url.host()
}
pub fn domain(&self) -> &str {
- domain(self.url())
+ self.url.domain()
}
pub fn is_homepage(&self) -> bool {
- is_homepage(self.url())
+ self.url.is_homepage()
}
pub fn metadata(&self) -> Vec<Meta> {
@@ -402,10 +413,43 @@ impl<'a> Html<'a> {
}
}
+#[derive(Debug, PartialEq, Clone)]
+pub struct Url(String);
+
+impl Display for Url {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}", &self.0)
+ }
+}
+
+impl From<String> for Url {
+ fn from(url: String) -> Self {
+ Url(url)
+ }
+}
+
+impl Url {
+ pub fn host(&self) -> &str {
+ host(&self.0)
+ }
+
+ pub fn domain(&self) -> &str {
+ domain(&self.0)
+ }
+
+ pub fn is_homepage(&self) -> bool {
+ is_homepage(&self.0)
+ }
+
+ pub fn protocol(&self) -> &str {
+ find_protocol(&self.0)
+ }
+}
+
#[derive(Debug, PartialEq)]
pub struct Link {
- pub source: String,
- pub destination: String,
+ pub source: Url,
+ pub destination: Url,
pub text: String,
}
@@ -443,8 +487,8 @@ mod tests {
assert_eq!(
webpage.links(),
vec![Link {
- source: "https://www.example.com/whatever".to_string(),
- destination: "example.com".to_string(),
+ source: "https://www.example.com/whatever".to_string().into(),
+ destination: "example.com".to_string().into(),
text: "Link to example".to_string()
}]
);
@@ -455,7 +499,10 @@ mod tests {
expected_meta.insert("content".to_string(), "value".to_string());
assert_eq!(webpage.metadata(), vec![expected_meta]);
- assert_eq!(webpage.url(), "https://www.example.com/whatever");
+ assert_eq!(
+ webpage.url().to_string().as_str(),
+ "https://www.example.com/whatever"
+ );
assert_eq!(webpage.host(), "www.example.com");
assert_eq!(webpage.domain(), "example.com");
}
@@ -595,4 +642,10 @@ mod tests {
assert!(webpage.text().is_some());
assert!(!webpage.text().unwrap().is_empty());
}
+
+ #[test]
+ fn test_find_protocol() {
+ assert_eq!(find_protocol("https://example.com"), "https");
+ assert_eq!(find_protocol("http://example.com"), "http");
+ }
}
diff --git a/templates/base.html b/templates/base.html
index 38112bf2..613fb66b 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -19,7 +19,6 @@
body {
font-family: Montserrat, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
- color: #333;
box-sizing: border-box;
padding: 0;
margin: 0;