Skip to content

web search done right

searchable backlinks and store centrality in search engine

Mikkel Denker74723d64bac9
diff --git a/src/indexer.rs b/src/indexer.rs
index 289c47ef..d329e607 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -7,7 +7,7 @@ use std::io::{self, BufRead};
use tokio::io::AsyncReadExt;
use crate::warc::{self, WarcFile};
-use crate::webpage::Webpage;
+use crate::webpage::Html;
use crate::{Config, Error, Result, WarcSource};
pub struct Indexer {
@@ -79,7 +79,7 @@ impl Indexer {
let raw_bytes = raw_bytes.unwrap();
let warc = WarcFile::new(&raw_bytes[..]);
for record in warc.flatten() {
- let webpage = Webpage::parse(&record.response.body, &record.request.url);
+ let webpage = Html::parse(&record.response.body, &record.request.url);
// println!("TEST: {:?}", webpage.title());
// println!();
}
diff --git a/src/lib.rs b/src/lib.rs
index e055300f..22130caf 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,6 +7,7 @@ use thiserror::Error;
mod indexer;
mod query;
mod ranking;
+mod schema;
mod search_index;
mod warc;
mod webgraph;
diff --git a/src/ranking.rs b/src/ranking.rs
index 64156481..bf819a2b 100644
--- a/src/ranking.rs
+++ b/src/ranking.rs
@@ -3,3 +3,11 @@ use tantivy::collector::TopDocs;
pub(crate) fn initial_collector() -> TopDocs {
TopDocs::with_limit(20) // TODO: take harmonic centrality into account initially
}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn harmonic_ranking() {
+ todo!();
+ }
+}
diff --git a/src/schema.rs b/src/schema.rs
new file mode 100644
index 00000000..82fedef6
--- /dev/null
+++ b/src/schema.rs
@@ -0,0 +1,57 @@
+use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
+
+#[derive(Clone)]
+pub enum Field {
+ Title,
+ Body,
+ Url,
+ BacklinkText,
+ Centrality,
+}
+pub static ALL_FIELDS: [Field; 5] = [
+ Field::Title,
+ Field::Body,
+ Field::Url,
+ Field::BacklinkText,
+ Field::Centrality,
+];
+
+impl Field {
+ fn default_options(&self) -> tantivy::schema::TextOptions {
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer("en_stem")
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ )
+ }
+
+ pub fn options(&self) -> tantivy::schema::TextOptions {
+ match self {
+ Field::Title => self.default_options().set_stored(),
+ Field::Body => self.default_options().set_stored(),
+ Field::Url => self.default_options().set_stored(),
+ Field::BacklinkText => self.default_options(),
+ Field::Centrality => self.default_options().set_fast(),
+ }
+ }
+
+ pub fn as_str(&self) -> &str {
+ match self {
+ Field::Title => "title",
+ Field::Body => "body",
+ Field::Url => "url",
+ Field::BacklinkText => "backlink_text",
+ Field::Centrality => "centrality",
+ }
+ }
+}
+
+pub fn create_schema() -> tantivy::schema::Schema {
+ let mut builder = tantivy::schema::Schema::builder();
+
+ for field in &ALL_FIELDS {
+ builder.add_text_field(field.as_str(), field.options());
+ }
+
+ builder.build()
+}
diff --git a/src/search_index.rs b/src/search_index.rs
index 86cd1593..f54428bc 100644
--- a/src/search_index.rs
+++ b/src/search_index.rs
@@ -1,56 +1,13 @@
use tantivy::collector::Count;
-use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
use tantivy::{DocAddress, Document, LeasedItem, Searcher};
use crate::query::Query;
use crate::ranking;
-use crate::{webpage::Webpage, Result};
+use crate::schema::{create_schema, Field, ALL_FIELDS};
+use crate::webpage::Webpage;
+use crate::Result;
use std::path::Path;
-#[derive(Clone)]
-pub enum Field {
- Title,
- Body,
- Url,
-}
-pub static ALL_FIELDS: [Field; 3] = [Field::Title, Field::Body, Field::Url];
-
-impl Field {
- fn default_options(&self) -> tantivy::schema::TextOptions {
- TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer("en_stem")
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- )
- .set_stored()
- }
- pub fn options(&self) -> tantivy::schema::TextOptions {
- match self {
- Field::Title => self.default_options(),
- Field::Body => self.default_options(),
- Field::Url => self.default_options(),
- }
- }
-
- pub fn as_str(&self) -> &str {
- match self {
- Field::Title => "title",
- Field::Body => "body",
- Field::Url => "url",
- }
- }
-}
-
-fn create_schema() -> tantivy::schema::Schema {
- let mut builder = tantivy::schema::Schema::builder();
-
- for field in &ALL_FIELDS {
- builder.add_text_field(field.as_str(), field.options());
- }
-
- builder.build()
-}
pub struct Index {
tantivy_index: tantivy::Index,
writer: tantivy::IndexWriter,
@@ -156,6 +113,7 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string()
}
+ Field::BacklinkText | Field::Centrality => {}
}
}
@@ -165,6 +123,8 @@ impl From<Document> for RetrievedWebpage {
#[cfg(test)]
mod tests {
+ use crate::webpage::Link;
+
use super::*;
#[test]
@@ -177,7 +137,7 @@ mod tests {
assert_eq!(result.num_docs, 0);
index
- .insert(Webpage::parse(
+ .insert(Webpage::new(
r#"
<html>
<head>
@@ -186,6 +146,8 @@ mod tests {
</html>
"#,
"https://www.example.com",
+ vec![],
+ 1.0,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -202,7 +164,7 @@ mod tests {
let query = Query::parse("this query should not match").expect("Failed to parse query");
index
- .insert(Webpage::parse(
+ .insert(Webpage::new(
r#"
<html>
<head>
@@ -211,6 +173,8 @@ mod tests {
</html>
"#,
"https://www.example.com",
+ vec![],
+ 1.0,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -226,7 +190,7 @@ mod tests {
let query = Query::parse("runner").expect("Failed to parse query");
index
- .insert(Webpage::parse(
+ .insert(Webpage::new(
r#"
<html>
<head>
@@ -235,6 +199,8 @@ mod tests {
</html>
"#,
"https://www.example.com",
+ vec![],
+ 1.0,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
@@ -244,6 +210,58 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
}
+ #[test]
+ fn searchable_backlinks() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("great site").expect("Failed to parse query");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <a href="https://www.b.com">B site is great</a>
+ </html>
+ "#,
+ "https://www.a.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ </html>
+ "#,
+ "https://www.b.com",
+ vec![Link {
+ source: "https://www.a.com".to_string(),
+ destination: "https://www.b.com".to_string(),
+ text: "B site is great".to_string(),
+ }],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+
+ let mut result = index.search(&query).expect("Search failed");
+
+ result
+ .documents
+ .sort_by(|a, b| a.url.partial_cmp(&b.url).unwrap());
+
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.a.com");
+ assert_eq!(result.documents[1].url, "https://www.b.com");
+ }
+
// #[test]
// fn snippet() {
// todo!();
diff --git a/src/webpage.rs b/src/webpage.rs
index e242bcfc..b266ae38 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -1,20 +1,65 @@
use crate::{Error, Result};
use itertools::Itertools;
-use scraper::{Html, Node, Selector};
+use scraper::Html as ScraperHtml;
+use scraper::{Node, Selector};
use std::collections::BTreeMap;
-use crate::search_index::{Field, ALL_FIELDS};
+use crate::schema::{Field, ALL_FIELDS};
-#[derive(Debug)]
pub struct Webpage {
- dom: Html,
- url: String,
+ pub html: Html,
+ pub backlinks: Vec<Link>,
+ pub centrality: f64,
}
impl Webpage {
+ pub fn new(html: &str, url: &str, backlinks: Vec<Link>, centrality: f64) -> Self {
+ let html = Html::parse(html, url);
+
+ Self {
+ html,
+ backlinks,
+ centrality,
+ }
+ }
+
+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {
+ let mut doc = self.html.into_tantivy(schema)?;
+
+ let backlink_text: String = itertools::intersperse(
+ self.backlinks.into_iter().map(|link| link.text),
+ "\n".to_string(),
+ )
+ .collect();
+
+ doc.add_text(
+ schema
+ .get_field(Field::BacklinkText.as_str())
+ .expect("Failed to get backlink-text field"),
+ backlink_text,
+ );
+
+ doc.add_f64(
+ schema
+ .get_field(Field::Centrality.as_str())
+ .expect("Failed to get centrality field"),
+ self.centrality,
+ );
+
+ Ok(doc)
+ }
+}
+
+#[derive(Debug)]
+pub struct Html {
+ dom: ScraperHtml,
+ url: String,
+}
+
+impl Html {
pub fn parse(html: &str, url: &str) -> Self {
Self {
- dom: Html::parse_document(html),
+ dom: ScraperHtml::parse_document(html),
url: url.to_string(),
}
}
@@ -32,7 +77,11 @@ impl Webpage {
.filter(|(dest, _)| dest.is_some())
.map(|(destination, text)| {
let destination = destination.unwrap().to_string();
- Link { destination, text }
+ Link {
+ source: self.url.clone(),
+ destination,
+ text,
+ }
})
.collect()
}
@@ -144,6 +193,7 @@ impl Webpage {
}
Field::Body => doc.add_text(tantivy_field, self.text()),
Field::Url => doc.add_text(tantivy_field, self.url()),
+ Field::BacklinkText | Field::Centrality => {}
}
}
@@ -153,8 +203,9 @@ impl Webpage {
#[derive(Debug, PartialEq)]
pub struct Link {
- destination: String,
- text: String,
+ pub source: String,
+ pub destination: String,
+ pub text: String,
}
pub type Meta = BTreeMap<String, String>;
@@ -177,13 +228,14 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw, "https://www.example.com/whatever");
+ let webpage = Html::parse(raw, "https://www.example.com/whatever");
assert_eq!(&webpage.text(), "Best example website ever");
assert_eq!(webpage.title(), Some("Best website".to_string()));
assert_eq!(
webpage.links(),
vec![Link {
+ source: "https://www.example.com/whatever".to_string(),
destination: "example.com".to_string(),
text: "Best example website ever".to_string()
}]
@@ -208,7 +260,7 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw, "https://www.example.com/whatever");
+ let webpage = Html::parse(raw, "https://www.example.com/whatever");
assert_eq!(&webpage.text(), "test website");
}
@@ -234,7 +286,7 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw, "https://www.example.com");
+ let webpage = Html::parse(raw, "https://www.example.com");
assert_eq!(
webpage.text(),
@@ -263,7 +315,7 @@ mod tests {
</html>
"#;
- let webpage = Webpage::parse(raw, "https://www.example.com");
+ let webpage = Html::parse(raw, "https://www.example.com");
assert_eq!(
webpage.text(),