web search done right
searchable backlinks and store centrality in search engine
Mikkel Denker74723d64bac9
diff --git a/src/indexer.rs b/src/indexer.rsindex 289c47ef..d329e607 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -7,7 +7,7 @@ use std::io::{self, BufRead};use tokio::io::AsyncReadExt;use crate::warc::{self, WarcFile};-use crate::webpage::Webpage;+use crate::webpage::Html;use crate::{Config, Error, Result, WarcSource};pub struct Indexer {@@ -79,7 +79,7 @@ impl Indexer {let raw_bytes = raw_bytes.unwrap();let warc = WarcFile::new(&raw_bytes[..]);for record in warc.flatten() {- let webpage = Webpage::parse(&record.response.body, &record.request.url);+ let webpage = Html::parse(&record.response.body, &record.request.url);// println!("TEST: {:?}", webpage.title());// println!();}diff --git a/src/lib.rs b/src/lib.rsindex e055300f..22130caf 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -7,6 +7,7 @@ use thiserror::Error;mod indexer;mod query;mod ranking;+mod schema;mod search_index;mod warc;mod webgraph;diff --git a/src/ranking.rs b/src/ranking.rsindex 64156481..bf819a2b 100644--- a/src/ranking.rs+++ b/src/ranking.rs@@ -3,3 +3,11 @@ use tantivy::collector::TopDocs;pub(crate) fn initial_collector() -> TopDocs {TopDocs::with_limit(20) // TODO: take harmonic centrality into account initially}++#[cfg(test)]+mod tests {+ #[test]+ fn harmonic_ranking() {+ todo!();+ }+}diff --git a/src/schema.rs b/src/schema.rsnew file mode 100644index 00000000..82fedef6--- /dev/null+++ b/src/schema.rs@@ -0,0 +1,57 @@+use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};++#[derive(Clone)]+pub enum Field {+ Title,+ Body,+ Url,+ BacklinkText,+ Centrality,+}+pub static ALL_FIELDS: [Field; 5] = [+ Field::Title,+ Field::Body,+ Field::Url,+ Field::BacklinkText,+ Field::Centrality,+];++impl Field {+ fn default_options(&self) -> tantivy::schema::TextOptions {+ TextOptions::default().set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer("en_stem")+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ )+ }++ pub fn options(&self) -> tantivy::schema::TextOptions {+ match self {+ Field::Title => self.default_options().set_stored(),+ Field::Body => self.default_options().set_stored(),+ Field::Url => self.default_options().set_stored(),+ Field::BacklinkText => self.default_options(),+ Field::Centrality => self.default_options().set_fast(),+ }+ }++ pub fn as_str(&self) -> &str {+ match self {+ Field::Title => "title",+ Field::Body => "body",+ Field::Url => "url",+ Field::BacklinkText => "backlink_text",+ Field::Centrality => "centrality",+ }+ }+}++pub fn create_schema() -> tantivy::schema::Schema {+ let mut builder = tantivy::schema::Schema::builder();++ for field in &ALL_FIELDS {+ builder.add_text_field(field.as_str(), field.options());+ }++ builder.build()+}diff --git a/src/search_index.rs b/src/search_index.rsindex 86cd1593..f54428bc 100644--- a/src/search_index.rs+++ b/src/search_index.rs@@ -1,56 +1,13 @@use tantivy::collector::Count;-use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};use tantivy::{DocAddress, Document, LeasedItem, Searcher};use crate::query::Query;use crate::ranking;-use crate::{webpage::Webpage, Result};+use crate::schema::{create_schema, Field, ALL_FIELDS};+use crate::webpage::Webpage;+use crate::Result;use std::path::Path;-#[derive(Clone)]-pub enum Field {- Title,- Body,- Url,-}-pub static ALL_FIELDS: [Field; 3] = [Field::Title, Field::Body, Field::Url];--impl Field {- fn default_options(&self) -> tantivy::schema::TextOptions {- TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_tokenizer("en_stem")- .set_index_option(IndexRecordOption::WithFreqsAndPositions),- )- .set_stored()- }- pub fn options(&self) -> tantivy::schema::TextOptions {- match self {- Field::Title => self.default_options(),- Field::Body => self.default_options(),- Field::Url => self.default_options(),- }- }-- pub fn as_str(&self) -> &str {- match self {- Field::Title => "title",- Field::Body => "body",- Field::Url => "url",- }- }-}--fn create_schema() -> tantivy::schema::Schema {- let mut builder = tantivy::schema::Schema::builder();-- for field in &ALL_FIELDS {- builder.add_text_field(field.as_str(), field.options());- }-- builder.build()-}pub struct Index {tantivy_index: tantivy::Index,writer: tantivy::IndexWriter,@@ -156,6 +113,7 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}+ Field::BacklinkText | Field::Centrality => {}}}@@ -165,6 +123,8 @@ impl From<Document> for RetrievedWebpage {#[cfg(test)]mod tests {+ use crate::webpage::Link;+use super::*;#[test]@@ -177,7 +137,7 @@ mod tests {assert_eq!(result.num_docs, 0);index- .insert(Webpage::parse(+ .insert(Webpage::new(r#"<html><head>@@ -186,6 +146,8 @@ mod tests {</html>"#,"https://www.example.com",+ vec![],+ 1.0,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -202,7 +164,7 @@ mod tests {let query = Query::parse("this query should not match").expect("Failed to parse query");index- .insert(Webpage::parse(+ .insert(Webpage::new(r#"<html><head>@@ -211,6 +173,8 @@ mod tests {</html>"#,"https://www.example.com",+ vec![],+ 1.0,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -226,7 +190,7 @@ mod tests {let query = Query::parse("runner").expect("Failed to parse query");index- .insert(Webpage::parse(+ .insert(Webpage::new(r#"<html><head>@@ -235,6 +199,8 @@ mod tests {</html>"#,"https://www.example.com",+ vec![],+ 1.0,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");@@ -244,6 +210,58 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");}+ #[test]+ fn searchable_backlinks() {+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("great site").expect("Failed to parse query");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ <a href="https://www.b.com">B site is great</a>+ </html>+ "#,+ "https://www.a.com",+ vec![],+ 1.0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ </html>+ "#,+ "https://www.b.com",+ vec![Link {+ source: "https://www.a.com".to_string(),+ destination: "https://www.b.com".to_string(),+ text: "B site is great".to_string(),+ }],+ 1.0,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");++ let mut result = index.search(&query).expect("Search failed");++ result+ .documents+ .sort_by(|a, b| a.url.partial_cmp(&b.url).unwrap());++ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.a.com");+ assert_eq!(result.documents[1].url, "https://www.b.com");+ }+// #[test]// fn snippet() {// todo!();diff --git a/src/webpage.rs b/src/webpage.rsindex e242bcfc..b266ae38 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -1,20 +1,65 @@use crate::{Error, Result};use itertools::Itertools;-use scraper::{Html, Node, Selector};+use scraper::Html as ScraperHtml;+use scraper::{Node, Selector};use std::collections::BTreeMap;-use crate::search_index::{Field, ALL_FIELDS};+use crate::schema::{Field, ALL_FIELDS};-#[derive(Debug)]pub struct Webpage {- dom: Html,- url: String,+ pub html: Html,+ pub backlinks: Vec<Link>,+ pub centrality: f64,}impl Webpage {+ pub fn new(html: &str, url: &str, backlinks: Vec<Link>, centrality: f64) -> Self {+ let html = Html::parse(html, url);++ Self {+ html,+ backlinks,+ centrality,+ }+ }++ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {+ let mut doc = self.html.into_tantivy(schema)?;++ let backlink_text: String = itertools::intersperse(+ self.backlinks.into_iter().map(|link| link.text),+ "\n".to_string(),+ )+ .collect();++ doc.add_text(+ schema+ .get_field(Field::BacklinkText.as_str())+ .expect("Failed to get backlink-text field"),+ backlink_text,+ );++ doc.add_f64(+ schema+ .get_field(Field::Centrality.as_str())+ .expect("Failed to get centrality field"),+ self.centrality,+ );++ Ok(doc)+ }+}++#[derive(Debug)]+pub struct Html {+ dom: ScraperHtml,+ url: String,+}++impl Html {pub fn parse(html: &str, url: &str) -> Self {Self {- dom: Html::parse_document(html),+ dom: ScraperHtml::parse_document(html),url: url.to_string(),}}@@ -32,7 +77,11 @@ impl Webpage {.filter(|(dest, _)| dest.is_some()).map(|(destination, text)| {let destination = destination.unwrap().to_string();- Link { destination, text }+ Link {+ source: self.url.clone(),+ destination,+ text,+ }}).collect()}@@ -144,6 +193,7 @@ impl Webpage {}Field::Body => doc.add_text(tantivy_field, self.text()),Field::Url => doc.add_text(tantivy_field, self.url()),+ Field::BacklinkText | Field::Centrality => {}}}@@ -153,8 +203,9 @@ impl Webpage {#[derive(Debug, PartialEq)]pub struct Link {- destination: String,- text: String,+ pub source: String,+ pub destination: String,+ pub text: String,}pub type Meta = BTreeMap<String, String>;@@ -177,13 +228,14 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw, "https://www.example.com/whatever");+ let webpage = Html::parse(raw, "https://www.example.com/whatever");assert_eq!(&webpage.text(), "Best example website ever");assert_eq!(webpage.title(), Some("Best website".to_string()));assert_eq!(webpage.links(),vec![Link {+ source: "https://www.example.com/whatever".to_string(),destination: "example.com".to_string(),text: "Best example website ever".to_string()}]@@ -208,7 +260,7 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw, "https://www.example.com/whatever");+ let webpage = Html::parse(raw, "https://www.example.com/whatever");assert_eq!(&webpage.text(), "test website");}@@ -234,7 +286,7 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw, "https://www.example.com");+ let webpage = Html::parse(raw, "https://www.example.com");assert_eq!(webpage.text(),@@ -263,7 +315,7 @@ mod tests {</html>"#;- let webpage = Webpage::parse(raw, "https://www.example.com");+ let webpage = Html::parse(raw, "https://www.example.com");assert_eq!(webpage.text(),