Skip to content

web search done right

simple snippet generation

Mikkel Denker2870c5f5ad05
diff --git a/src/lib.rs b/src/lib.rs
index bbc864b8..f4efedec 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -24,6 +24,7 @@ mod query;
mod ranking;
mod schema;
mod search_index;
+mod snippet;
mod warc;
mod webgraph;
mod webpage;
diff --git a/src/search_index.rs b/src/search_index.rs
index 59e5a83b..ed23d903 100644
--- a/src/search_index.rs
+++ b/src/search_index.rs
@@ -17,10 +17,10 @@ use tantivy::collector::Count;
use tantivy::{DocAddress, Document, LeasedItem, Searcher};
use crate::query::Query;
-use crate::ranking;
use crate::schema::{create_schema, Field, ALL_FIELDS};
use crate::webpage::Webpage;
use crate::Result;
+use crate::{ranking, snippet};
use std::path::Path;
pub struct Index {
@@ -66,16 +66,20 @@ impl Index {
let (count, docs) =
searcher.search(&tantivy_query, &(Count, ranking::initial_collector()))?;
- let fast_pages: Vec<RetrievedWebpage> = docs
+ let mut webpages: Vec<RetrievedWebpage> = docs
.into_iter()
.map(|(_score, doc_address)| Index::retrieve_doc(doc_address, &searcher))
.filter(|page| page.is_ok())
.map(|page| page.unwrap())
.collect();
+ for page in &mut webpages {
+ page.snippet = snippet::generate(query, &page.body, &searcher)?;
+ }
+
Ok(SearchResult {
num_docs: count,
- documents: fast_pages,
+ documents: webpages,
})
}
@@ -276,9 +280,4 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.a.com");
assert_eq!(result.documents[1].url, "https://www.b.com");
}
-
- // #[test]
- // fn snippet() {
- // todo!();
- // }
}
diff --git a/src/snippet.rs b/src/snippet.rs
new file mode 100644
index 00000000..5351116c
--- /dev/null
+++ b/src/snippet.rs
@@ -0,0 +1,94 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{Document, Searcher, SnippetGenerator};
+
+use crate::{query::Query, schema::Field, Result};
+
+/// For now we use the snippet generator from tantivy.
+/// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf.
+/// This will require us to store each paragraph of the webpage separately to get adequate performance.
+/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
+pub fn generate(query: &Query, text: &str, searcher: &Searcher) -> Result<String> {
+ let generator = SnippetGenerator::create(
+ &searcher,
+ &query.tantivy(searcher.schema(), searcher.index().tokenizers()),
+ searcher
+ .schema()
+ .get_field(Field::Body.as_str())
+ .expect("Failed to get body field"),
+ )?;
+
+ let snippet = generator.snippet(text);
+ let highlighted = snippet.to_html() + "...";
+
+ Ok(highlighted)
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{search_index::Index, webpage::Webpage};
+
+ use super::*;
+
+ const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored by
+Mozilla which describes it as a "safe, concurrent, practical language", supporting functional and
+imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?],
+but its designers intend it to provide better memory safety while still maintaining
+performance.
+Rust is free and open-source software, released under an MIT License, or Apache License
+2.0. Its designers have refined the language through the experiences of writing the Servo
+web browser layout engine[14] and the Rust compiler. A large proportion of current commits
+to the project are from community members.[15]
+Rust won first place for "most loved programming language" in the Stack Overflow Developer
+Survey in 2016, 2017, and 2018."#;
+
+ #[test]
+ fn snippet_during_search() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("rust language").expect("Failed to parse query");
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website for runners</title>
+ </head>
+ {}
+ </html>
+ "#,
+ TEST_TEXT
+ ),
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index.search(&query).expect("Search failed");
+
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(
+ result.documents[0].snippet,
+ "<b>Rust</b> is a systems programming <b>language</b> sponsored by\nMozilla which describes it as a &quot;safe, concurrent, practical <b>language</b>&quot;, supporting functional and..."
+ .to_string()
+ );
+ }
+}