web search done right
Merge search indexes and make indexes serializable by freezing
Mikkel Denkerec0385b66890
diff --git a/Cargo.toml b/Cargo.tomlindex 934e588f..b4da6c4d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -29,6 +29,7 @@ rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"scraper = "0.13.0"serde = "1.0.137"+serde_json = "1.0.81"sled = { version = "0.34.7", features = ["compression"] }tantivy = "0.18.0"thiserror = "1.0.31"diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 78c79998..8163999f 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -13,7 +13,6 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use std::fs::File;use std::io::{self, BufRead};@@ -41,18 +40,8 @@ impl From<IndexingConfig> for Indexer {impl Indexer {pub fn run(self) -> Result<()> {- let pb = ProgressBar::new(self.warc_paths.len() as u64);- pb.set_style(- ProgressStyle::default_bar()- .template(- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",- )- .progress_chars("#>-"),- );-self.warc_paths.into_iter()- .progress_with(pb).map(|warc_path| {let source = self.config.warc_source.clone();diff --git a/src/index.rs b/src/index.rsindex dd47aca8..5f82e7ca 100644--- a/src/index.rs+++ b/src/index.rs@@ -1,3 +1,4 @@+use serde::{Deserialize, Serialize};// Cuely is an open source web search engine.// Copyright (C) 2022 Cuely ApS//@@ -16,15 +17,19 @@use tantivy::collector::{Collector, Count};use tantivy::{DocAddress, Document, LeasedItem};+use crate::directory::{self, DirEntry};use crate::query::Query;use crate::schema::{Field, ALL_FIELDS};use crate::searcher::SearchResult;use crate::snippet;use crate::webpage::Webpage;use crate::Result;+use crate::{schema::create_schema, tokenizer::Tokenizer};+use std::fs;use std::path::Path;pub struct Index {+ path: String,tantivy_index: tantivy::Index,writer: tantivy::IndexWriter,reader: tantivy::IndexReader,@@ -33,11 +38,20 @@ pub struct Index {impl Index {#[cfg(test)]- pub fn temporary() -> Result<Index> {- use crate::{schema::create_schema, tokenizer::Tokenizer};+ pub fn temporary() -> Result<Self> {+ let path = crate::gen_temp_path();+ Self::open(path)+ }+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let schema = create_schema();- let tantivy_index = tantivy::Index::create_in_ram(schema);++ let tantivy_index = if path.as_ref().exists() {+ tantivy::Index::open_in_dir(path.as_ref())?+ } else {+ fs::create_dir_all(path.as_ref())?;+ tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?+ };tantivy_index.tokenizers()@@ -46,13 +60,11 @@ impl Index {Ok(Index {writer: tantivy_index.writer(100_000_000)?,reader: tantivy_index.reader()?,- schema: create_schema(),+ schema,+ path: path.as_ref().to_str().unwrap().to_string(),tantivy_index,})}- pub fn open<P: AsRef<Path>>(_path: P) -> Result<Self> {- todo!();- }pub fn insert(&mut self, webpage: Webpage) -> Result<()> {self.writer@@ -100,6 +112,52 @@ impl Index {let doc = searcher.doc(doc_address)?;Ok(RetrievedWebpage::from(doc))}++ pub fn merge(mut self, mut other: Index) -> Self {+ other.commit().expect("failed to commit index");+ self.commit().expect("failed to commit index");++ let other_meta = other+ .tantivy_index+ .load_metas()+ .expect("failed to laod tantivy metadata for index");++ let mut meta = self+ .tantivy_index+ .load_metas()+ .expect("failed to laod tantivy metadata for index");++ let x = other.path.clone();+ let other_path = Path::new(x.as_str());+ drop(other);++ let path = self.path.clone();+ let self_path = Path::new(path.as_str());+ drop(self);++ for segment in other_meta.segments {+ // TODO: handle case where current index has segment with same name+ for file in segment.list_files() {+ let p = other_path.join(&file);+ if p.exists() {+ fs::rename(p, self_path.join(&file)).unwrap();+ }+ }+ meta.segments.push(segment);+ }++ fs::remove_dir_all(other_path).unwrap();++ let self_path = Path::new(&path);++ std::fs::write(+ self_path.join("meta.json"),+ serde_json::to_string(&meta).unwrap(),+ )+ .unwrap();++ Self::open(path).expect("failed to open index")+ }}#[derive(Default, Debug)]@@ -145,6 +203,40 @@ impl From<Document> for RetrievedWebpage {}}+#[derive(Debug, Serialize, Deserialize)]+pub struct FrozenIndex {+ pub root: DirEntry,+}++impl From<FrozenIndex> for Index {+ fn from(frozen: FrozenIndex) -> Self {+ directory::recreate_folder(&frozen.root).unwrap();++ match frozen.root {+ DirEntry::Folder { name, entries: _ } => {+ Index::open(name).expect("failed to open index")+ }+ DirEntry::File {+ name: _,+ content: _,+ } => {+ panic!("Cannot open index from a file - must be directory.")+ }+ }+ }+}++impl From<Index> for FrozenIndex {+ fn from(mut index: Index) -> Self {+ index.commit().expect("failed to commit index");+ let path = index.path.clone();+ drop(index);+ let root = directory::scan_folder(path).unwrap();++ Self { root }+ }+}+#[cfg(test)]mod tests {use crate::{ranking::Ranker, webpage::Link};@@ -388,4 +480,92 @@ mod tests {assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.dr.dk");}++ #[test]+ fn serialize_deserialize_bincode() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ vec![],+ 1.0,+ ))+ .expect("failed to parse webpage");++ let path = index.path.clone();+ let frozen: FrozenIndex = index.into();+ let bytes = bincode::serialize(&frozen).unwrap();++ std::fs::remove_dir_all(path).unwrap();++ let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();+ let index: Index = deserialized_frozen.into();+ let query = Query::parse("website").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.example.com");+ }++ #[test]+ fn merge() {+ let mut index1 = Index::temporary().expect("Unable to open index");++ index1+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ vec![],+ 1.0,+ ))+ .expect("failed to parse webpage");++ let mut index2 = Index::temporary().expect("Unable to open index");++ index2+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ vec![],+ 1.,+ ))+ .expect("failed to parse webpage");++ let index = index1.merge(index2);++ let query = Query::parse("website").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 2);+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.example.com");+ assert_eq!(result.documents[1].url, "https://www.example.com");+ }}diff --git a/src/lib.rs b/src/lib.rsindex 3df89b4c..356aa182 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -17,6 +17,7 @@ use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{self, BufRead};use std::num::ParseIntError;+use std::path::PathBuf;use tantivy::TantivyError;use thiserror::Error;@@ -167,3 +168,31 @@ pub enum Error {}pub(crate) type Result<T> = std::result::Result<T, Error>;++// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445+#[allow(unused)]+fn gen_temp_path() -> PathBuf {+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::time::SystemTime;++ static SALT_COUNTER: AtomicUsize = AtomicUsize::new(0);++ let seed = SALT_COUNTER.fetch_add(1, Ordering::SeqCst) as u128;++ let now = SystemTime::now()+ .duration_since(SystemTime::UNIX_EPOCH)+ .unwrap()+ .as_nanos()+ << 48;++ let pid = u128::from(std::process::id());++ let salt = (pid << 16) + now + seed;++ if cfg!(target_os = "linux") {+ // use shared memory for temporary linux files+ format!("/dev/shm/pagecache.tmp.{}", salt).into()+ } else {+ std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))+ }+}