Skip to content

web search done right

Merge search indexes and make indexes serializable by freezing

Mikkel Denkerec0385b66890
diff --git a/Cargo.toml b/Cargo.toml
index 934e588f..b4da6c4d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -29,6 +29,7 @@ rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
scraper = "0.13.0"
serde = "1.0.137"
+serde_json = "1.0.81"
sled = { version = "0.34.7", features = ["compression"] }
tantivy = "0.18.0"
thiserror = "1.0.31"
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 78c79998..8163999f 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -13,7 +13,6 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use std::fs::File;
use std::io::{self, BufRead};
@@ -41,18 +40,8 @@ impl From<IndexingConfig> for Indexer {
impl Indexer {
pub fn run(self) -> Result<()> {
- let pb = ProgressBar::new(self.warc_paths.len() as u64);
- pb.set_style(
- ProgressStyle::default_bar()
- .template(
- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",
- )
- .progress_chars("#>-"),
- );
-
self.warc_paths
.into_iter()
- .progress_with(pb)
.map(|warc_path| {
let source = self.config.warc_source.clone();
diff --git a/src/index.rs b/src/index.rs
index dd47aca8..5f82e7ca 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -1,3 +1,4 @@
+use serde::{Deserialize, Serialize};
// Cuely is an open source web search engine.
// Copyright (C) 2022 Cuely ApS
//
@@ -16,15 +17,19 @@
use tantivy::collector::{Collector, Count};
use tantivy::{DocAddress, Document, LeasedItem};
+use crate::directory::{self, DirEntry};
use crate::query::Query;
use crate::schema::{Field, ALL_FIELDS};
use crate::searcher::SearchResult;
use crate::snippet;
use crate::webpage::Webpage;
use crate::Result;
+use crate::{schema::create_schema, tokenizer::Tokenizer};
+use std::fs;
use std::path::Path;
pub struct Index {
+ path: String,
tantivy_index: tantivy::Index,
writer: tantivy::IndexWriter,
reader: tantivy::IndexReader,
@@ -33,11 +38,20 @@ pub struct Index {
impl Index {
#[cfg(test)]
- pub fn temporary() -> Result<Index> {
- use crate::{schema::create_schema, tokenizer::Tokenizer};
+ pub fn temporary() -> Result<Self> {
+ let path = crate::gen_temp_path();
+ Self::open(path)
+ }
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let schema = create_schema();
- let tantivy_index = tantivy::Index::create_in_ram(schema);
+
+ let tantivy_index = if path.as_ref().exists() {
+ tantivy::Index::open_in_dir(path.as_ref())?
+ } else {
+ fs::create_dir_all(path.as_ref())?;
+ tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?
+ };
tantivy_index
.tokenizers()
@@ -46,13 +60,11 @@ impl Index {
Ok(Index {
writer: tantivy_index.writer(100_000_000)?,
reader: tantivy_index.reader()?,
- schema: create_schema(),
+ schema,
+ path: path.as_ref().to_str().unwrap().to_string(),
tantivy_index,
})
}
- pub fn open<P: AsRef<Path>>(_path: P) -> Result<Self> {
- todo!();
- }
pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
self.writer
@@ -100,6 +112,52 @@ impl Index {
let doc = searcher.doc(doc_address)?;
Ok(RetrievedWebpage::from(doc))
}
+
+ pub fn merge(mut self, mut other: Index) -> Self {
+ other.commit().expect("failed to commit index");
+ self.commit().expect("failed to commit index");
+
+ let other_meta = other
+ .tantivy_index
+ .load_metas()
+ .expect("failed to laod tantivy metadata for index");
+
+ let mut meta = self
+ .tantivy_index
+ .load_metas()
+ .expect("failed to laod tantivy metadata for index");
+
+ let x = other.path.clone();
+ let other_path = Path::new(x.as_str());
+ drop(other);
+
+ let path = self.path.clone();
+ let self_path = Path::new(path.as_str());
+ drop(self);
+
+ for segment in other_meta.segments {
+ // TODO: handle case where current index has segment with same name
+ for file in segment.list_files() {
+ let p = other_path.join(&file);
+ if p.exists() {
+ fs::rename(p, self_path.join(&file)).unwrap();
+ }
+ }
+ meta.segments.push(segment);
+ }
+
+ fs::remove_dir_all(other_path).unwrap();
+
+ let self_path = Path::new(&path);
+
+ std::fs::write(
+ self_path.join("meta.json"),
+ serde_json::to_string(&meta).unwrap(),
+ )
+ .unwrap();
+
+ Self::open(path).expect("failed to open index")
+ }
}
#[derive(Default, Debug)]
@@ -145,6 +203,40 @@ impl From<Document> for RetrievedWebpage {
}
}
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FrozenIndex {
+ pub root: DirEntry,
+}
+
+impl From<FrozenIndex> for Index {
+ fn from(frozen: FrozenIndex) -> Self {
+ directory::recreate_folder(&frozen.root).unwrap();
+
+ match frozen.root {
+ DirEntry::Folder { name, entries: _ } => {
+ Index::open(name).expect("failed to open index")
+ }
+ DirEntry::File {
+ name: _,
+ content: _,
+ } => {
+ panic!("Cannot open index from a file - must be directory.")
+ }
+ }
+ }
+}
+
+impl From<Index> for FrozenIndex {
+ fn from(mut index: Index) -> Self {
+ index.commit().expect("failed to commit index");
+ let path = index.path.clone();
+ drop(index);
+ let root = directory::scan_folder(path).unwrap();
+
+ Self { root }
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::{ranking::Ranker, webpage::Link};
@@ -388,4 +480,92 @@ mod tests {
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
+
+ #[test]
+ fn serialize_deserialize_bincode() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+
+ let path = index.path.clone();
+ let frozen: FrozenIndex = index.into();
+ let bytes = bincode::serialize(&frozen).unwrap();
+
+ std::fs::remove_dir_all(path).unwrap();
+
+ let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();
+ let index: Index = deserialized_frozen.into();
+ let query = Query::parse("website").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
+
+ #[test]
+ fn merge() {
+ let mut index1 = Index::temporary().expect("Unable to open index");
+
+ index1
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+
+ let mut index2 = Index::temporary().expect("Unable to open index");
+
+ index2
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ vec![],
+ 1.,
+ ))
+ .expect("failed to parse webpage");
+
+ let index = index1.merge(index2);
+
+ let query = Query::parse("website").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[1].url, "https://www.example.com");
+ }
}
diff --git a/src/lib.rs b/src/lib.rs
index 3df89b4c..356aa182 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -17,6 +17,7 @@ use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{self, BufRead};
use std::num::ParseIntError;
+use std::path::PathBuf;
use tantivy::TantivyError;
use thiserror::Error;
@@ -167,3 +168,31 @@ pub enum Error {
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
+
+// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445
+#[allow(unused)]
+fn gen_temp_path() -> PathBuf {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::time::SystemTime;
+
+ static SALT_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+ let seed = SALT_COUNTER.fetch_add(1, Ordering::SeqCst) as u128;
+
+ let now = SystemTime::now()
+ .duration_since(SystemTime::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ << 48;
+
+ let pid = u128::from(std::process::id());
+
+ let salt = (pid << 16) + now + seed;
+
+ if cfg!(target_os = "linux") {
+ // use shared memory for temporary linux files
+ format!("/dev/shm/pagecache.tmp.{}", salt).into()
+ } else {
+ std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))
+ }
+}