web search done right
Merge all segments at the end of map, don't merge automatically. This hugely increases the indexing performance as we don't merge as often.
Mikkel Denker21149d8ced9c
diff --git a/src/directory.rs b/src/directory.rsindex 43802b18..13eb9024 100644--- a/src/directory.rs+++ b/src/directory.rs@@ -16,7 +16,7 @@use crate::Result;use serde::{Deserialize, Serialize};-use std::{fs, path::Path};+use std::fs;#[derive(Serialize, Deserialize, Debug)]pub enum DirEntry {@@ -56,9 +56,6 @@ fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {pub fn recreate_folder(entry: &DirEntry) -> Result<()> {match entry {DirEntry::Folder { name, entries } => {- if Path::new(name).exists() {- fs::remove_dir_all(name)?;- }fs::create_dir(name)?;for entry in entries {diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex f8ee8558..e0447712 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -95,6 +95,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {}}index.commit().unwrap();+ index.merge_all_segments().unwrap();info!("{} done", name);diff --git a/src/index.rs b/src/index.rsindex 20fce9ee..a2d43497 100644--- a/src/index.rs+++ b/src/index.rs@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::collector::{Collector, Count};-use tantivy::merge_policy::LogMergePolicy;+use tantivy::merge_policy::NoMergePolicy;use tantivy::{DocAddress, Document, LeasedItem};use crate::directory::{self, DirEntry};@@ -60,9 +60,7 @@ impl Index {let writer = tantivy_index.writer(10_000_000_000)?;- let mut merge_policy = LogMergePolicy::default();- merge_policy.set_max_docs_before_merge(100_000_000);-+ let merge_policy = NoMergePolicy::default();writer.set_merge_policy(Box::new(merge_policy));Ok(Index {@@ -113,6 +111,20 @@ impl Index {})}+ pub fn merge_all_segments(&mut self) -> Result<()> {+ let segment_ids: Vec<_> = self+ .tantivy_index+ .load_metas()?+ .segments+ .into_iter()+ .map(|segment| segment.id())+ .collect();++ self.writer.merge(&segment_ids[..]).wait()?;++ Ok(())+ }+fn retrieve_doc(doc_address: DocAddress,searcher: &LeasedItem<tantivy::Searcher>,@@ -137,11 +149,11 @@ impl Index {let x = other.path.clone();let other_path = Path::new(x.as_str());- drop(other);+ other.writer.wait_merging_threads().unwrap();let path = self.path.clone();let self_path = Path::new(path.as_str());- drop(self);+ self.writer.wait_merging_threads().unwrap();for segment in other_meta.segments {// TODO: handle case where current index has segment with same name@@ -221,19 +233,22 @@ pub struct FrozenIndex {impl From<FrozenIndex> for Index {fn from(frozen: FrozenIndex) -> Self {- directory::recreate_folder(&frozen.root).unwrap();-- match frozen.root {- DirEntry::Folder { name, entries: _ } => {- Index::open(name).expect("failed to open index")- }+ let path = match &frozen.root {+ DirEntry::Folder { name, entries: _ } => name.clone(),DirEntry::File {name: _,content: _,} => {panic!("Cannot open index from a file - must be directory.")}+ };++ if Path::new(&path).exists() {+ fs::remove_dir_all(&path).unwrap();}++ directory::recreate_folder(&frozen.root).unwrap();+ Index::open(path).expect("failed to open index")}}@@ -241,7 +256,7 @@ impl From<Index> for FrozenIndex {fn from(mut index: Index) -> Self {index.commit().expect("failed to commit index");let path = index.path.clone();- drop(index);+ index.writer.wait_merging_threads().unwrap();let root = directory::scan_folder(path).unwrap();Self { root }diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 67ec53fe..4786b940 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -243,8 +243,9 @@ impl Manager {match self.try_map(&job) {Ok(res) => return res,Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),- Err(_) => {+ Err(err) => {warn!("Worker failed - rescheduling job");+ debug!("{:?}", err);}}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex b1229e3f..2e0d9fe0 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -138,7 +138,7 @@ mod tests {<head><title>Website B</title></head>- dr dk dr dk dr dk dr dk+ dr dk dr dk and some other text</html>"#,"https://www.b.com",diff --git a/src/schema.rs b/src/schema.rsindex 286de329..2241b736 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -75,7 +75,7 @@ impl Field {pub fn boost(&self) -> Option<f32> {match self {- Field::Host => Some(30.0),+ Field::Host => Some(5.0),Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {None}diff --git a/src/webpage.rs b/src/webpage.rsindex 39554f30..64d41255 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -219,6 +219,7 @@ impl<'a> Html<'a> {.collect::<Vec<String>>()}+ // TODO: implement something like this: http://corpus.tools/wiki/Justext/Algorithmpub fn text(&self) -> Option<String> {let strings: Vec<String> = match &self.dom {Dom::ScraperHtml(dom) => {diff --git a/templates/search.html b/templates/search.htmlindex 0ff2db8b..da1fbcd6 100644--- a/templates/search.html+++ b/templates/search.html@@ -4,7 +4,7 @@{% block content %}<div class="flex flex-col w-full">- <div class="flex items-center h-24 border">+ <div class="flex items-center h-24 border bg-neutral-50"><div class="ml-14 mr-14"><img src="/static/images/logo.svg" /></div>@@ -15,7 +15,7 @@{% for item in search_result %}<div class="search-result-item bg-white max-w-2xl p-3"><div class="text-sm mb-1">- <a class="hover:no-underline text-gray-500" href="{{ item.url }}">{{ item.pretty_url }}</a>+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a></div><div class="text-lg font-medium mb-1"><a class="title-link" href="{{ item.url }}">{{ item.title }}</a>