Skip to content

web search done right

Merge all segments at the end of map, don't merge automatically. This hugely increases the indexing performance as we don't merge as often.

Mikkel Denker21149d8ced9c
diff --git a/src/directory.rs b/src/directory.rs
index 43802b18..13eb9024 100644
--- a/src/directory.rs
+++ b/src/directory.rs
@@ -16,7 +16,7 @@
use crate::Result;
use serde::{Deserialize, Serialize};
-use std::{fs, path::Path};
+use std::fs;
#[derive(Serialize, Deserialize, Debug)]
pub enum DirEntry {
@@ -56,9 +56,6 @@ fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
pub fn recreate_folder(entry: &DirEntry) -> Result<()> {
match entry {
DirEntry::Folder { name, entries } => {
- if Path::new(name).exists() {
- fs::remove_dir_all(name)?;
- }
fs::create_dir(name)?;
for entry in entries {
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index f8ee8558..e0447712 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -95,6 +95,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
}
}
index.commit().unwrap();
+ index.merge_all_segments().unwrap();
info!("{} done", name);
diff --git a/src/index.rs b/src/index.rs
index 20fce9ee..a2d43497 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::collector::{Collector, Count};
-use tantivy::merge_policy::LogMergePolicy;
+use tantivy::merge_policy::NoMergePolicy;
use tantivy::{DocAddress, Document, LeasedItem};
use crate::directory::{self, DirEntry};
@@ -60,9 +60,7 @@ impl Index {
let writer = tantivy_index.writer(10_000_000_000)?;
- let mut merge_policy = LogMergePolicy::default();
- merge_policy.set_max_docs_before_merge(100_000_000);
-
+ let merge_policy = NoMergePolicy::default();
writer.set_merge_policy(Box::new(merge_policy));
Ok(Index {
@@ -113,6 +111,20 @@ impl Index {
})
}
+ pub fn merge_all_segments(&mut self) -> Result<()> {
+ let segment_ids: Vec<_> = self
+ .tantivy_index
+ .load_metas()?
+ .segments
+ .into_iter()
+ .map(|segment| segment.id())
+ .collect();
+
+ self.writer.merge(&segment_ids[..]).wait()?;
+
+ Ok(())
+ }
+
fn retrieve_doc(
doc_address: DocAddress,
searcher: &LeasedItem<tantivy::Searcher>,
@@ -137,11 +149,11 @@ impl Index {
let x = other.path.clone();
let other_path = Path::new(x.as_str());
- drop(other);
+ other.writer.wait_merging_threads().unwrap();
let path = self.path.clone();
let self_path = Path::new(path.as_str());
- drop(self);
+ self.writer.wait_merging_threads().unwrap();
for segment in other_meta.segments {
// TODO: handle case where current index has segment with same name
@@ -221,19 +233,22 @@ pub struct FrozenIndex {
impl From<FrozenIndex> for Index {
fn from(frozen: FrozenIndex) -> Self {
- directory::recreate_folder(&frozen.root).unwrap();
-
- match frozen.root {
- DirEntry::Folder { name, entries: _ } => {
- Index::open(name).expect("failed to open index")
- }
+ let path = match &frozen.root {
+ DirEntry::Folder { name, entries: _ } => name.clone(),
DirEntry::File {
name: _,
content: _,
} => {
panic!("Cannot open index from a file - must be directory.")
}
+ };
+
+ if Path::new(&path).exists() {
+ fs::remove_dir_all(&path).unwrap();
}
+
+ directory::recreate_folder(&frozen.root).unwrap();
+ Index::open(path).expect("failed to open index")
}
}
@@ -241,7 +256,7 @@ impl From<Index> for FrozenIndex {
fn from(mut index: Index) -> Self {
index.commit().expect("failed to commit index");
let path = index.path.clone();
- drop(index);
+ index.writer.wait_merging_threads().unwrap();
let root = directory::scan_folder(path).unwrap();
Self { root }
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 67ec53fe..4786b940 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -243,8 +243,9 @@ impl Manager {
match self.try_map(&job) {
Ok(res) => return res,
Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),
- Err(_) => {
+ Err(err) => {
warn!("Worker failed - rescheduling job");
+ debug!("{:?}", err);
}
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index b1229e3f..2e0d9fe0 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -138,7 +138,7 @@ mod tests {
<head>
<title>Website B</title>
</head>
- dr dk dr dk dr dk dr dk
+ dr dk dr dk and some other text
</html>
"#,
"https://www.b.com",
diff --git a/src/schema.rs b/src/schema.rs
index 286de329..2241b736 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -75,7 +75,7 @@ impl Field {
pub fn boost(&self) -> Option<f32> {
match self {
- Field::Host => Some(30.0),
+ Field::Host => Some(5.0),
Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {
None
}
diff --git a/src/webpage.rs b/src/webpage.rs
index 39554f30..64d41255 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -219,6 +219,7 @@ impl<'a> Html<'a> {
.collect::<Vec<String>>()
}
+ // TODO: implement something like this: http://corpus.tools/wiki/Justext/Algorithm
pub fn text(&self) -> Option<String> {
let strings: Vec<String> = match &self.dom {
Dom::ScraperHtml(dom) => {
diff --git a/templates/search.html b/templates/search.html
index 0ff2db8b..da1fbcd6 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -4,7 +4,7 @@
{% block content %}
<div class="flex flex-col w-full">
- <div class="flex items-center h-24 border">
+ <div class="flex items-center h-24 border bg-neutral-50">
<div class="ml-14 mr-14">
<img src="/static/images/logo.svg" />
</div>
@@ -15,7 +15,7 @@
{% for item in search_result %}
<div class="search-result-item bg-white max-w-2xl p-3">
<div class="text-sm mb-1">
- <a class="hover:no-underline text-gray-500" href="{{ item.url }}">{{ item.pretty_url }}</a>
+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url }}</a>
</div>
<div class="text-lg font-medium mb-1">
<a class="title-link" href="{{ item.url }}">{{ item.title }}</a>