Skip to content

web search done right

clear wal after inserts

Mikkel Denker1b1677dd1558
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index bb406412..05d6b2ef 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -50,6 +50,7 @@ pub fn parse_duration<'de, D: serde::de::Deserializer<'de>>(
"h" => Duration::from_secs(num * 60 * 60),
"hours" => Duration::from_secs(num * 60 * 60),
"d" => Duration::from_secs(num * 24 * 60 * 60),
+ "day" => Duration::from_secs(num * 24 * 60 * 60),
"days" => Duration::from_secs(num * 24 * 60 * 60),
other => return Err(err(format!("invalid suffix {other}"))),
};
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 3db081b5..cb217b78 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -205,11 +205,16 @@ impl InvertedIndex {
}
pub fn segment_ids(&self) -> Vec<SegmentId> {
- self.tantivy_index.searchable_segment_ids().unwrap()
+ self.tantivy_index
+ .searchable_segment_ids()
+ .unwrap_or_default()
}
pub fn num_segments(&self) -> usize {
- self.tantivy_index.searchable_segments().unwrap().len()
+ self.tantivy_index
+ .searchable_segments()
+ .unwrap_or_default()
+ .len()
}
pub fn num_documents(&self) -> u64 {
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
index e58f3329..f901be20 100644
--- a/crates/core/src/live_index/crawler/crawlable_site.rs
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -44,7 +44,7 @@ use super::{
Client,
};
-pub const MAX_PENDING_BUDGET: u64 = 128;
+pub const MAX_PENDING_BUDGET: u64 = 32;
struct InnerCrawlableSite {
site: site_stats::Site,
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index a0507a40..804d6521 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -225,6 +225,7 @@ impl InnerIndex {
.write_ahead_log
.iter()
.unwrap()
+ .unique_by(|page| page.url.clone())
.chunks(BATCH_SIZE)
.into_iter()
{
@@ -233,6 +234,7 @@ impl InnerIndex {
self.index.insert(&webpage).unwrap();
}
}
+ self.write_ahead_log.clear().unwrap();
self.index.commit().unwrap();
self.update_meta();
self.has_inserts = false;
@@ -271,6 +273,7 @@ impl LiveIndex {
.unwrap_or_else(|e| e.into_inner())
.commit(),
);
+ tracing::debug!("index committed");
}
pub fn prune_segments(&self) {
@@ -297,7 +300,7 @@ impl LiveIndex {
}
pub fn insert(&self, pages: &[IndexableWebpage]) {
- tracing::debug!("inserting pages into index");
+ tracing::debug!("inserting {} pages into index", pages.len());
self.inner
.write()
.unwrap_or_else(|e| e.into_inner())