web search done right
clear wal after inserts
Mikkel Denker1b1677dd1558
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex bb406412..05d6b2ef 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -50,6 +50,7 @@ pub fn parse_duration<'de, D: serde::de::Deserializer<'de>>("h" => Duration::from_secs(num * 60 * 60),"hours" => Duration::from_secs(num * 60 * 60),"d" => Duration::from_secs(num * 24 * 60 * 60),+ "day" => Duration::from_secs(num * 24 * 60 * 60),"days" => Duration::from_secs(num * 24 * 60 * 60),other => return Err(err(format!("invalid suffix {other}"))),};diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 3db081b5..cb217b78 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -205,11 +205,16 @@ impl InvertedIndex {}pub fn segment_ids(&self) -> Vec<SegmentId> {- self.tantivy_index.searchable_segment_ids().unwrap()+ self.tantivy_index+ .searchable_segment_ids()+ .unwrap_or_default()}pub fn num_segments(&self) -> usize {- self.tantivy_index.searchable_segments().unwrap().len()+ self.tantivy_index+ .searchable_segments()+ .unwrap_or_default()+ .len()}pub fn num_documents(&self) -> u64 {diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsindex e58f3329..f901be20 100644--- a/crates/core/src/live_index/crawler/crawlable_site.rs+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -44,7 +44,7 @@ use super::{Client,};-pub const MAX_PENDING_BUDGET: u64 = 128;+pub const MAX_PENDING_BUDGET: u64 = 32;struct InnerCrawlableSite {site: site_stats::Site,diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex a0507a40..804d6521 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -225,6 +225,7 @@ impl InnerIndex {.write_ahead_log.iter().unwrap()+ .unique_by(|page| page.url.clone()).chunks(BATCH_SIZE).into_iter(){@@ -233,6 +234,7 @@ impl InnerIndex {self.index.insert(&webpage).unwrap();}}+ self.write_ahead_log.clear().unwrap();self.index.commit().unwrap();self.update_meta();self.has_inserts = false;@@ -271,6 +273,7 @@ impl LiveIndex {.unwrap_or_else(|e| e.into_inner()).commit(),);+ tracing::debug!("index committed");}pub fn prune_segments(&self) {@@ -297,7 +300,7 @@ impl LiveIndex {}pub fn insert(&self, pages: &[IndexableWebpage]) {- tracing::debug!("inserting pages into index");+ tracing::debug!("inserting {} pages into index", pages.len());self.inner.write().unwrap_or_else(|e| e.into_inner())