web search done right
index compaction shouldn’t update creation date as this would cause all segments to eventaully have same creation date
Mikkel Denker87fbd3d7095d
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex 5b6ec21b..fa7c7fe5 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -154,20 +154,21 @@ impl InvertedIndex {}#[allow(clippy::missing_panics_doc)] // cannot panic as writer is prepared- pub fn merge_segments_by_id(&mut self, segments: &[SegmentId]) -> Result<()> {+ pub fn merge_segments_by_id(&mut self, segments: &[SegmentId]) -> Result<Option<SegmentId>> {self.prepare_writer()?;if segments.is_empty() {- return Ok(());+ return Ok(None);}- self.writer+ let res = self+ .writer.as_mut().expect("writer has not been prepared").merge(segments).wait()?;- Ok(())+ Ok(res.map(|seg| seg.id()))}#[must_use]diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex 2406ed3c..f812ff96 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -141,17 +141,17 @@ impl InnerIndex {.delete_segments_by_id(&old_segments).unwrap();- self.update_meta();+ self.sync_meta_with_index();}pub fn compact_segments_by_date(&mut self) {- let mut segments_by_date: HashMap<NaiveDate, Vec<SegmentId>> = HashMap::new();+ let mut segments_by_date: HashMap<NaiveDate, Vec<Segment>> = HashMap::new();for segment in self.meta.segments.clone() {segments_by_date.entry(segment.created.date_naive()).or_default()- .push(segment.id);+ .push(segment);}for (_, segments) in segments_by_date {@@ -159,13 +159,22 @@ impl InnerIndex {continue;}- self.index- .inverted_index- .merge_segments_by_id(&segments)- .unwrap();+ let segment_ids: Vec<SegmentId> = segments.iter().map(|s| s.id).collect();+ let newest_creation_date = segments.iter().map(|s| s.created).max().unwrap();++ if let Ok(Some(new_segment_id)) =+ self.index.inverted_index.merge_segments_by_id(&segment_ids)+ {+ // Update meta with the new segment, using the newest creation date+ self.meta.segments.retain(|s| !segment_ids.contains(&s.id));+ self.meta.segments.push(Segment {+ id: new_segment_id,+ created: newest_creation_date,+ });+ }}- self.update_meta();+ self.save_meta();self.re_open();}@@ -174,7 +183,7 @@ impl InnerIndex {self.index.prepare_writer().unwrap();}- fn update_meta(&mut self) {+ fn sync_meta_with_index(&mut self) {let segments_in_index: HashSet<_> = self.index.inverted_index@@ -259,7 +268,7 @@ impl InnerIndex {}self.index.commit().unwrap();self.write_ahead_log.clear().unwrap();- self.update_meta();+ self.sync_meta_with_index();self.has_inserts = false;self.re_open();}