web search done right
webgraph didn't properly merge segments. The new segment paths didn't line up with where the segment should actually be stored. This should now be fixed
Mikkel Denkerea231fc780df
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.tomlindex 9d3a11c2..bbdc0389 100644--- a/configs/webgraph/local.toml+++ b/configs/webgraph/local.toml@@ -1,4 +1,4 @@-limit_warc_files = 10+limit_warc_files = 5graph_base_path = "data/webgraph"batch_size = 1diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 138d9537..a2754950 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -319,6 +319,7 @@ impl Webgraph {.create(true).write(true).read(true)+ .truncate(true).open(path).unwrap(),);@@ -554,6 +555,7 @@ impl Webgraph {.unwrap();self.meta.comitted_segments = self.segments.iter().map(|segment| segment.id()).collect();+ self.save_metadata();self.garbage_collect();}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex d1f5936f..f0446546 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -278,12 +278,9 @@ impl StoredSegment {let new_segment_id = uuid::Uuid::new_v4().to_string();- let new_adjacency = Store::open(- segments[0].path().join(&new_segment_id).join("adjacency"),- &new_segment_id,- )- .unwrap();- let folder_path = segments[0].folder_path.clone();+ let new_path = segments[0].path().parent().unwrap().join(&new_segment_id);++ let new_adjacency = Store::open(new_path.join("adjacency"), &new_segment_id).unwrap();let next_segment_node_id = SegmentNodeID(0);@@ -311,14 +308,8 @@ impl StoredSegment {);let new_adjacency = packet.new_adjacency;- let new_reversed_adjacency = Store::open(- segments[0]- .path()- .join(&new_segment_id)- .join("reversed_adjacency"),- &new_segment_id,- )- .unwrap();+ let new_reversed_adjacency =+ Store::open(new_path.join("reversed_adjacency"), &new_segment_id).unwrap();let mut packet = MergePacket {next_segment_node_id: packet.next_segment_node_id,@@ -350,7 +341,7 @@ impl StoredSegment {num_nodes: packet.next_segment_node_id.0,},id: new_segment_id,- folder_path,+ folder_path: new_path.to_str().unwrap().to_string(),};res.flush();diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex 6fe78f33..1e444d82 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -158,7 +158,7 @@ import { askama } from "../askama";{askama.if_("let Some(sidebar) = sidebar",- <div class="row-start-2 justify-center md:pt-10 md:col-start-2 md:row-span-2 md:row-start-1" style="max-width: 90vw; margin-left: auto; margin-right: auto;">+ <div class="row-start-2 justify-center md:pt-10 md:col-start-2 md:row-span-2 md:row-start-1 max-w-[90vw] mx-auto"><Sidebar sidebar="sidebar" /></div>)diff --git a/git-assets/screenshot.png b/git-assets/screenshot.pngindex d22f7508..0428e1f1 100644Binary files a/git-assets/screenshot.png and b/git-assets/screenshot.png differ