Skip to content

web search done right

webgraph didn't properly merge segments. The new segment paths didn't line up with where the segment should actually be stored. This should now be fixed

Mikkel Denkerea231fc780df
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index 9d3a11c2..bbdc0389 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -1,4 +1,4 @@
-limit_warc_files = 10
+limit_warc_files = 5
graph_base_path = "data/webgraph"
batch_size = 1
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 138d9537..a2754950 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -319,6 +319,7 @@ impl Webgraph {
.create(true)
.write(true)
.read(true)
+ .truncate(true)
.open(path)
.unwrap(),
);
@@ -554,6 +555,7 @@ impl Webgraph {
.unwrap();
self.meta.comitted_segments = self.segments.iter().map(|segment| segment.id()).collect();
+ self.save_metadata();
self.garbage_collect();
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index d1f5936f..f0446546 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -278,12 +278,9 @@ impl StoredSegment {
let new_segment_id = uuid::Uuid::new_v4().to_string();
- let new_adjacency = Store::open(
- segments[0].path().join(&new_segment_id).join("adjacency"),
- &new_segment_id,
- )
- .unwrap();
- let folder_path = segments[0].folder_path.clone();
+ let new_path = segments[0].path().parent().unwrap().join(&new_segment_id);
+
+ let new_adjacency = Store::open(new_path.join("adjacency"), &new_segment_id).unwrap();
let next_segment_node_id = SegmentNodeID(0);
@@ -311,14 +308,8 @@ impl StoredSegment {
);
let new_adjacency = packet.new_adjacency;
- let new_reversed_adjacency = Store::open(
- segments[0]
- .path()
- .join(&new_segment_id)
- .join("reversed_adjacency"),
- &new_segment_id,
- )
- .unwrap();
+ let new_reversed_adjacency =
+ Store::open(new_path.join("reversed_adjacency"), &new_segment_id).unwrap();
let mut packet = MergePacket {
next_segment_node_id: packet.next_segment_node_id,
@@ -350,7 +341,7 @@ impl StoredSegment {
num_nodes: packet.next_segment_node_id.0,
},
id: new_segment_id,
- folder_path,
+ folder_path: new_path.to_str().unwrap().to_string(),
};
res.flush();
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 6fe78f33..1e444d82 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -158,7 +158,7 @@ import { askama } from "../askama";
{
askama.if_(
"let Some(sidebar) = sidebar",
- <div class="row-start-2 justify-center md:pt-10 md:col-start-2 md:row-span-2 md:row-start-1" style="max-width: 90vw; margin-left: auto; margin-right: auto;">
+ <div class="row-start-2 justify-center md:pt-10 md:col-start-2 md:row-span-2 md:row-start-1 max-w-[90vw] mx-auto">
<Sidebar sidebar="sidebar" />
</div>
)
diff --git a/git-assets/screenshot.png b/git-assets/screenshot.png
index d22f7508..0428e1f1 100644
Binary files a/git-assets/screenshot.png and b/git-assets/screenshot.png differ