Skip to content

web search done right

Fixed alignment issue for rkyv. Subslice of a memory mapped file might have alignmet 1 but should have atleast 8 to work well with rkyv

Mikkel Denker2258da4ca277
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 02ae0ba4..1c248e28 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use rkyv::{Archive, Deserialize};
+use rkyv::Archive;
use std::{
collections::{BTreeMap, HashSet},
fmt::Debug,
@@ -100,15 +100,13 @@ impl StoredSegment {
.and_then(|segment_id| {
self.adjacency.get(&segment_id).map(|edges| {
edges
- .iter()
+ .into_iter()
.map(move |edge| {
if load_label {
Edge {
from: *node,
to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),
- label: Loaded::Some(
- edge.label.deserialize(&mut rkyv::Infallible).unwrap(),
- ),
+ label: Loaded::Some(edge.label),
}
} else {
Edge {
@@ -129,7 +127,7 @@ impl StoredSegment {
.and_then(|segment_id| {
self.reversed_adjacency.get(&segment_id).map(|edges| {
edges
- .iter()
+ .into_iter()
.map(move |edge| {
if load_label {
Edge {
@@ -137,9 +135,7 @@ impl StoredSegment {
.rev_id_mapping(&SegmentNodeID(edge.other.0))
.unwrap(),
to: *node,
- label: Loaded::Some(
- edge.label.deserialize(&mut rkyv::Infallible).unwrap(),
- ),
+ label: Loaded::Some(edge.label),
}
} else {
Edge {
@@ -177,7 +173,7 @@ impl StoredSegment {
self.adjacency.iter().flat_map(move |(node_id, edges)| {
let from = self.rev_id_mapping(node_id).unwrap();
- edges.iter().map(move |stored_edge| Edge {
+ edges.into_iter().map(move |stored_edge| Edge {
from,
to: self
.rev_id_mapping(&SegmentNodeID(stored_edge.other.0))
@@ -296,11 +292,7 @@ impl StoredSegment {
Self::merge_adjacency(
|segment: &StoredSegment, node_id: &NodeID| {
segment.id_mapping.get(node_id).map(|segment_node_id| {
- segment
- .adjacency
- .get(segment_node_id)
- .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())
- .unwrap_or_default()
+ segment.adjacency.get(segment_node_id).unwrap_or_default()
})
},
&mut packet,
@@ -324,7 +316,6 @@ impl StoredSegment {
segment
.reversed_adjacency
.get(segment_node_id)
- .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())
.unwrap_or_default()
})
},
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 88b8a158..e7394953 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -22,7 +22,7 @@ use std::{
path::{Path, PathBuf},
};
-use rkyv::{ser::serializers::AllocSerializer, Archived};
+use rkyv::{ser::serializers::AllocSerializer, AlignedVec};
type Result<T> = std::result::Result<T, Error>;
@@ -55,7 +55,8 @@ pub struct Store<K, V> {
impl<K, V> Store<K, V>
where
K: serde::de::DeserializeOwned + serde::Serialize + Ord,
- V: rkyv::Archive,
+ V: rkyv::Archive + 'static,
+ V::Archived: rkyv::Deserialize<V, rkyv::de::deserializers::SharedDeserializeMap> + 'static,
{
pub fn open<P: AsRef<Path>>(folder_path: P, segment_name: &str) -> Result<Self> {
if folder_path.as_ref().is_file() {
@@ -110,19 +111,34 @@ where
})
}
- pub fn get(&self, key: &K) -> Option<&Archived<V>> {
+ pub fn get(&self, key: &K) -> Option<V> {
let range = self.locations.get(key)?;
let bytes = &self.map[range.clone()];
- let archived = unsafe { rkyv::archived_root::<V>(bytes) };
- Some(archived)
+ let mut aligned = AlignedVec::with_capacity(bytes.len());
+ aligned.extend_from_slice(bytes);
+
+ // SAFETY:
+ // We ensure the target `V` is `'static` and contains only owned data so it's safe to
+ // temporarily extend the lifetime so we can allocate the type entirely.
+ let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };
+
+ Some(v)
}
- pub fn iter(&self) -> impl Iterator<Item = (&K, &Archived<V>)> + '_ {
+ pub fn iter(&self) -> impl Iterator<Item = (&K, V)> + '_ {
self.locations.iter().map(move |(key, range)| {
let bytes = &self.map[range.clone()];
- let archived = unsafe { rkyv::archived_root::<V>(bytes) };
- (key, archived)
+
+ let mut aligned = AlignedVec::with_capacity(bytes.len());
+ aligned.extend_from_slice(bytes);
+
+ // SAFETY:
+ // We ensure the target `V` is `'static` and contains only owned data so it's safe to
+ // temporarily extend the lifetime so we can allocate the type entirely.
+ let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };
+
+ (key, v)
})
}
@@ -135,11 +151,12 @@ where
}
let bytes = rkyv::to_bytes::<_, SCRATCH_SIZE>(value).unwrap();
- self.insert_raw(key, &bytes)
+ self.insert_raw(key, &bytes[..])
}
fn insert_raw(&mut self, key: K, bytes: &[u8]) -> Result<()> {
let old_end = self.cur_end_range;
+
let new_end = bytes.len() + self.cur_end_range;
if new_end > self.map.len() {
@@ -240,7 +257,7 @@ mod tests {
kv.flush().unwrap();
let archived = kv.get(&"test".to_string()).unwrap();
- assert_eq!(&test_struct, archived);
+ assert_eq!(test_struct, archived);
}
#[test]
@@ -264,7 +281,7 @@ mod tests {
let kv = Store::<String, TestStruct>::open(path, segment_name).unwrap();
let archived = kv.get(&"test".to_string()).unwrap();
- assert_eq!(&test_struct, archived);
+ assert_eq!(test_struct, archived);
}
#[test]
@@ -283,7 +300,7 @@ mod tests {
kv.flush().unwrap();
let archived = kv.get(&"test".to_string()).unwrap();
- assert_eq!(&test_struct, archived);
+ assert_eq!(test_struct, archived);
let test_struct = TestStruct {
a: "test".to_string(),