web search done right
Fixed alignment issue for rkyv. Subslice of a memory mapped file might have alignmet 1 but should have atleast 8 to work well with rkyv
Mikkel Denker2258da4ca277
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 02ae0ba4..1c248e28 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use rkyv::{Archive, Deserialize};+use rkyv::Archive;use std::{collections::{BTreeMap, HashSet},fmt::Debug,@@ -100,15 +100,13 @@ impl StoredSegment {.and_then(|segment_id| {self.adjacency.get(&segment_id).map(|edges| {edges- .iter()+ .into_iter().map(move |edge| {if load_label {Edge {from: *node,to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- label: Loaded::Some(- edge.label.deserialize(&mut rkyv::Infallible).unwrap(),- ),+ label: Loaded::Some(edge.label),}} else {Edge {@@ -129,7 +127,7 @@ impl StoredSegment {.and_then(|segment_id| {self.reversed_adjacency.get(&segment_id).map(|edges| {edges- .iter()+ .into_iter().map(move |edge| {if load_label {Edge {@@ -137,9 +135,7 @@ impl StoredSegment {.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),to: *node,- label: Loaded::Some(- edge.label.deserialize(&mut rkyv::Infallible).unwrap(),- ),+ label: Loaded::Some(edge.label),}} else {Edge {@@ -177,7 +173,7 @@ impl StoredSegment {self.adjacency.iter().flat_map(move |(node_id, edges)| {let from = self.rev_id_mapping(node_id).unwrap();- edges.iter().map(move |stored_edge| Edge {+ edges.into_iter().map(move |stored_edge| Edge {from,to: self.rev_id_mapping(&SegmentNodeID(stored_edge.other.0))@@ -296,11 +292,7 @@ impl StoredSegment {Self::merge_adjacency(|segment: &StoredSegment, node_id: &NodeID| {segment.id_mapping.get(node_id).map(|segment_node_id| {- segment- .adjacency- .get(segment_node_id)- .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())- .unwrap_or_default()+ segment.adjacency.get(segment_node_id).unwrap_or_default()})},&mut packet,@@ -324,7 +316,6 @@ impl StoredSegment {segment.reversed_adjacency.get(segment_node_id)- .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap()).unwrap_or_default()})},diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 88b8a158..e7394953 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -22,7 +22,7 @@ use std::{path::{Path, PathBuf},};-use rkyv::{ser::serializers::AllocSerializer, Archived};+use rkyv::{ser::serializers::AllocSerializer, AlignedVec};type Result<T> = std::result::Result<T, Error>;@@ -55,7 +55,8 @@ pub struct Store<K, V> {impl<K, V> Store<K, V>whereK: serde::de::DeserializeOwned + serde::Serialize + Ord,- V: rkyv::Archive,+ V: rkyv::Archive + 'static,+ V::Archived: rkyv::Deserialize<V, rkyv::de::deserializers::SharedDeserializeMap> + 'static,{pub fn open<P: AsRef<Path>>(folder_path: P, segment_name: &str) -> Result<Self> {if folder_path.as_ref().is_file() {@@ -110,19 +111,34 @@ where})}- pub fn get(&self, key: &K) -> Option<&Archived<V>> {+ pub fn get(&self, key: &K) -> Option<V> {let range = self.locations.get(key)?;let bytes = &self.map[range.clone()];- let archived = unsafe { rkyv::archived_root::<V>(bytes) };- Some(archived)+ let mut aligned = AlignedVec::with_capacity(bytes.len());+ aligned.extend_from_slice(bytes);++ // SAFETY:+ // We ensure the target `V` is `'static` and contains only owned data so it's safe to+ // temporarily extend the lifetime so we can allocate the type entirely.+ let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };++ Some(v)}- pub fn iter(&self) -> impl Iterator<Item = (&K, &Archived<V>)> + '_ {+ pub fn iter(&self) -> impl Iterator<Item = (&K, V)> + '_ {self.locations.iter().map(move |(key, range)| {let bytes = &self.map[range.clone()];- let archived = unsafe { rkyv::archived_root::<V>(bytes) };- (key, archived)++ let mut aligned = AlignedVec::with_capacity(bytes.len());+ aligned.extend_from_slice(bytes);++ // SAFETY:+ // We ensure the target `V` is `'static` and contains only owned data so it's safe to+ // temporarily extend the lifetime so we can allocate the type entirely.+ let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };++ (key, v)})}@@ -135,11 +151,12 @@ where}let bytes = rkyv::to_bytes::<_, SCRATCH_SIZE>(value).unwrap();- self.insert_raw(key, &bytes)+ self.insert_raw(key, &bytes[..])}fn insert_raw(&mut self, key: K, bytes: &[u8]) -> Result<()> {let old_end = self.cur_end_range;+let new_end = bytes.len() + self.cur_end_range;if new_end > self.map.len() {@@ -240,7 +257,7 @@ mod tests {kv.flush().unwrap();let archived = kv.get(&"test".to_string()).unwrap();- assert_eq!(&test_struct, archived);+ assert_eq!(test_struct, archived);}#[test]@@ -264,7 +281,7 @@ mod tests {let kv = Store::<String, TestStruct>::open(path, segment_name).unwrap();let archived = kv.get(&"test".to_string()).unwrap();- assert_eq!(&test_struct, archived);+ assert_eq!(test_struct, archived);}#[test]@@ -283,7 +300,7 @@ mod tests {kv.flush().unwrap();let archived = kv.get(&"test".to_string()).unwrap();- assert_eq!(&test_struct, archived);+ assert_eq!(test_struct, archived);let test_struct = TestStruct {a: "test".to_string(),