Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Centralize validated location reads

Manav Rathi7e01f52ce0ba
diff --git a/rust/crates/location/src/binary.rs b/rust/crates/location/src/binary.rs
index acaf6297f3..090eb6bb61 100644
--- a/rust/crates/location/src/binary.rs
+++ b/rust/crates/location/src/binary.rs
@@ -1,5 +1,52 @@
use std::ops::Range;
+// For field ranges already checked by the index validators.
+pub(crate) struct ByteReader<'a> {
+ bytes: &'a [u8],
+}
+
+impl<'a> ByteReader<'a> {
+ pub(crate) fn at(bytes: &'a [u8], offset: usize) -> Self {
+ Self {
+ bytes: &bytes[offset..],
+ }
+ }
+
+ fn array<const N: usize>(&mut self) -> [u8; N] {
+ let (value, rest) = self
+ .bytes
+ .split_first_chunk::<N>()
+ .expect("validated binary field");
+ self.bytes = rest;
+ *value
+ }
+
+ pub(crate) fn byte(&mut self) -> u8 {
+ self.array::<1>()[0]
+ }
+
+ pub(crate) fn u16(&mut self) -> u16 {
+ u16::from_le_bytes(self.array())
+ }
+
+ pub(crate) fn u24(&mut self) -> u32 {
+ let [a, b, c] = self.array();
+ u32::from_le_bytes([a, b, c, 0])
+ }
+
+ pub(crate) fn u32(&mut self) -> u32 {
+ u32::from_le_bytes(self.array())
+ }
+
+ pub(crate) fn u64(&mut self) -> u64 {
+ u64::from_le_bytes(self.array())
+ }
+
+ pub(crate) fn f32(&mut self) -> f32 {
+ f32::from_le_bytes(self.array())
+ }
+}
+
pub(crate) fn array<const N: usize>(bytes: &[u8], offset: usize) -> Option<[u8; N]> {
bytes.get(offset..offset.checked_add(N)?)?.try_into().ok()
}
diff --git a/rust/crates/location/src/city.rs b/rust/crates/location/src/city.rs
index b15a07eb7d..a3add7a541 100644
--- a/rust/crates/location/src/city.rs
+++ b/rust/crates/location/src/city.rs
@@ -6,7 +6,7 @@ use std::path::Path;
use unicode_normalization::UnicodeNormalization;
use unicode_normalization::char::is_combining_mark;
-use crate::binary::{f32_at, range, u16_at, u24_at, u32_at};
+use crate::binary::{ByteReader, f32_at, range, u16_at, u24_at, u32_at};
use crate::{Coordinate, CountryCode, Error};
const MAGIC: &[u8; 4] = b"CITY";
@@ -465,17 +465,17 @@ impl CityIndex {
}
fn node(&self, tree: Tree, index: usize) -> usize {
- u24_at(&self.bytes, tree.nodes + index * NODE_LEN).expect("validated node") as usize
+ ByteReader::at(&self.bytes, tree.nodes + index * NODE_LEN).u24() as usize
}
fn point(&self, index: usize) -> Point {
- let offset = self.layout.points + index * POINT_LEN;
+ let mut reader = ByteReader::at(&self.bytes, self.layout.points + index * POINT_LEN);
Point {
- latitude: f32_at(&self.bytes, offset).expect("validated point"),
- longitude: f32_at(&self.bytes, offset + 4).expect("validated point"),
- name: u24_at(&self.bytes, offset + 8).expect("validated point") as usize,
- country: usize::from(self.bytes[offset + 11]),
- source_id: u24_at(&self.bytes, offset + 12).expect("validated point"),
+ latitude: reader.f32(),
+ longitude: reader.f32(),
+ name: reader.u24() as usize,
+ country: usize::from(reader.byte()),
+ source_id: reader.u24(),
}
}
@@ -731,9 +731,9 @@ fn validate_tree(bytes: &[u8], layout: Layout, tree: Tree) -> crate::Result<()>
return Err(invalid("invalid tree node"));
}
visited_points[point] = true;
- let point_offset = layout.points + point * POINT_LEN;
- let latitude = f64::from(f32_at(bytes, point_offset).expect("validated point"));
- let longitude = f64::from(f32_at(bytes, point_offset + 4).expect("validated point"));
+ let mut reader = ByteReader::at(bytes, layout.points + point * POINT_LEN);
+ let latitude = f64::from(reader.f32());
+ let longitude = f64::from(reader.f32());
if !(min_lat..=max_lat).contains(&latitude) || !(min_lng..=max_lng).contains(&longitude) {
return Err(invalid("invalid KD ordering"));
}
@@ -858,16 +858,16 @@ fn string_offset(bytes: &[u8], offset: usize, length: usize) -> crate::Result<us
}
fn table_entry_u24(bytes: &[u8], offsets: usize, blob: usize, index: usize) -> &str {
- let start =
- u24_at(bytes, offsets + index * NAME_OFFSET_LEN).expect("validated string table") as usize;
- let end = u24_at(bytes, offsets + (index + 1) * NAME_OFFSET_LEN)
- .expect("validated string table") as usize;
+ let mut reader = ByteReader::at(bytes, offsets + index * NAME_OFFSET_LEN);
+ let start = reader.u24() as usize;
+ let end = reader.u24() as usize;
std::str::from_utf8(&bytes[blob + start..blob + end]).expect("validated UTF-8")
}
fn table_entry_u32(bytes: &[u8], offsets: usize, blob: usize, index: usize) -> &str {
- let start = u32_at(bytes, offsets + index * 4).expect("validated string table") as usize;
- let end = u32_at(bytes, offsets + (index + 1) * 4).expect("validated string table") as usize;
+ let mut reader = ByteReader::at(bytes, offsets + index * 4);
+ let start = reader.u32() as usize;
+ let end = reader.u32() as usize;
std::str::from_utf8(&bytes[blob + start..blob + end]).expect("validated UTF-8")
}
diff --git a/rust/crates/location/src/country.rs b/rust/crates/location/src/country.rs
index e8cab43f1c..d752ced19b 100644
--- a/rust/crates/location/src/country.rs
+++ b/rust/crates/location/src/country.rs
@@ -1,4 +1,4 @@
-use crate::binary::{array, range, u16_at, u32_at};
+use crate::binary::{ByteReader, range, u16_at, u32_at};
use crate::geometry::{Point, Reader};
use crate::{CountryCode, Error};
@@ -115,19 +115,18 @@ impl CountryGeometry {
fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {
let block_index = cell_index / BLOCK_SIZE;
let bit_index = cell_index % BLOCK_SIZE;
- let offset = self.layout.blocks + block_index * BLOCK_LEN;
- let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));
+ let mut reader = ByteReader::at(&self.bytes, self.layout.blocks + block_index * BLOCK_LEN);
+ let mask = reader.u64();
let bit = 1_u64 << bit_index;
if mask & bit == 0 {
return None;
}
- let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;
+ let first = reader.u32() as usize;
Some(first + (mask & bit.wrapping_sub(1)).count_ones() as usize)
}
fn cell_offset(&self, dense_index: usize) -> usize {
- u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)
- .expect("validated cell offset") as usize
+ ByteReader::at(&self.bytes, self.layout.cell_offsets + dense_index * 4).u32() as usize
}
fn country_code(&self, index: u8) -> CountryCode {
@@ -265,10 +264,10 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat
}
let mut expected_dense = 0;
+ let mut reader = ByteReader::at(bytes, layout.blocks);
for block in 0..layout.block_count {
- let offset = layout.blocks + block * BLOCK_LEN;
- let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));
- let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;
+ let mask = reader.u64();
+ let first = reader.u32() as usize;
if first != expected_dense {
return Err(invalid("invalid block dense offset"));
}
@@ -286,9 +285,9 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat
let cells_length = bytes.len() - layout.cells;
let mut previous = 0;
+ let mut reader = ByteReader::at(bytes, layout.cell_offsets);
for index in 0..=layout.nonempty_count {
- let offset = u32_at(bytes, layout.cell_offsets + index * 4).expect("validated offset range")
- as usize;
+ let offset = reader.u32() as usize;
if offset < previous || offset > cells_length {
return Err(invalid("invalid cell offset"));
}
diff --git a/rust/crates/location/src/dispute.rs b/rust/crates/location/src/dispute.rs
index c69bda9519..54d19d78a3 100644
--- a/rust/crates/location/src/dispute.rs
+++ b/rust/crates/location/src/dispute.rs
@@ -1,6 +1,6 @@
use std::str;
-use crate::binary::{range, u16_at, u32_at};
+use crate::binary::{ByteReader, range, u16_at, u32_at};
use crate::country::{CountryGeometry, PreparedCell};
use crate::{CountryCode, Error};
@@ -176,11 +176,11 @@ impl<'a> DisputeMatch<'a> {
fn record(self) -> TerritoryRecord {
let offset = self.record_offset();
TerritoryRecord {
- id: u16_at(&self.index.bytes, offset).expect("validated territory"),
+ id: ByteReader::at(&self.index.bytes, offset).u16(),
default_country: self.index.bytes[offset + 4],
candidate_count: self.index.bytes[offset + 5],
- candidate_start: u16_at(&self.index.bytes, offset + 6).expect("validated territory"),
- name_start: u16_at(&self.index.bytes, offset + 8).expect("validated territory"),
+ candidate_start: ByteReader::at(&self.index.bytes, offset + 6).u16(),
+ name_start: ByteReader::at(&self.index.bytes, offset + 8).u16(),
name_length: self.index.bytes[offset + 10],
}
}
@@ -301,7 +301,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat
let mut geometry_codes = Vec::with_capacity(layout.territory_count);
for territory in 0..layout.territory_count {
let offset = layout.territories + territory * TERRITORY_LEN;
- let id = read_u16_infallible(bytes, offset);
+ let id = ByteReader::at(bytes, offset).u16();
if id == 0 || ids.contains(&id) {
return Err(invalid("invalid or duplicate territory ID"));
}
@@ -313,7 +313,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat
geometry_codes.push(geometry_code);
validate_optional_country(bytes[offset + 4], layout.country_count)?;
- let candidate_start = usize::from(read_u16_infallible(bytes, offset + 6));
+ let candidate_start = usize::from(ByteReader::at(bytes, offset + 6).u16());
let candidate_end = candidate_start
.checked_add(usize::from(bytes[offset + 5]))
.ok_or(invalid("candidate range overflow"))?;
@@ -331,7 +331,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat
validate_optional_country(country, layout.country_count)?;
}
- let name_start = usize::from(read_u16_infallible(bytes, offset + 8));
+ let name_start = usize::from(ByteReader::at(bytes, offset + 8).u16());
let name_length = usize::from(bytes[offset + 10]);
if name_length == 0 {
return Err(invalid("empty territory name"));
@@ -440,10 +440,6 @@ fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {
Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)
}
-fn read_u16_infallible(bytes: &[u8], offset: usize) -> u16 {
- u16_at(bytes, offset).expect("validated territory field")
-}
-
fn invalid(reason: &'static str) -> Error {
Error::invalid(SECTION, reason)
}
diff --git a/rust/crates/location/src/geometry.rs b/rust/crates/location/src/geometry.rs
index ff717bdb5d..4890e25cf2 100644
--- a/rust/crates/location/src/geometry.rs
+++ b/rust/crates/location/src/geometry.rs
@@ -35,9 +35,10 @@ impl<'a> Reader<'a> {
}
pub(crate) fn u16(&mut self) -> crate::Result<u16> {
- Ok(u16::from_le_bytes(
- self.take(2)?.try_into().expect("two-byte geometry slice"),
- ))
+ let value =
+ crate::binary::u16_at(self.bytes, self.position).ok_or(self.invalid("truncated"))?;
+ self.position += 2;
+ Ok(value)
}
pub(crate) fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {
diff --git a/rust/crates/location/src/urban.rs b/rust/crates/location/src/urban.rs
index 4939b95b95..9c2025e1a2 100644
--- a/rust/crates/location/src/urban.rs
+++ b/rust/crates/location/src/urban.rs
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
-use crate::binary::{array, f32_at, range, u16_at, u32_at};
+use crate::binary::{ByteReader, f32_at, range, u16_at, u32_at};
use crate::city::SearchPattern;
use crate::geometry::{Point, Reader as GeometryReader};
use crate::{City, CityMatch, Coordinate, CountryCode, Error};
@@ -129,8 +129,7 @@ impl UrbanCenterIndex {
let end = self.cell_offset(dense + 1);
for reference in start..end {
let feature = usize::from(
- u16_at(&self.bytes, self.layout.references + reference * 2)
- .expect("validated reference"),
+ ByteReader::at(&self.bytes, self.layout.references + reference * 2).u16(),
);
if matching.is_some_and(|matching| !matching[feature]) {
continue;
@@ -199,19 +198,19 @@ impl UrbanCenterIndex {
}
fn feature(&self, index: usize) -> Feature {
- let offset = self.layout.records + index * RECORD_LEN;
+ let mut reader = ByteReader::at(&self.bytes, self.layout.records + index * RECORD_LEN);
Feature {
- minimum_longitude: f32_at(&self.bytes, offset).expect("validated feature"),
- minimum_latitude: f32_at(&self.bytes, offset + 4).expect("validated feature"),
- maximum_longitude: f32_at(&self.bytes, offset + 8).expect("validated feature"),
- maximum_latitude: f32_at(&self.bytes, offset + 12).expect("validated feature"),
- latitude: f32_at(&self.bytes, offset + 16).expect("validated feature"),
- longitude: f32_at(&self.bytes, offset + 20).expect("validated feature"),
- name: u32_at(&self.bytes, offset + 24).expect("validated feature") as usize,
- geometry_start: u32_at(&self.bytes, offset + 28).expect("validated feature") as usize,
- geometry_end: u32_at(&self.bytes, offset + 32).expect("validated feature") as usize,
- source_id: u32_at(&self.bytes, offset + 36).expect("validated feature"),
- country: usize::from(self.bytes[offset + 40]),
+ minimum_longitude: reader.f32(),
+ minimum_latitude: reader.f32(),
+ maximum_longitude: reader.f32(),
+ maximum_latitude: reader.f32(),
+ latitude: reader.f32(),
+ longitude: reader.f32(),
+ name: reader.u32() as usize,
+ geometry_start: reader.u32() as usize,
+ geometry_end: reader.u32() as usize,
+ source_id: reader.u32(),
+ country: usize::from(reader.byte()),
}
}
@@ -225,10 +224,9 @@ impl UrbanCenterIndex {
}
fn country(&self, index: usize) -> &str {
- let start = u32_at(&self.bytes, self.layout.country_offsets + index * 4)
- .expect("validated country offset") as usize;
- let end = u32_at(&self.bytes, self.layout.country_offsets + (index + 1) * 4)
- .expect("validated country offset") as usize;
+ let mut reader = ByteReader::at(&self.bytes, self.layout.country_offsets + index * 4);
+ let start = reader.u32() as usize;
+ let end = reader.u32() as usize;
std::str::from_utf8(&self.bytes[self.layout.countries + start..self.layout.countries + end])
.expect("validated country")
}
@@ -241,18 +239,17 @@ impl UrbanCenterIndex {
fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {
let block_index = cell_index / BLOCK_SIZE;
let bit_index = cell_index % BLOCK_SIZE;
- let offset = self.layout.blocks + block_index * BLOCK_LEN;
- let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));
+ let mut reader = ByteReader::at(&self.bytes, self.layout.blocks + block_index * BLOCK_LEN);
+ let mask = reader.u64();
let bit = 1_u64 << bit_index;
(mask & bit != 0).then(|| {
- let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;
+ let first = reader.u32() as usize;
first + (mask & bit.wrapping_sub(1)).count_ones() as usize
})
}
fn cell_offset(&self, dense_index: usize) -> usize {
- u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)
- .expect("validated cell offset") as usize
+ ByteReader::at(&self.bytes, self.layout.cell_offsets + dense_index * 4).u32() as usize
}
}
@@ -458,10 +455,10 @@ fn validate_geometry(bytes: &[u8]) -> crate::Result<()> {
fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Result<()> {
let mut expected_dense = 0;
+ let mut reader = ByteReader::at(bytes, layout.blocks);
for block in 0..layout.block_count {
- let offset = layout.blocks + block * BLOCK_LEN;
- let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));
- let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;
+ let mask = reader.u64();
+ let first = reader.u32() as usize;
if first != expected_dense {
return Err(invalid("invalid block dense offset"));
}
@@ -477,9 +474,9 @@ fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Resu
return Err(invalid("wrong nonempty cell count"));
}
let mut previous = 0;
- for index in 0..=layout.nonempty_count {
- let offset = u32_at(bytes, layout.cell_offsets + index * 4)
- .expect("validated cell offset range") as usize;
+ let mut reader = ByteReader::at(bytes, layout.cell_offsets);
+ for _ in 0..=layout.nonempty_count {
+ let offset = reader.u32() as usize;
if offset < previous || offset > layout.reference_count {
return Err(invalid("invalid cell offset"));
}
@@ -488,10 +485,9 @@ fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Resu
if previous != layout.reference_count {
return Err(invalid("last cell offset is not terminal"));
}
- for index in 0..layout.reference_count {
- let feature = usize::from(
- u16_at(bytes, layout.references + index * 2).expect("validated reference range"),
- );
+ let mut reader = ByteReader::at(bytes, layout.references);
+ for _ in 0..layout.reference_count {
+ let feature = usize::from(reader.u16());
if feature >= layout.feature_count {
return Err(invalid("cell references unknown feature"));
}