π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Centralize validated location reads
Manav Rathi7e01f52ce0ba
diff --git a/rust/crates/location/src/binary.rs b/rust/crates/location/src/binary.rsindex acaf6297f3..090eb6bb61 100644--- a/rust/crates/location/src/binary.rs+++ b/rust/crates/location/src/binary.rs@@ -1,5 +1,52 @@use std::ops::Range;+// For field ranges already checked by the index validators.+pub(crate) struct ByteReader<'a> {+ bytes: &'a [u8],+}++impl<'a> ByteReader<'a> {+ pub(crate) fn at(bytes: &'a [u8], offset: usize) -> Self {+ Self {+ bytes: &bytes[offset..],+ }+ }++ fn array<const N: usize>(&mut self) -> [u8; N] {+ let (value, rest) = self+ .bytes+ .split_first_chunk::<N>()+ .expect("validated binary field");+ self.bytes = rest;+ *value+ }++ pub(crate) fn byte(&mut self) -> u8 {+ self.array::<1>()[0]+ }++ pub(crate) fn u16(&mut self) -> u16 {+ u16::from_le_bytes(self.array())+ }++ pub(crate) fn u24(&mut self) -> u32 {+ let [a, b, c] = self.array();+ u32::from_le_bytes([a, b, c, 0])+ }++ pub(crate) fn u32(&mut self) -> u32 {+ u32::from_le_bytes(self.array())+ }++ pub(crate) fn u64(&mut self) -> u64 {+ u64::from_le_bytes(self.array())+ }++ pub(crate) fn f32(&mut self) -> f32 {+ f32::from_le_bytes(self.array())+ }+}+pub(crate) fn array<const N: usize>(bytes: &[u8], offset: usize) -> Option<[u8; N]> {bytes.get(offset..offset.checked_add(N)?)?.try_into().ok()}diff --git a/rust/crates/location/src/city.rs b/rust/crates/location/src/city.rsindex b15a07eb7d..a3add7a541 100644--- a/rust/crates/location/src/city.rs+++ b/rust/crates/location/src/city.rs@@ -6,7 +6,7 @@ use std::path::Path;use unicode_normalization::UnicodeNormalization;use unicode_normalization::char::is_combining_mark;-use crate::binary::{f32_at, range, u16_at, u24_at, u32_at};+use crate::binary::{ByteReader, f32_at, range, u16_at, u24_at, u32_at};use crate::{Coordinate, CountryCode, Error};const MAGIC: &[u8; 4] = b"CITY";@@ -465,17 +465,17 @@ impl CityIndex {}fn node(&self, tree: Tree, index: usize) -> usize {- u24_at(&self.bytes, tree.nodes + index * NODE_LEN).expect("validated node") as usize+ ByteReader::at(&self.bytes, tree.nodes + index * NODE_LEN).u24() as usize}fn point(&self, index: usize) -> Point {- let offset = self.layout.points + index * POINT_LEN;+ let mut reader = ByteReader::at(&self.bytes, self.layout.points + index * POINT_LEN);Point {- latitude: f32_at(&self.bytes, offset).expect("validated point"),- longitude: f32_at(&self.bytes, offset + 4).expect("validated point"),- name: u24_at(&self.bytes, offset + 8).expect("validated point") as usize,- country: usize::from(self.bytes[offset + 11]),- source_id: u24_at(&self.bytes, offset + 12).expect("validated point"),+ latitude: reader.f32(),+ longitude: reader.f32(),+ name: reader.u24() as usize,+ country: usize::from(reader.byte()),+ source_id: reader.u24(),}}@@ -731,9 +731,9 @@ fn validate_tree(bytes: &[u8], layout: Layout, tree: Tree) -> crate::Result<()>return Err(invalid("invalid tree node"));}visited_points[point] = true;- let point_offset = layout.points + point * POINT_LEN;- let latitude = f64::from(f32_at(bytes, point_offset).expect("validated point"));- let longitude = f64::from(f32_at(bytes, point_offset + 4).expect("validated point"));+ let mut reader = ByteReader::at(bytes, layout.points + point * POINT_LEN);+ let latitude = f64::from(reader.f32());+ let longitude = f64::from(reader.f32());if !(min_lat..=max_lat).contains(&latitude) || !(min_lng..=max_lng).contains(&longitude) {return Err(invalid("invalid KD ordering"));}@@ -858,16 +858,16 @@ fn string_offset(bytes: &[u8], offset: usize, length: usize) -> crate::Result<us}fn table_entry_u24(bytes: &[u8], offsets: usize, blob: usize, index: usize) -> &str {- let start =- u24_at(bytes, offsets + index * NAME_OFFSET_LEN).expect("validated string table") as usize;- let end = u24_at(bytes, offsets + (index + 1) * NAME_OFFSET_LEN)- .expect("validated string table") as usize;+ let mut reader = ByteReader::at(bytes, offsets + index * NAME_OFFSET_LEN);+ let start = reader.u24() as usize;+ let end = reader.u24() as usize;std::str::from_utf8(&bytes[blob + start..blob + end]).expect("validated UTF-8")}fn table_entry_u32(bytes: &[u8], offsets: usize, blob: usize, index: usize) -> &str {- let start = u32_at(bytes, offsets + index * 4).expect("validated string table") as usize;- let end = u32_at(bytes, offsets + (index + 1) * 4).expect("validated string table") as usize;+ let mut reader = ByteReader::at(bytes, offsets + index * 4);+ let start = reader.u32() as usize;+ let end = reader.u32() as usize;std::str::from_utf8(&bytes[blob + start..blob + end]).expect("validated UTF-8")}diff --git a/rust/crates/location/src/country.rs b/rust/crates/location/src/country.rsindex e8cab43f1c..d752ced19b 100644--- a/rust/crates/location/src/country.rs+++ b/rust/crates/location/src/country.rs@@ -1,4 +1,4 @@-use crate::binary::{array, range, u16_at, u32_at};+use crate::binary::{ByteReader, range, u16_at, u32_at};use crate::geometry::{Point, Reader};use crate::{CountryCode, Error};@@ -115,19 +115,18 @@ impl CountryGeometry {fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {let block_index = cell_index / BLOCK_SIZE;let bit_index = cell_index % BLOCK_SIZE;- let offset = self.layout.blocks + block_index * BLOCK_LEN;- let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));+ let mut reader = ByteReader::at(&self.bytes, self.layout.blocks + block_index * BLOCK_LEN);+ let mask = reader.u64();let bit = 1_u64 << bit_index;if mask & bit == 0 {return None;}- let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;+ let first = reader.u32() as usize;Some(first + (mask & bit.wrapping_sub(1)).count_ones() as usize)}fn cell_offset(&self, dense_index: usize) -> usize {- u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)- .expect("validated cell offset") as usize+ ByteReader::at(&self.bytes, self.layout.cell_offsets + dense_index * 4).u32() as usize}fn country_code(&self, index: u8) -> CountryCode {@@ -265,10 +264,10 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat}let mut expected_dense = 0;+ let mut reader = ByteReader::at(bytes, layout.blocks);for block in 0..layout.block_count {- let offset = layout.blocks + block * BLOCK_LEN;- let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));- let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;+ let mask = reader.u64();+ let first = reader.u32() as usize;if first != expected_dense {return Err(invalid("invalid block dense offset"));}@@ -286,9 +285,9 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> cratlet cells_length = bytes.len() - layout.cells;let mut previous = 0;+ let mut reader = ByteReader::at(bytes, layout.cell_offsets);for index in 0..=layout.nonempty_count {- let offset = u32_at(bytes, layout.cell_offsets + index * 4).expect("validated offset range")- as usize;+ let offset = reader.u32() as usize;if offset < previous || offset > cells_length {return Err(invalid("invalid cell offset"));}diff --git a/rust/crates/location/src/dispute.rs b/rust/crates/location/src/dispute.rsindex c69bda9519..54d19d78a3 100644--- a/rust/crates/location/src/dispute.rs+++ b/rust/crates/location/src/dispute.rs@@ -1,6 +1,6 @@use std::str;-use crate::binary::{range, u16_at, u32_at};+use crate::binary::{ByteReader, range, u16_at, u32_at};use crate::country::{CountryGeometry, PreparedCell};use crate::{CountryCode, Error};@@ -176,11 +176,11 @@ impl<'a> DisputeMatch<'a> {fn record(self) -> TerritoryRecord {let offset = self.record_offset();TerritoryRecord {- id: u16_at(&self.index.bytes, offset).expect("validated territory"),+ id: ByteReader::at(&self.index.bytes, offset).u16(),default_country: self.index.bytes[offset + 4],candidate_count: self.index.bytes[offset + 5],- candidate_start: u16_at(&self.index.bytes, offset + 6).expect("validated territory"),- name_start: u16_at(&self.index.bytes, offset + 8).expect("validated territory"),+ candidate_start: ByteReader::at(&self.index.bytes, offset + 6).u16(),+ name_start: ByteReader::at(&self.index.bytes, offset + 8).u16(),name_length: self.index.bytes[offset + 10],}}@@ -301,7 +301,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> cratlet mut geometry_codes = Vec::with_capacity(layout.territory_count);for territory in 0..layout.territory_count {let offset = layout.territories + territory * TERRITORY_LEN;- let id = read_u16_infallible(bytes, offset);+ let id = ByteReader::at(bytes, offset).u16();if id == 0 || ids.contains(&id) {return Err(invalid("invalid or duplicate territory ID"));}@@ -313,7 +313,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> cratgeometry_codes.push(geometry_code);validate_optional_country(bytes[offset + 4], layout.country_count)?;- let candidate_start = usize::from(read_u16_infallible(bytes, offset + 6));+ let candidate_start = usize::from(ByteReader::at(bytes, offset + 6).u16());let candidate_end = candidate_start.checked_add(usize::from(bytes[offset + 5])).ok_or(invalid("candidate range overflow"))?;@@ -331,7 +331,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> cratvalidate_optional_country(country, layout.country_count)?;}- let name_start = usize::from(read_u16_infallible(bytes, offset + 8));+ let name_start = usize::from(ByteReader::at(bytes, offset + 8).u16());let name_length = usize::from(bytes[offset + 10]);if name_length == 0 {return Err(invalid("empty territory name"));@@ -440,10 +440,6 @@ fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)}-fn read_u16_infallible(bytes: &[u8], offset: usize) -> u16 {- u16_at(bytes, offset).expect("validated territory field")-}-fn invalid(reason: &'static str) -> Error {Error::invalid(SECTION, reason)}diff --git a/rust/crates/location/src/geometry.rs b/rust/crates/location/src/geometry.rsindex ff717bdb5d..4890e25cf2 100644--- a/rust/crates/location/src/geometry.rs+++ b/rust/crates/location/src/geometry.rs@@ -35,9 +35,10 @@ impl<'a> Reader<'a> {}pub(crate) fn u16(&mut self) -> crate::Result<u16> {- Ok(u16::from_le_bytes(- self.take(2)?.try_into().expect("two-byte geometry slice"),- ))+ let value =+ crate::binary::u16_at(self.bytes, self.position).ok_or(self.invalid("truncated"))?;+ self.position += 2;+ Ok(value)}pub(crate) fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {diff --git a/rust/crates/location/src/urban.rs b/rust/crates/location/src/urban.rsindex 4939b95b95..9c2025e1a2 100644--- a/rust/crates/location/src/urban.rs+++ b/rust/crates/location/src/urban.rs@@ -2,7 +2,7 @@ use std::collections::BTreeMap;use std::fs;use std::path::Path;-use crate::binary::{array, f32_at, range, u16_at, u32_at};+use crate::binary::{ByteReader, f32_at, range, u16_at, u32_at};use crate::city::SearchPattern;use crate::geometry::{Point, Reader as GeometryReader};use crate::{City, CityMatch, Coordinate, CountryCode, Error};@@ -129,8 +129,7 @@ impl UrbanCenterIndex {let end = self.cell_offset(dense + 1);for reference in start..end {let feature = usize::from(- u16_at(&self.bytes, self.layout.references + reference * 2)- .expect("validated reference"),+ ByteReader::at(&self.bytes, self.layout.references + reference * 2).u16(),);if matching.is_some_and(|matching| !matching[feature]) {continue;@@ -199,19 +198,19 @@ impl UrbanCenterIndex {}fn feature(&self, index: usize) -> Feature {- let offset = self.layout.records + index * RECORD_LEN;+ let mut reader = ByteReader::at(&self.bytes, self.layout.records + index * RECORD_LEN);Feature {- minimum_longitude: f32_at(&self.bytes, offset).expect("validated feature"),- minimum_latitude: f32_at(&self.bytes, offset + 4).expect("validated feature"),- maximum_longitude: f32_at(&self.bytes, offset + 8).expect("validated feature"),- maximum_latitude: f32_at(&self.bytes, offset + 12).expect("validated feature"),- latitude: f32_at(&self.bytes, offset + 16).expect("validated feature"),- longitude: f32_at(&self.bytes, offset + 20).expect("validated feature"),- name: u32_at(&self.bytes, offset + 24).expect("validated feature") as usize,- geometry_start: u32_at(&self.bytes, offset + 28).expect("validated feature") as usize,- geometry_end: u32_at(&self.bytes, offset + 32).expect("validated feature") as usize,- source_id: u32_at(&self.bytes, offset + 36).expect("validated feature"),- country: usize::from(self.bytes[offset + 40]),+ minimum_longitude: reader.f32(),+ minimum_latitude: reader.f32(),+ maximum_longitude: reader.f32(),+ maximum_latitude: reader.f32(),+ latitude: reader.f32(),+ longitude: reader.f32(),+ name: reader.u32() as usize,+ geometry_start: reader.u32() as usize,+ geometry_end: reader.u32() as usize,+ source_id: reader.u32(),+ country: usize::from(reader.byte()),}}@@ -225,10 +224,9 @@ impl UrbanCenterIndex {}fn country(&self, index: usize) -> &str {- let start = u32_at(&self.bytes, self.layout.country_offsets + index * 4)- .expect("validated country offset") as usize;- let end = u32_at(&self.bytes, self.layout.country_offsets + (index + 1) * 4)- .expect("validated country offset") as usize;+ let mut reader = ByteReader::at(&self.bytes, self.layout.country_offsets + index * 4);+ let start = reader.u32() as usize;+ let end = reader.u32() as usize;std::str::from_utf8(&self.bytes[self.layout.countries + start..self.layout.countries + end]).expect("validated country")}@@ -241,18 +239,17 @@ impl UrbanCenterIndex {fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {let block_index = cell_index / BLOCK_SIZE;let bit_index = cell_index % BLOCK_SIZE;- let offset = self.layout.blocks + block_index * BLOCK_LEN;- let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));+ let mut reader = ByteReader::at(&self.bytes, self.layout.blocks + block_index * BLOCK_LEN);+ let mask = reader.u64();let bit = 1_u64 << bit_index;(mask & bit != 0).then(|| {- let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;+ let first = reader.u32() as usize;first + (mask & bit.wrapping_sub(1)).count_ones() as usize})}fn cell_offset(&self, dense_index: usize) -> usize {- u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)- .expect("validated cell offset") as usize+ ByteReader::at(&self.bytes, self.layout.cell_offsets + dense_index * 4).u32() as usize}}@@ -458,10 +455,10 @@ fn validate_geometry(bytes: &[u8]) -> crate::Result<()> {fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Result<()> {let mut expected_dense = 0;+ let mut reader = ByteReader::at(bytes, layout.blocks);for block in 0..layout.block_count {- let offset = layout.blocks + block * BLOCK_LEN;- let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));- let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;+ let mask = reader.u64();+ let first = reader.u32() as usize;if first != expected_dense {return Err(invalid("invalid block dense offset"));}@@ -477,9 +474,9 @@ fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Resureturn Err(invalid("wrong nonempty cell count"));}let mut previous = 0;- for index in 0..=layout.nonempty_count {- let offset = u32_at(bytes, layout.cell_offsets + index * 4)- .expect("validated cell offset range") as usize;+ let mut reader = ByteReader::at(bytes, layout.cell_offsets);+ for _ in 0..=layout.nonempty_count {+ let offset = reader.u32() as usize;if offset < previous || offset > layout.reference_count {return Err(invalid("invalid cell offset"));}@@ -488,10 +485,9 @@ fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Resuif previous != layout.reference_count {return Err(invalid("last cell offset is not terminal"));}- for index in 0..layout.reference_count {- let feature = usize::from(- u16_at(bytes, layout.references + index * 2).expect("validated reference range"),- );+ let mut reader = ByteReader::at(bytes, layout.references);+ for _ in 0..layout.reference_count {+ let feature = usize::from(reader.u16());if feature >= layout.feature_count {return Err(invalid("cell references unknown feature"));}