π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Add offline country lookup
Neeraj Gupta86bda9ceebe4
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 1e6f782d65..aec02142c2 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -880,6 +880,12 @@ dependencies = ["tinyvec",]+[[package]]+name = "bufrw"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "394c22b416e3981206df472c3bcbbf9ae130883619aefc2199e6087517af1bba"+[[package]]name = "build-target"version = "0.4.0"@@ -1884,6 +1890,17 @@ dependencies = ["parking_lot_core",]+[[package]]+name = "dbase"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "abde818b4b82df61e01348c2b4610c221bcecb97cb7f567da086edf295d8d126"+dependencies = [+ "bufrw",+ "byteorder",+ "time",+]+[[package]]name = "dbus"version = "0.9.11"@@ -2641,8 +2658,10 @@ name = "ente-location-dataset"version = "0.0.0"dependencies = ["ente-location",+ "geo","reqwest 0.12.28","sha2 0.11.0",+ "shapefile","thiserror 2.0.18","zip 8.6.0",]@@ -3208,6 +3227,12 @@ dependencies = ["zlib-rs",]+[[package]]+name = "float_next_after"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "37007738a80ea34f969af54a3390dd72cacdef654974cfd449c9f6f72dbaac10"+[[package]]name = "flutter_rust_bridge"version = "2.12.0"@@ -3604,6 +3629,46 @@ dependencies = ["zeroize",]+[[package]]+name = "geo"+version = "0.33.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "30eb1fdc57c1e5cfd11826fe0caec4b9dc7901f3758263bb506228d88c8d9e9a"+dependencies = [+ "float_next_after",+ "geo-types",+ "geographiclib-rs",+ "i_overlay",+ "log",+ "num-traits",+ "rand 0.10.1",+ "rand_pcg",+ "robust",+ "rstar",+ "sif-itree",+]++[[package]]+name = "geo-types"+version = "0.7.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5"+dependencies = [+ "approx",+ "num-traits",+ "rstar",+ "serde",+]++[[package]]+name = "geographiclib-rs"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7"+dependencies = [+ "libm",+]+[[package]]name = "getrandom"version = "0.2.17"@@ -3862,6 +3927,15 @@ dependencies = ["zerocopy 0.8.48",]+[[package]]+name = "hash32"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606"+dependencies = [+ "byteorder",+]+[[package]]name = "hashbrown"version = "0.12.3"@@ -3900,6 +3974,16 @@ dependencies = ["hashbrown 0.16.1",]+[[package]]+name = "heapless"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad"+dependencies = [+ "hash32",+ "stable_deref_trait",+]+[[package]]name = "heck"version = "0.4.1"@@ -4167,6 +4251,48 @@ dependencies = ["windows-registry",]+[[package]]+name = "i_float"+version = "1.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "813145bb0ad5b60f55cbbf3c74cdceda1c0a9d253b35c4cc36ae0df7887cb78f"+dependencies = [+ "libm",+]++[[package]]+name = "i_key_sort"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d73d122b937fca067feb0ad74f62388920272b27c356d4df2d0cfdd59e044cf0"++[[package]]+name = "i_overlay"+version = "4.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8dd314b4668e2b3a12508f2e125558c82a6c0a8636fa5107a900f79ce414e450"+dependencies = [+ "i_float",+ "i_key_sort",+ "i_shape",+ "i_tree",+]++[[package]]+name = "i_shape"+version = "1.18.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bfa9eac533d7509a8ab87672b60ac610c17240f9ea4851d26227689fdfe349c8"+dependencies = [+ "i_float",+]++[[package]]+name = "i_tree"+version = "0.18.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4804bdc1dc124eb7e1aa9e144ecc04096bcf787a10a15fa44af682b51f0f6cce"+[[package]]name = "iana-time-zone"version = "0.1.65"@@ -7322,6 +7448,15 @@ dependencies = ["rand 0.8.6",]+[[package]]+name = "rand_pcg"+version = "0.10.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"+dependencies = [+ "rand_core 0.10.1",+]+[[package]]name = "rav1d"version = "1.1.0"@@ -7741,6 +7876,12 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "robust"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839"+[[package]]name = "rsqlite-vfs"version = "0.1.0"@@ -7751,6 +7892,17 @@ dependencies = ["thiserror 2.0.18",]+[[package]]+name = "rstar"+version = "0.12.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb"+dependencies = [+ "heapless",+ "num-traits",+ "smallvec",+]+[[package]]name = "rubato"version = "0.16.2"@@ -8478,6 +8630,17 @@ dependencies = ["sponge-cursor",]+[[package]]+name = "shapefile"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92fedc0c44dc53a02655cd8dccd2fdf3159c372c0f6eb59115e889f11325e4f0"+dependencies = [+ "byteorder",+ "dbase",+ "geo-types",+]+[[package]]name = "shell-words"version = "1.1.1"@@ -8490,6 +8653,12 @@ version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"+[[package]]+name = "sif-itree"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7f45b8998ced5134fb1d75732c77842a3e888f19c1ff98481822e8fbfbf930b"+[[package]]name = "signal-hook-registry"version = "1.4.8"diff --git a/rust/crates/location-dataset/Cargo.toml b/rust/crates/location-dataset/Cargo.tomlindex 89c5542fcc..85141b006a 100644--- a/rust/crates/location-dataset/Cargo.toml+++ b/rust/crates/location-dataset/Cargo.toml@@ -5,7 +5,9 @@ publish = false[dependencies]ente-location.workspace = true+geo = { version = "0.33.1", default-features = false }reqwest = { workspace = true, features = ["blocking", "rustls-tls-webpki-roots"] }sha2.workspace = true+shapefile = { version = "0.9.0", features = ["geo-types"] }thiserror.workspace = truezip = { version = "8.6.0", default-features = false, features = ["deflate-flate2-zlib-rs"] }diff --git a/rust/crates/location-dataset/README.md b/rust/crates/location-dataset/README.mdindex 217695fc11..9c81f92152 100644--- a/rust/crates/location-dataset/README.md+++ b/rust/crates/location-dataset/README.md@@ -1,14 +1,14 @@# ente-location-dataset-Host-side generator for the city asset consumed by `ente-location`.+Host-side generator for the assets consumed by `ente-location`.```shcargo run --release -p ente-location-dataset -- \build --output /tmp/ente-location```-The command downloads and caches its sources, writes `cities.bin`, and prints-its size and SHA-256 hash.+The command downloads and caches its sources, writes `cities.bin`,+`countries.bin`, and `disputes.bin`, and prints their sizes and SHA-256 hashes.## Source@@ -16,3 +16,6 @@ City data is derived from [GeoNames](https://www.geonames.org/) `cities5000`and `countryInfo`, licensed under[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Ente filters andconverts the data, which is provided without warranty.++Country boundaries are derived from [Natural Earth](https://www.naturalearthdata.com/),+which is in the public domain.diff --git a/rust/crates/location-dataset/src/catalog.rs b/rust/crates/location-dataset/src/catalog.rsnew file mode 100644index 0000000000..5086b12bd4--- /dev/null+++ b/rust/crates/location-dataset/src/catalog.rs@@ -0,0 +1,128 @@+use ente_location::TerritoryId;++pub(crate) struct DisputedAreaSource {+ pub source_id: &'static str,+ pub geometry_code: [u8; 2],+ pub territory: TerritoryId,+ pub additional_countries: &'static [[u8; 2]],+}++pub(crate) struct UkrainianRegionSource {+ pub iso_3166_2: &'static str,+ pub geometry_code: [u8; 2],+ pub territory: TerritoryId,+}++pub(crate) const WORLDVIEWS: &[([u8; 2], &str)] = &[+ (*b"AR", "ADM0_A3_AR"),+ (*b"BD", "ADM0_A3_BD"),+ (*b"BR", "ADM0_A3_BR"),+ (*b"CN", "ADM0_A3_CN"),+ (*b"DE", "ADM0_A3_DE"),+ (*b"EG", "ADM0_A3_EG"),+ (*b"ES", "ADM0_A3_ES"),+ (*b"FR", "ADM0_A3_FR"),+ (*b"GB", "ADM0_A3_GB"),+ (*b"GR", "ADM0_A3_GR"),+ (*b"IN", "ADM0_A3_IN"),+ (*b"ID", "ADM0_A3_ID"),+ (*b"IL", "ADM0_A3_IL"),+ (*b"IT", "ADM0_A3_IT"),+ (*b"JP", "ADM0_A3_JP"),+ (*b"KR", "ADM0_A3_KO"),+ (*b"MA", "ADM0_A3_MA"),+ (*b"NP", "ADM0_A3_NP"),+ (*b"NL", "ADM0_A3_NL"),+ (*b"PK", "ADM0_A3_PK"),+ (*b"PL", "ADM0_A3_PL"),+ (*b"PT", "ADM0_A3_PT"),+ (*b"PS", "ADM0_A3_PS"),+ (*b"RU", "ADM0_A3_RU"),+ (*b"SA", "ADM0_A3_SA"),+ (*b"SE", "ADM0_A3_SE"),+ (*b"TR", "ADM0_A3_TR"),+ (*b"TW", "ADM0_A3_TW"),+ (*b"UA", "ADM0_A3_UA"),+ (*b"US", "ADM0_A3_US"),+ (*b"VN", "ADM0_A3_VN"),+];++const NONE: &[[u8; 2]] = &[];+const PALESTINE: &[[u8; 2]] = &[*b"PS"];++pub(crate) const DISPUTED_AREAS: &[DisputedAreaSource] = &[+ area("B00", *b"AP", TerritoryId::ARUNACHAL_PRADESH, NONE),+ area("B01", *b"TV", TerritoryId::TIRPANI_VALLEYS, NONE),+ area("B02", *b"BH", TerritoryId::BARA_HOTII_VALLEYS, NONE),+ area("B03", *b"DM", TerritoryId::DEMCHOK, NONE),+ area("B04", *b"SV", TerritoryId::SAMDU_VALLEYS, NONE),+ area("B05", *b"KM", TerritoryId::KASHMIR, NONE),+ area("B06", *b"TK", TerritoryId::TRANS_KARAKORAM_TRACT, NONE),+ area("B07", *b"AC", TerritoryId::AKSAI_CHIN, NONE),+ area("B08", *b"GB", TerritoryId::GILGIT_BALTISTAN, NONE),+ area("B09", *b"AK", TerritoryId::AZAD_KASHMIR, NONE),+ area("B45", *b"SC", TerritoryId::SIACHEN_GLACIER, NONE),+ area("B53", *b"GZ", TerritoryId::GAZA_STRIP, NONE),+ area("B54", *b"WB", TerritoryId::WEST_BANK, NONE),+ area("B16", *b"GH", TerritoryId::GOLAN_HEIGHTS, NONE),+ area("B58", *b"SF", TerritoryId::SHEBAA_FARMS, NONE),+ area("B98", *b"EJ", TerritoryId::EAST_JERUSALEM, NONE),+ area("B78", *b"LF", TerritoryId::LATRUN_FORT, NONE),+ area("B79", *b"JN", TerritoryId::JERUSALEM_NO_MANS_LAND, NONE),+ area("B99", *b"MS", TerritoryId::MOUNT_SCOPUS, PALESTINE),+ area("B77", *b"TW", TerritoryId::TAIWAN, NONE),+ area("B89", *b"CR", TerritoryId::CRIMEA, NONE),+ area("B57", *b"KO", TerritoryId::KOSOVO, NONE),+ area(+ "B19",+ *b"WM",+ TerritoryId::WESTERN_SAHARA_MOROCCAN_AREA,+ NONE,+ ),+ area(+ "B28",+ *b"WS",+ TerritoryId::WESTERN_SAHARA_SELF_ADMINISTERED_AREA,+ NONE,+ ),+ area("B20", *b"NC", TerritoryId::NORTHERN_CYPRUS, NONE),+ area("B43", *b"CB", TerritoryId::CYPRUS_BUFFER_ZONE, NONE),+ area("B12", *b"FK", TerritoryId::FALKLAND_ISLANDS, NONE),+];++pub(crate) const UKRAINIAN_REGIONS: &[UkrainianRegionSource] = &[+ UkrainianRegionSource {+ iso_3166_2: "UA-14",+ geometry_code: *b"DO",+ territory: TerritoryId::DONETSK_REGION,+ },+ UkrainianRegionSource {+ iso_3166_2: "UA-09",+ geometry_code: *b"LU",+ territory: TerritoryId::LUHANSK_REGION,+ },+ UkrainianRegionSource {+ iso_3166_2: "UA-65",+ geometry_code: *b"KH",+ territory: TerritoryId::KHERSON_REGION,+ },+ UkrainianRegionSource {+ iso_3166_2: "UA-23",+ geometry_code: *b"ZP",+ territory: TerritoryId::ZAPORIZHZHIA_REGION,+ },+];++const fn area(+ source_id: &'static str,+ geometry_code: [u8; 2],+ territory: TerritoryId,+ additional_countries: &'static [[u8; 2]],+) -> DisputedAreaSource {+ DisputedAreaSource {+ source_id,+ geometry_code,+ territory,+ additional_countries,+ }+}diff --git a/rust/crates/location-dataset/src/country.rs b/rust/crates/location-dataset/src/country.rsnew file mode 100644index 0000000000..321293c52e--- /dev/null+++ b/rust/crates/location-dataset/src/country.rs@@ -0,0 +1,383 @@+use std::collections::{BTreeMap, BTreeSet};+use std::path::Path;++use geo::{BooleanOps, BoundingRect, Contains, Coord, LineString, MultiPolygon, Rect};+use shapefile::dbase::{FieldValue, Record};+use shapefile::{Reader, Shape};++use crate::{Result, invalid};++const COLUMNS: u16 = 360;+const ROWS: u16 = 180;+const HEADER_LEN: usize = 48;+const BLOCK_SIZE: usize = 64;+const BLOCK_LEN: usize = 12;++pub(crate) struct Area {+ pub code: [u8; 2],+ pub geometry: MultiPolygon<f64>,+}++#[derive(Default)]+struct Cell {+ containing: BTreeSet<[u8; 2]>,+ intersecting: BTreeMap<[u8; 2], CellAreas>,+}++#[derive(Default)]+struct CellAreas {+ outer: Vec<Vec<[u16; 2]>>,+ inner: Vec<Vec<[u16; 2]>>,+}++pub(crate) fn build_countries(path: &Path) -> Result<Vec<u8>> {+ let mut reader = Reader::from_path(path)?;+ let mut areas = Vec::new();+ for shape_record in reader.iter_shapes_and_records() {+ let (shape, record) = shape_record?;+ let Some(code) = natural_earth_country_code(&record) else {+ continue;+ };+ if &code == b"AQ" {+ continue;+ }+ areas.push(Area {+ code,+ geometry: polygon_geometry(shape, "country")?,+ });+ }+ encode(&areas)+}++pub(crate) fn encode(areas: &[Area]) -> Result<Vec<u8>> {+ if areas.is_empty() {+ return Err(invalid("country geometry is empty"));+ }+ let mut cells: Vec<Cell> = (0..usize::from(COLUMNS) * usize::from(ROWS))+ .map(|_| Cell::default())+ .collect();+ for area in areas {+ validate_code(area.code)?;+ add_area(&mut cells, area)?;+ }+ encode_cells(&cells)+}++fn add_area(cells: &mut [Cell], area: &Area) -> Result<()> {+ for polygon in &area.geometry {+ add_polygon(cells, area.code, polygon)?;+ }+ Ok(())+}++fn add_polygon(cells: &mut [Cell], code: [u8; 2], polygon: &geo::Polygon<f64>) -> Result<()> {+ let bounds = polygon+ .bounding_rect()+ .ok_or_else(|| invalid("country polygon has no bounds"))?;+ let minimum_column = longitude_column(bounds.min().x);+ let maximum_column = longitude_column(bounds.max().x);+ let northern_row = latitude_row(bounds.max().y);+ let southern_row = latitude_row(bounds.min().y);++ for row in northern_row..=southern_row {+ for column in minimum_column..=maximum_column {+ let bounds = cell_rect(column, row);+ let cell = &mut cells[row * usize::from(COLUMNS) + column];+ if polygon.contains(&bounds) {+ cell.containing.insert(code);+ cell.intersecting.remove(&code);+ continue;+ }+ if cell.containing.contains(&code) {+ continue;+ }+ let intersection = polygon.intersection(&bounds.to_polygon());+ if intersection.0.is_empty() {+ continue;+ }+ let areas = cell.intersecting.entry(code).or_default();+ for polygon in intersection {+ if let Some(ring) = encode_ring(polygon.exterior(), bounds) {+ areas.outer.push(ring);+ }+ for ring in polygon.interiors() {+ if let Some(ring) = encode_ring(ring, bounds) {+ areas.inner.push(ring);+ }+ }+ }+ if areas.outer.is_empty() {+ cell.intersecting.remove(&code);+ }+ }+ }+ Ok(())+}++fn encode_cells(cells: &[Cell]) -> Result<Vec<u8>> {+ let countries: Vec<[u8; 2]> = cells+ .iter()+ .flat_map(|cell| {+ cell.containing+ .iter()+ .copied()+ .chain(cell.intersecting.keys().copied())+ })+ .collect::<BTreeSet<_>>()+ .into_iter()+ .collect();+ if countries.is_empty() || countries.len() > 256 {+ return Err(invalid("country table does not fit one-byte indices"));+ }+ let country_indices: BTreeMap<[u8; 2], u8> = countries+ .iter()+ .copied()+ .enumerate()+ .map(|(index, code)| (code, index as u8))+ .collect();++ let encoded_cells: Vec<Option<Vec<u8>>> = cells+ .iter()+ .map(|cell| encode_cell(cell, &country_indices))+ .collect::<Result<_>>()?;+ let cell_count = cells.len();+ let block_count = cell_count.div_ceil(BLOCK_SIZE);+ let nonempty_count = encoded_cells.iter().filter(|cell| cell.is_some()).count();+ let countries_offset = HEADER_LEN;+ let blocks_offset = countries_offset + countries.len() * 2;+ let cell_offsets_offset = blocks_offset + block_count * BLOCK_LEN;+ let cells_offset = cell_offsets_offset + (nonempty_count + 1) * 4;+ let data_length: usize = encoded_cells.iter().flatten().map(Vec::len).sum();+ let file_length = cells_offset+ .checked_add(data_length)+ .ok_or_else(|| invalid("country index size overflow"))?;+ let mut output = Vec::with_capacity(file_length);++ output.extend_from_slice(b"CTRY");+ push_u16(&mut output, 1);+ push_u16(&mut output, HEADER_LEN as u16);+ push_u16(&mut output, COLUMNS);+ push_u16(&mut output, ROWS);+ push_u16(+ &mut output,+ u16::try_from(countries.len()).map_err(|_| invalid("too many countries"))?,+ );+ push_u16(&mut output, BLOCK_SIZE as u16);+ for value in [+ cell_count,+ block_count,+ nonempty_count,+ countries_offset,+ blocks_offset,+ cell_offsets_offset,+ cells_offset,+ file_length,+ ] {+ push_len(&mut output, value)?;+ }+ for code in &countries {+ output.extend_from_slice(code);+ }++ let mut dense_start = 0;+ for block in 0..block_count {+ let mut mask = 0_u64;+ for bit in 0..BLOCK_SIZE {+ let index = block * BLOCK_SIZE + bit;+ if index < cell_count && encoded_cells[index].is_some() {+ mask |= 1_u64 << bit;+ }+ }+ output.extend_from_slice(&mask.to_le_bytes());+ push_len(&mut output, dense_start)?;+ dense_start += mask.count_ones() as usize;+ }++ let mut cell_offset = 0;+ push_len(&mut output, cell_offset)?;+ for cell in encoded_cells.iter().flatten() {+ cell_offset += cell.len();+ push_len(&mut output, cell_offset)?;+ }+ for cell in encoded_cells.into_iter().flatten() {+ output.extend_from_slice(&cell);+ }+ debug_assert_eq!(output.len(), file_length);+ Ok(output)+}++fn encode_cell(cell: &Cell, countries: &BTreeMap<[u8; 2], u8>) -> Result<Option<Vec<u8>>> {+ if cell.containing.is_empty() && cell.intersecting.is_empty() {+ return Ok(None);+ }+ let mut output = Vec::new();+ output.push(+ u8::try_from(cell.containing.len())+ .map_err(|_| invalid("too many containing countries in one cell"))?,+ );+ output.extend(cell.containing.iter().map(|code| countries[code]));+ output.push(+ u8::try_from(cell.intersecting.len())+ .map_err(|_| invalid("too many intersecting countries in one cell"))?,+ );+ for (code, areas) in &cell.intersecting {+ output.push(countries[code]);+ encode_rings(&mut output, &areas.outer)?;+ encode_rings(&mut output, &areas.inner)?;+ }+ Ok(Some(output))+}++fn encode_rings(output: &mut Vec<u8>, rings: &[Vec<[u16; 2]>]) -> Result<()> {+ output.push(+ u8::try_from(rings.len()).map_err(|_| invalid("too many clipped rings in one cell"))?,+ );+ for ring in rings {+ push_u16(+ output,+ u16::try_from(ring.len()).map_err(|_| invalid("clipped ring is too long"))?,+ );+ for &[x, y] in ring {+ push_u16(output, x);+ push_u16(output, y);+ }+ }+ Ok(())+}++fn encode_ring(ring: &LineString<f64>, bounds: Rect<f64>) -> Option<Vec<[u16; 2]>> {+ let width = bounds.width();+ let height = bounds.height();+ let mut points: Vec<[u16; 2]> = ring+ .0+ .iter()+ .map(|coordinate| {+ [+ quantize(coordinate.x, bounds.min().x, width),+ quantize(coordinate.y, bounds.min().y, height),+ ]+ })+ .collect();+ points.dedup();+ while points.len() > 3 && points.first() == points.last() {+ points.pop();+ }+ loop {+ if points.len() <= 3 {+ break;+ }+ let removable: Vec<bool> = (0..points.len())+ .map(|index| {+ lies_on_segment(+ points[(index + points.len() - 1) % points.len()],+ points[index],+ points[(index + 1) % points.len()],+ )+ })+ .collect();+ let retained = removable.iter().filter(|&&remove| !remove).count();+ if retained < 3 || retained == points.len() {+ break;+ }+ points = points+ .into_iter()+ .zip(removable)+ .filter_map(|(point, remove)| (!remove).then_some(point))+ .collect();+ }+ (points.len() >= 3).then_some(points)+}++fn cell_rect(column: usize, row: usize) -> Rect<f64> {+ let minimum_longitude = -180.0 + column as f64 * 360.0 / f64::from(COLUMNS);+ let maximum_latitude = 90.0 - row as f64 * 180.0 / f64::from(ROWS);+ Rect::new(+ Coord {+ x: minimum_longitude,+ y: maximum_latitude - 180.0 / f64::from(ROWS),+ },+ Coord {+ x: minimum_longitude + 360.0 / f64::from(COLUMNS),+ y: maximum_latitude,+ },+ )+}++fn longitude_column(longitude: f64) -> usize {+ (((longitude + 180.0) / 360.0) * f64::from(COLUMNS))+ .floor()+ .clamp(0.0, f64::from(COLUMNS - 1)) as usize+}++fn latitude_row(latitude: f64) -> usize {+ (((90.0 - latitude) / 180.0) * f64::from(ROWS))+ .floor()+ .clamp(0.0, f64::from(ROWS - 1)) as usize+}++fn quantize(value: f64, minimum: f64, width: f64) -> u16 {+ (((value - minimum) * f64::from(u16::MAX) / width).clamp(0.0, f64::from(u16::MAX))) as u16+}++fn lies_on_segment(start: [u16; 2], point: [u16; 2], end: [u16; 2]) -> bool {+ let cross = (i64::from(point[0]) - i64::from(start[0]))+ * (i64::from(end[1]) - i64::from(start[1]))+ - (i64::from(point[1]) - i64::from(start[1])) * (i64::from(end[0]) - i64::from(start[0]));+ cross == 0+ && point[0] >= start[0].min(end[0])+ && point[0] <= start[0].max(end[0])+ && point[1] >= start[1].min(end[1])+ && point[1] <= start[1].max(end[1])+}++pub(crate) fn natural_earth_country_code(record: &Record) -> Option<[u8; 2]> {+ ["ISO_A2_EH", "ISO_A2"]+ .into_iter()+ .filter_map(|field| character_field(record, field))+ .find_map(|value| {+ let code: [u8; 2] = value.as_bytes().try_into().ok()?;+ code.iter().all(u8::is_ascii_uppercase).then_some(code)+ })+}++pub(crate) fn character_field<'a>(record: &'a Record, field: &str) -> Option<&'a str> {+ match record.get(field) {+ Some(FieldValue::Character(Some(value))) => Some(value.trim()),+ _ => None,+ }+}++pub(crate) fn polygon_geometry(shape: Shape, source: &str) -> Result<MultiPolygon<f64>> {+ let Shape::Polygon(polygon) = shape else {+ return Err(invalid(format!("{source} contains a non-polygon shape")));+ };+ let geometry: MultiPolygon<f64> = polygon+ .try_into()+ .map_err(|error| invalid(format!("cannot convert {source} polygon: {error}")))?;+ if geometry.0.is_empty() {+ Err(invalid(format!("{source} has empty geometry")))+ } else {+ Ok(geometry)+ }+}++fn validate_code(code: [u8; 2]) -> Result<()> {+ if code.iter().all(u8::is_ascii_uppercase) {+ Ok(())+ } else {+ Err(invalid("geometry has an invalid country code"))+ }+}++fn push_u16(output: &mut Vec<u8>, value: u16) {+ output.extend_from_slice(&value.to_le_bytes());+}++fn push_len(output: &mut Vec<u8>, value: usize) -> Result<()> {+ output.extend_from_slice(+ &u32::try_from(value)+ .map_err(|_| invalid("country index exceeds 4 GiB"))?+ .to_le_bytes(),+ );+ Ok(())+}diff --git a/rust/crates/location-dataset/src/dispute.rs b/rust/crates/location-dataset/src/dispute.rsnew file mode 100644index 0000000000..decd885242--- /dev/null+++ b/rust/crates/location-dataset/src/dispute.rs@@ -0,0 +1,447 @@+use std::collections::{BTreeMap, BTreeSet};+use std::path::Path;++use ente_location::TerritoryId;+use shapefile::Reader;+use shapefile::dbase::Record;++use crate::catalog::{DISPUTED_AREAS, UKRAINIAN_REGIONS, WORLDVIEWS};+use crate::country::{+ Area, character_field, encode as encode_geometry, natural_earth_country_code, polygon_geometry,+};+use crate::{Result, invalid};++const HEADER_LEN: usize = 64;+const TERRITORY_LEN: usize = 12;+const MISSING_COUNTRY: u8 = u8::MAX;++#[derive(Clone)]+struct SourceDispute {+ territory: TerritoryId,+ geometry_code: [u8; 2],+ name: String,+ source_default: Option<[u8; 2]>,+ possible_countries: Vec<[u8; 2]>,+ worldview_assignments: Vec<Option<[u8; 2]>>,+}++struct EncodedRecord {+ territory: TerritoryId,+ geometry_code: [u8; 2],+ default_country: u8,+ candidate_start: u16,+ candidate_count: u8,+ name_start: u16,+ name_length: u8,+}++pub(crate) fn build(+ countries_path: &Path,+ disputes_path: &Path,+ admin1_path: &Path,+) -> Result<Vec<u8>> {+ let country_codes = read_country_codes(countries_path)?;+ let (mut metadata, mut areas) = read_disputed_areas(disputes_path, &country_codes)?;+ let (ukraine_metadata, ukraine_areas) = read_ukrainian_regions(admin1_path)?;+ metadata.extend(ukraine_metadata);+ areas.extend(ukraine_areas);+ if metadata.len() != DISPUTED_AREAS.len() + UKRAINIAN_REGIONS.len() {+ return Err(invalid("Priority-1 dispute catalog is incomplete"));+ }+ let geometry = encode_geometry(&areas)?;+ encode(&geometry, &metadata)+}++fn read_country_codes(path: &Path) -> Result<BTreeMap<String, [u8; 2]>> {+ let mut reader = Reader::from_path(path)?;+ let mut codes = BTreeMap::new();+ for shape_record in reader.iter_shapes_and_records() {+ let (_, record) = shape_record?;+ let Some(code) = natural_earth_country_code(&record) else {+ continue;+ };+ for field in [+ "ADM0_A3",+ "SOV_A3",+ "ISO_A3_EH",+ "ISO_A3",+ "GU_A3",+ "SU_A3",+ "ADM0_TLC",+ ] {+ let Some(value) = character_field(&record, field) else {+ continue;+ };+ if value.len() == 3 && value.bytes().all(|byte| byte.is_ascii_uppercase()) {+ codes.entry(value.to_owned()).or_insert(code);+ }+ }+ }+ Ok(codes)+}++fn read_disputed_areas(+ path: &Path,+ country_codes: &BTreeMap<String, [u8; 2]>,+) -> Result<(Vec<SourceDispute>, Vec<Area>)> {+ let selections: BTreeMap<&str, _> = DISPUTED_AREAS+ .iter()+ .map(|source| (source.source_id, source))+ .collect();+ let mut seen = BTreeSet::new();+ let mut metadata = Vec::new();+ let mut areas = Vec::new();+ let mut reader = Reader::from_path(path)?;+ for shape_record in reader.iter_shapes_and_records() {+ let (shape, record) = shape_record?;+ let Some(source_id) = character_field(&record, "BRK_A3") else {+ continue;+ };+ let Some(selection) = selections.get(source_id) else {+ continue;+ };+ if !seen.insert(source_id.to_owned()) {+ return Err(invalid(format!(+ "duplicate disputed-area source {source_id}"+ )));+ }+ let name = required_field(&record, "NAME_EN", source_id)?.to_owned();+ let geometry = polygon_geometry(shape, source_id)?;+ metadata.push(SourceDispute {+ territory: selection.territory,+ geometry_code: selection.geometry_code,+ name,+ source_default: assigned_country(&record, "ADM0_A3", country_codes),+ possible_countries: selection.additional_countries.to_vec(),+ worldview_assignments: WORLDVIEWS+ .iter()+ .map(|&(_, field)| assigned_country(&record, field, country_codes))+ .collect(),+ });+ areas.push(Area {+ code: selection.geometry_code,+ geometry,+ });+ }+ let missing: Vec<&str> = selections+ .keys()+ .copied()+ .filter(|source| !seen.contains(*source))+ .collect();+ if !missing.is_empty() {+ return Err(invalid(format!(+ "disputed-area source is missing {missing:?}"+ )));+ }+ Ok((metadata, areas))+}++fn read_ukrainian_regions(path: &Path) -> Result<(Vec<SourceDispute>, Vec<Area>)> {+ let selections: BTreeMap<&str, _> = UKRAINIAN_REGIONS+ .iter()+ .map(|source| (source.iso_3166_2, source))+ .collect();+ let mut seen = BTreeSet::new();+ let mut metadata = Vec::new();+ let mut areas = Vec::new();+ let mut reader = Reader::from_path(path)?;+ for shape_record in reader.iter_shapes_and_records() {+ let (shape, record) = shape_record?;+ let Some(iso_code) = character_field(&record, "iso_3166_2") else {+ continue;+ };+ let Some(selection) = selections.get(iso_code) else {+ continue;+ };+ if !seen.insert(iso_code.to_owned()) {+ return Err(invalid(format!("duplicate Admin-1 source {iso_code}")));+ }+ let name = required_field(&record, "name_en", iso_code)?;+ let geometry = polygon_geometry(shape, iso_code)?;+ metadata.push(SourceDispute {+ territory: selection.territory,+ geometry_code: selection.geometry_code,+ name: format!("{name} Region"),+ source_default: Some(*b"UA"),+ possible_countries: vec![*b"RU", *b"UA"],+ worldview_assignments: WORLDVIEWS+ .iter()+ .map(|&(viewpoint, _)| Some(if viewpoint == *b"RU" { *b"RU" } else { *b"UA" }))+ .collect(),+ });+ areas.push(Area {+ code: selection.geometry_code,+ geometry,+ });+ }+ let missing: Vec<&str> = selections+ .keys()+ .copied()+ .filter(|source| !seen.contains(*source))+ .collect();+ if !missing.is_empty() {+ return Err(invalid(format!("Admin-1 source is missing {missing:?}")));+ }+ Ok((metadata, areas))+}++fn encode(geometry: &[u8], source: &[SourceDispute]) -> Result<Vec<u8>> {+ let mut disputes = source.to_vec();+ disputes.sort_unstable_by_key(|dispute| dispute.territory);+ validate_source(&disputes)?;+ let countries = collect_countries(&disputes)?;+ let country_indices: BTreeMap<[u8; 2], u8> = countries+ .iter()+ .copied()+ .enumerate()+ .map(|(index, code)| (code, index as u8))+ .collect();++ let mut candidates = Vec::new();+ let mut assignments = Vec::with_capacity(disputes.len() * WORLDVIEWS.len());+ let mut strings = Vec::new();+ let mut records = Vec::with_capacity(disputes.len());+ for dispute in &disputes {+ let candidate_start = u16_length(candidates.len(), "too many dispute candidates")?;+ let possible = complete_candidates(dispute);+ let candidate_count = u8::try_from(possible.len())+ .map_err(|_| invalid("too many candidates for one territory"))?;+ candidates.extend(possible.into_iter().map(|code| country_indices[&code]));++ let name_start = u16_length(strings.len(), "dispute strings exceed 64 KiB")?;+ let name_length = u8::try_from(dispute.name.len())+ .map_err(|_| invalid("territory name exceeds 255 bytes"))?;+ strings.extend_from_slice(dispute.name.as_bytes());++ records.push(EncodedRecord {+ territory: dispute.territory,+ geometry_code: dispute.geometry_code,+ default_country: encode_optional(dispute.source_default, &country_indices),+ candidate_start,+ candidate_count,+ name_start,+ name_length,+ });+ assignments.extend(+ dispute+ .worldview_assignments+ .iter()+ .copied()+ .map(|country| encode_optional(country, &country_indices)),+ );+ }+ u16_length(strings.len(), "dispute strings exceed 64 KiB")?;++ let countries_offset = HEADER_LEN;+ let worldviews_offset = countries_offset + countries.len() * 2;+ let territories_offset = worldviews_offset + WORLDVIEWS.len();+ let candidates_offset = territories_offset + records.len() * TERRITORY_LEN;+ let assignments_offset = candidates_offset + candidates.len();+ let strings_offset = assignments_offset + assignments.len();+ let geometry_offset = strings_offset + strings.len();+ let file_length = geometry_offset+ .checked_add(geometry.len())+ .ok_or_else(|| invalid("dispute index size overflow"))?;+ let mut output = Vec::with_capacity(file_length);++ output.extend_from_slice(b"DSPT");+ push_u16(&mut output, 1);+ push_u16(&mut output, HEADER_LEN as u16);+ push_u16(+ &mut output,+ u16_length(WORLDVIEWS.len(), "too many worldviews")?,+ );+ push_u16(+ &mut output,+ u16_length(records.len(), "too many territories")?,+ );+ push_u16(+ &mut output,+ u16_length(countries.len(), "too many countries")?,+ );+ push_u16(&mut output, TERRITORY_LEN as u16);+ for value in [+ countries_offset,+ worldviews_offset,+ territories_offset,+ candidates_offset,+ assignments_offset,+ strings_offset,+ geometry_offset,+ file_length,+ ] {+ push_length(&mut output, value)?;+ }+ output.resize(HEADER_LEN, 0);+ for country in &countries {+ output.extend_from_slice(country);+ }+ for &(worldview, _) in WORLDVIEWS {+ output.push(country_indices[&worldview]);+ }+ for record in records {+ push_u16(&mut output, record.territory.get());+ output.extend_from_slice(&record.geometry_code);+ output.push(record.default_country);+ output.push(record.candidate_count);+ push_u16(&mut output, record.candidate_start);+ push_u16(&mut output, record.name_start);+ output.push(record.name_length);+ output.push(0);+ }+ output.extend_from_slice(&candidates);+ output.extend_from_slice(&assignments);+ output.extend_from_slice(&strings);+ output.extend_from_slice(geometry);+ debug_assert_eq!(output.len(), file_length);+ Ok(output)+}++fn validate_source(disputes: &[SourceDispute]) -> Result<()> {+ let mut ids = BTreeSet::new();+ let mut geometry_codes = BTreeSet::new();+ for dispute in disputes {+ if !ids.insert(dispute.territory)+ || !geometry_codes.insert(dispute.geometry_code)+ || dispute.name.is_empty()+ || dispute.worldview_assignments.len() != WORLDVIEWS.len()+ {+ return Err(invalid("invalid or duplicate dispute metadata"));+ }+ }+ Ok(())+}++fn collect_countries(disputes: &[SourceDispute]) -> Result<Vec<[u8; 2]>> {+ let mut countries: BTreeSet<[u8; 2]> = WORLDVIEWS.iter().map(|&(code, _)| code).collect();+ for dispute in disputes {+ countries.extend(complete_candidates(dispute));+ }+ if countries.len() >= usize::from(MISSING_COUNTRY) {+ return Err(invalid("dispute country table exceeds one-byte indices"));+ }+ if countries+ .iter()+ .flatten()+ .any(|byte| !byte.is_ascii_uppercase())+ {+ return Err(invalid("dispute catalog contains an invalid country code"));+ }+ Ok(countries.into_iter().collect())+}++fn complete_candidates(dispute: &SourceDispute) -> BTreeSet<[u8; 2]> {+ let mut countries: BTreeSet<[u8; 2]> = dispute.possible_countries.iter().copied().collect();+ countries.extend(dispute.source_default);+ countries.extend(dispute.worldview_assignments.iter().flatten().copied());+ countries+}++fn assigned_country(+ record: &Record,+ field: &str,+ country_codes: &BTreeMap<String, [u8; 2]>,+) -> Option<[u8; 2]> {+ character_field(record, field).and_then(|code| country_codes.get(code).copied())+}++fn required_field<'a>(record: &'a Record, field: &str, source: &str) -> Result<&'a str> {+ character_field(record, field)+ .ok_or_else(|| invalid(format!("source {source} is missing {field}")))+}++fn encode_optional(country: Option<[u8; 2]>, country_indices: &BTreeMap<[u8; 2], u8>) -> u8 {+ country.map_or(MISSING_COUNTRY, |code| country_indices[&code])+}++fn u16_length(value: usize, message: &'static str) -> Result<u16> {+ u16::try_from(value).map_err(|_| invalid(message))+}++fn push_u16(output: &mut Vec<u8>, value: u16) {+ output.extend_from_slice(&value.to_le_bytes());+}++fn push_length(output: &mut Vec<u8>, value: usize) -> Result<()> {+ output.extend_from_slice(+ &u32::try_from(value)+ .map_err(|_| invalid("dispute index exceeds 4 GiB"))?+ .to_le_bytes(),+ );+ Ok(())+}++#[cfg(test)]+pub(crate) fn test_bytes() -> Vec<u8> {+ use geo::{LineString, MultiPolygon, Polygon};++ let geometry = encode_geometry(&[Area {+ code: *b"AC",+ geometry: MultiPolygon(vec![Polygon::new(+ LineString::from(vec![+ (79.0, 35.0),+ (80.0, 35.0),+ (80.0, 36.0),+ (79.0, 36.0),+ (79.0, 35.0),+ ]),+ vec![],+ )]),+ }])+ .unwrap();+ encode(+ &geometry,+ &[SourceDispute {+ territory: TerritoryId::AKSAI_CHIN,+ geometry_code: *b"AC",+ name: "Aksai Chin".to_owned(),+ source_default: Some(*b"CN"),+ possible_countries: vec![*b"CN", *b"IN"],+ worldview_assignments: WORLDVIEWS+ .iter()+ .map(|&(region, _)| Some(if region == *b"IN" { *b"IN" } else { *b"CN" }))+ .collect(),+ }],+ )+ .unwrap()+}++#[cfg(test)]+mod tests {+ use ente_location::{Coordinate, CountryCode, CountryIndex, CountryView, TerritoryId};+ use geo::{LineString, MultiPolygon, Polygon};++ use crate::country::{Area, encode as encode_geometry};++ use super::test_bytes;++ #[test]+ fn builds_and_resolves_a_dispute_overlay() {+ let countries = encode_geometry(&[Area {+ code: *b"AA",+ geometry: MultiPolygon(vec![Polygon::new(+ LineString::from(vec![+ (79.0, 35.0),+ (80.0, 35.0),+ (80.0, 36.0),+ (79.0, 36.0),+ (79.0, 35.0),+ ]),+ vec![],+ )]),+ }])+ .unwrap();+ let index = CountryIndex::from_bytes(countries, test_bytes()).unwrap();+ let disputed = index.lookup(Coordinate::new(35.5, 79.5)).unwrap().disputes[0];+ let india = CountryCode::from_bytes(*b"IN").unwrap();++ assert_eq!(disputed.territory_id(), TerritoryId::AKSAI_CHIN);+ assert_eq!(disputed.name(), "Aksai Chin");+ assert_eq!(disputed.resolve(CountryView::Neutral), None);+ assert_eq!(+ disputed.resolve(CountryView::SourceDefault),+ Some(CountryCode::from_bytes(*b"CN").unwrap())+ );+ assert_eq!(disputed.resolve(CountryView::Region(india)), Some(india));+ }+}diff --git a/rust/crates/location-dataset/src/lib.rs b/rust/crates/location-dataset/src/lib.rsindex 2b660a2ffe..417e669103 100644--- a/rust/crates/location-dataset/src/lib.rs+++ b/rust/crates/location-dataset/src/lib.rs@@ -1,17 +1,22 @@use std::fmt::Write as _;use std::path::PathBuf;-use ente_location::CityIndex;+use ente_location::{CityIndex, CountryIndex};use sha2::{Digest, Sha256};use thiserror::Error;+mod catalog;mod city;+mod country;+mod dispute;mod download;mod sources;pub use sources::{RemoteSources, SourcePaths};pub const CITY_FILE: &str = "cities.bin";+pub const COUNTRY_FILE: &str = "countries.bin";+pub const DISPUTE_FILE: &str = "disputes.bin";pub type Result<T> = std::result::Result<T, Error>;@@ -23,6 +28,8 @@ pub enum Error {Http(#[from] reqwest::Error),#[error("ZIP archive is invalid: {0}")]Zip(#[from] zip::result::ZipError),+ #[error("shapefile is invalid: {0}")]+ Shapefile(#[from] shapefile::Error),#[error("invalid location data: {0}")]InvalidData(String),#[error(transparent)]@@ -36,23 +43,57 @@ pub struct BuildOptions {pub struct BuildOutput {pub city_count: usize,+ pub territory_count: usize,+ pub byte_length: usize,+ pub files: [BuildFile; 3],+}++pub struct BuildFile {+ pub name: &'static str,pub byte_length: usize,pub sha256: String,}pub fn build(options: &BuildOptions) -> Result<BuildOutput> {- let bytes = city::build(&options.sources.cities, &options.sources.country_info)?;- let city_count = CityIndex::from_bytes(bytes.as_slice())?.len();+ let cities = city::build(&options.sources.cities, &options.sources.country_info)?;+ let countries = country::build_countries(&options.sources.countries)?;+ let disputes = dispute::build(+ &options.sources.countries,+ &options.sources.disputes,+ &options.sources.admin1,+ )?;+ let city_count = CityIndex::from_bytes(cities.as_slice())?.len();+ CountryIndex::from_bytes(countries.as_slice(), disputes.as_slice())?;+ let files = [+ build_file(CITY_FILE, &cities),+ build_file(COUNTRY_FILE, &countries),+ build_file(DISPUTE_FILE, &disputes),+ ];let output = BuildOutput {city_count,- byte_length: bytes.len(),- sha256: hex(Sha256::digest(&bytes)),+ territory_count: catalog::DISPUTED_AREAS.len() + catalog::UKRAINIAN_REGIONS.len(),+ byte_length: files.iter().map(|file| file.byte_length).sum(),+ files,};std::fs::create_dir_all(&options.output)?;- std::fs::write(options.output.join(CITY_FILE), bytes)?;+ for (name, bytes) in [+ (CITY_FILE, cities.as_slice()),+ (COUNTRY_FILE, countries.as_slice()),+ (DISPUTE_FILE, disputes.as_slice()),+ ] {+ std::fs::write(options.output.join(name), bytes)?;+ }Ok(output)}+fn build_file(name: &'static str, bytes: &[u8]) -> BuildFile {+ BuildFile {+ name,+ byte_length: bytes.len(),+ sha256: hex(Sha256::digest(bytes)),+ }+}+fn invalid(message: impl Into<String>) -> Error {Error::InvalidData(message.into())}diff --git a/rust/crates/location-dataset/src/main.rs b/rust/crates/location-dataset/src/main.rsindex 4ed8dad10b..1981fbe424 100644--- a/rust/crates/location-dataset/src/main.rs+++ b/rust/crates/location-dataset/src/main.rs@@ -2,11 +2,12 @@ use std::env;use std::error::Error;use std::path::PathBuf;-use ente_location_dataset::{BuildOptions, CITY_FILE, RemoteSources, SourcePaths, build};+use ente_location_dataset::{BuildOptions, RemoteSources, SourcePaths, build};const USAGE: &str = concat!("usage: ente-location-dataset build --output <directory> [--cache <directory>] ",- "[--cities <file> --country-info <file>]"+ "[--cities <file> --country-info <file> ",+ "--countries <shp> --disputes <shp> --admin1 <shp>]");fn main() {@@ -22,21 +23,33 @@ fn run() -> Result<(), Box<dyn Error>> {cache,cities,country_info,+ countries,+ disputes,+ admin1,} = Arguments::parse()?;- let sources = match (cities, country_info) {- (None, None) => RemoteSources::new(cache).fetch()?,- (Some(cities), Some(country_info)) => SourcePaths {- cities,- country_info,- },- _ => return Err("local generation requires both source paths".into()),+ let sources = match (cities, country_info, countries, disputes, admin1) {+ (None, None, None, None, None) => RemoteSources::new(cache).fetch()?,+ (Some(cities), Some(country_info), Some(countries), Some(disputes), Some(admin1)) => {+ SourcePaths {+ cities,+ country_info,+ countries,+ disputes,+ admin1,+ }+ }+ _ => return Err("local generation requires all five source paths".into()),};let result = build(&BuildOptions { sources, output })?;println!("cities: {}", result.city_count);- println!(- "{CITY_FILE}: {} bytes, sha256 {}",- result.byte_length, result.sha256- );+ println!("Priority-1 territories: {}", result.territory_count);+ println!("bytes: {}", result.byte_length);+ for file in result.files {+ println!(+ "{}: {} bytes, sha256 {}",+ file.name, file.byte_length, file.sha256+ );+ }Ok(())}@@ -45,6 +58,9 @@ struct Arguments {cache: PathBuf,cities: Option<PathBuf>,country_info: Option<PathBuf>,+ countries: Option<PathBuf>,+ disputes: Option<PathBuf>,+ admin1: Option<PathBuf>,}impl Arguments {@@ -57,6 +73,9 @@ impl Arguments {let mut cache = None;let mut cities = None;let mut country_info = None;+ let mut countries = None;+ let mut disputes = None;+ let mut admin1 = None;while let Some(argument) = args.next() {let value = args.next().ok_or(USAGE)?;match argument.as_str() {@@ -64,6 +83,9 @@ impl Arguments {"--cache" => cache = Some(value.into()),"--cities" => cities = Some(value.into()),"--country-info" => country_info = Some(value.into()),+ "--countries" => countries = Some(value.into()),+ "--disputes" => disputes = Some(value.into()),+ "--admin1" => admin1 = Some(value.into()),_ => return Err(USAGE.into()),}}@@ -72,6 +94,9 @@ impl Arguments {cache: cache.unwrap_or_else(default_cache),cities,country_info,+ countries,+ disputes,+ admin1,})}}diff --git a/rust/crates/location-dataset/src/sources.rs b/rust/crates/location-dataset/src/sources.rsindex 75b0ef90da..828f1ace49 100644--- a/rust/crates/location-dataset/src/sources.rs+++ b/rust/crates/location-dataset/src/sources.rs@@ -7,11 +7,23 @@ use crate::download::{download, extract};const CITIES_URL: &str = "https://download.geonames.org/export/dump/cities5000.zip";const COUNTRY_INFO_URL: &str = "https://download.geonames.org/export/dump/countryInfo.txt";+const COUNTRIES: &str = "ne_10m_admin_0_countries";+const COUNTRIES_URL: &str =+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_countries.zip";+const DISPUTES: &str = "ne_10m_admin_0_disputed_areas";+const DISPUTES_URL: &str =+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_disputed_areas.zip";+const ADMIN1: &str = "ne_10m_admin_1_states_provinces";+const ADMIN1_URL: &str =+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_1_states_provinces.zip";#[derive(Clone, Debug)]pub struct SourcePaths {pub cities: PathBuf,pub country_info: PathBuf,+ pub countries: PathBuf,+ pub disputes: PathBuf,+ pub admin1: PathBuf,}#[derive(Clone, Debug)]@@ -37,9 +49,25 @@ impl RemoteSources {download(&client, CITIES_URL, &cities_archive)?;download(&client, COUNTRY_INFO_URL, &country_info)?;extract(&cities_archive, &cities, "cities5000.txt")?;+ let countries = self.fetch_shapefile(&client, COUNTRIES, COUNTRIES_URL)?;+ let disputes = self.fetch_shapefile(&client, DISPUTES, DISPUTES_URL)?;+ let admin1 = self.fetch_shapefile(&client, ADMIN1, ADMIN1_URL)?;Ok(SourcePaths {cities,country_info,+ countries,+ disputes,+ admin1,})}++ fn fetch_shapefile(&self, client: &Client, name: &str, url: &str) -> Result<PathBuf> {+ let archive = self.cache.join(format!("{name}.zip"));+ download(client, url, &archive)?;+ for extension in ["shp", "shx", "dbf"] {+ let file = format!("{name}.{extension}");+ extract(&archive, &self.cache.join(&file), &file)?;+ }+ Ok(self.cache.join(format!("{name}.shp")))+ }}diff --git a/rust/crates/location/src/country.rs b/rust/crates/location/src/country.rsnew file mode 100644index 0000000000..9f2492f1b1--- /dev/null+++ b/rust/crates/location/src/country.rs@@ -0,0 +1,431 @@+use crate::binary::{array, range, u16_at, u32_at};+use crate::{CountryCode, Error};++const MAGIC: &[u8; 4] = b"CTRY";+const VERSION: u16 = 1;+const HEADER_LEN: usize = 48;+const BLOCK_SIZE: usize = 64;+const BLOCK_LEN: usize = 12;+const SECTION: &str = "country index";++#[derive(Clone, Copy, Debug)]+struct Layout {+ columns: usize,+ rows: usize,+ country_count: usize,+ cell_count: usize,+ block_count: usize,+ nonempty_count: usize,+ countries: usize,+ blocks: usize,+ cell_offsets: usize,+ cells: usize,+}++#[derive(Debug)]+pub(crate) struct CountryGeometry {+ bytes: Box<[u8]>,+ layout: Layout,+}++impl CountryGeometry {+ pub(crate) fn from_bytes(bytes: impl Into<Box<[u8]>>) -> crate::Result<Self> {+ let bytes = bytes.into();+ let layout = validate(&bytes)?;+ Ok(Self { bytes, layout })+ }++ pub(crate) const fn country_count(&self) -> usize {+ self.layout.country_count+ }++ pub(crate) fn country_codes(&self) -> impl ExactSizeIterator<Item = CountryCode> + '_ {+ (0..self.layout.country_count).map(|index| self.country_code(index as u8))+ }++ pub(crate) const fn columns(&self) -> usize {+ self.layout.columns+ }++ pub(crate) const fn rows(&self) -> usize {+ self.layout.rows+ }++ pub(crate) fn prepare_cell(+ &self,+ latitude: f64,+ longitude: f64,+ ) -> crate::Result<PreparedCell> {+ if !latitude.is_finite()+ || !longitude.is_finite()+ || !(-90.0..=90.0).contains(&latitude)+ || !(-180.0..=180.0).contains(&longitude)+ {+ return Err(Error::InvalidCoordinate);+ }++ let longitude = normalize(longitude, -180.0, 360.0);+ let cell_x = (((longitude + 180.0) * self.layout.columns as f64 / 360.0).floor() as usize)+ .min(self.layout.columns - 1);+ let cell_y = ((self.layout.rows as f64 * (90.0 - latitude) / 180.0).ceil() as usize)+ .saturating_sub(1);+ Ok(PreparedCell {+ latitude,+ longitude,+ x: cell_x,+ y: cell_y,+ index: cell_y * self.layout.columns + cell_x,+ })+ }++ pub(crate) fn lookup_prepared(+ &self,+ location: PreparedCell,+ ) -> crate::Result<Vec<CountryCode>> {+ let Some(dense_index) = self.dense_cell_index(location.index) else {+ return Ok(Vec::new());+ };+ let point = Point {+ x: {+ let cell_longitude =+ -180.0 + 360.0 * location.x as f64 / self.layout.columns as f64;+ ((location.longitude - cell_longitude)+ * self.layout.columns as f64+ * f64::from(u16::MAX)+ / 360.0) as u16+ },+ y: {+ let cell_latitude =+ 90.0 - 180.0 * (location.y + 1) as f64 / self.layout.rows as f64;+ ((location.latitude - cell_latitude)+ * self.layout.rows as f64+ * f64::from(u16::MAX)+ / 180.0) as u16+ },+ };+ let start = self.cell_offset(dense_index);+ let end = self.cell_offset(dense_index + 1);+ self.lookup_cell(+ &self.bytes[self.layout.cells + start..self.layout.cells + end],+ point,+ )+ }++ fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {+ let block_index = cell_index / BLOCK_SIZE;+ let bit_index = cell_index % BLOCK_SIZE;+ let offset = self.layout.blocks + block_index * BLOCK_LEN;+ let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));+ let bit = 1_u64 << bit_index;+ if mask & bit == 0 {+ return None;+ }+ let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;+ Some(first + (mask & bit.wrapping_sub(1)).count_ones() as usize)+ }++ fn cell_offset(&self, dense_index: usize) -> usize {+ u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)+ .expect("validated cell offset") as usize+ }++ fn country_code(&self, index: u8) -> CountryCode {+ let offset = self.layout.countries + usize::from(index) * 2;+ CountryCode::from_validated([self.bytes[offset], self.bytes[offset + 1]])+ }++ fn lookup_cell(&self, bytes: &[u8], point: Point) -> crate::Result<Vec<CountryCode>> {+ let mut reader = Reader::new(bytes);+ let mut matches = Vec::new();+ for _ in 0..reader.byte()? {+ push_unique(&mut matches, self.country_code(reader.byte()?));+ }+ for _ in 0..reader.byte()? {+ let country = self.country_code(reader.byte()?);+ let mut winding = 0_i32;+ for _ in 0..reader.byte()? {+ if reader.ring_contains(point)? {+ winding += 1;+ }+ }+ for _ in 0..reader.byte()? {+ if reader.ring_contains(point)? {+ winding -= 1;+ }+ }+ if winding > 0 {+ push_unique(&mut matches, country);+ }+ }+ debug_assert_eq!(reader.position, bytes.len());+ Ok(matches)+ }+}++#[derive(Clone, Copy)]+pub(crate) struct PreparedCell {+ latitude: f64,+ longitude: f64,+ x: usize,+ y: usize,+ index: usize,+}++#[derive(Clone, Copy)]+struct Point {+ x: u16,+ y: u16,+}++struct Reader<'a> {+ bytes: &'a [u8],+ position: usize,+}++impl<'a> Reader<'a> {+ const fn new(bytes: &'a [u8]) -> Self {+ Self { bytes, position: 0 }+ }++ fn byte(&mut self) -> crate::Result<u8> {+ let value = *self.bytes.get(self.position).ok_or(invalid("truncated"))?;+ self.position += 1;+ Ok(value)+ }++ fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {+ let end = self+ .position+ .checked_add(count)+ .ok_or(invalid("truncated"))?;+ let bytes = self+ .bytes+ .get(self.position..end)+ .ok_or(invalid("truncated"))?;+ self.position = end;+ Ok(bytes)+ }++ fn ring_contains(&mut self, point: Point) -> crate::Result<bool> {+ let count = usize::from(u16::from_le_bytes(+ self.take(2)?.try_into().expect("two-byte slice"),+ ));+ if count == 0 {+ return Ok(false);+ }+ let ring = self.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;+ let mut winding = 0_i32;+ let mut previous = point_at(ring, count - 1);+ for index in 0..count {+ let current = point_at(ring, index);+ if previous.y <= point.y {+ if current.y > point.y && is_left(previous, current, point) > 0 {+ winding += 1;+ }+ } else if current.y <= point.y && is_left(previous, current, point) < 0 {+ winding -= 1;+ }+ previous = current;+ }+ Ok(winding != 0)+ }+}++fn validate(bytes: &[u8]) -> crate::Result<Layout> {+ if bytes.len() < HEADER_LEN {+ return Err(invalid("truncated"));+ }+ if &bytes[..4] != MAGIC {+ return Err(invalid("invalid magic"));+ }+ let version = u16_at(bytes, 4).ok_or(invalid("truncated"))?;+ if version != VERSION {+ return Err(Error::invalid(+ SECTION,+ format!("unsupported format version {version}"),+ ));+ }+ if usize::from(u16_at(bytes, 6).ok_or(invalid("truncated"))?) != HEADER_LEN+ || usize::from(u16_at(bytes, 14).ok_or(invalid("truncated"))?) != BLOCK_SIZE+ {+ return Err(invalid("unexpected header or block length"));+ }+ let layout = Layout {+ columns: usize::from(u16_at(bytes, 8).ok_or(invalid("truncated"))?),+ rows: usize::from(u16_at(bytes, 10).ok_or(invalid("truncated"))?),+ country_count: usize::from(u16_at(bytes, 12).ok_or(invalid("truncated"))?),+ cell_count: read_usize(bytes, 16)?,+ block_count: read_usize(bytes, 20)?,+ nonempty_count: read_usize(bytes, 24)?,+ countries: read_usize(bytes, 28)?,+ blocks: read_usize(bytes, 32)?,+ cell_offsets: read_usize(bytes, 36)?,+ cells: read_usize(bytes, 40)?,+ };+ let declared_length = read_usize(bytes, 44)?;+ validate_layout(bytes, layout, declared_length)?;+ Ok(layout)+}++fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {+ if layout.columns == 0+ || layout.rows == 0+ || layout.country_count == 0+ || layout.country_count > 256+ {+ return Err(invalid("invalid grid or country count"));+ }+ if layout.cell_count+ != layout+ .columns+ .checked_mul(layout.rows)+ .ok_or(invalid("cell count overflow"))?+ || layout.block_count != layout.cell_count.div_ceil(BLOCK_SIZE)+ || layout.nonempty_count > layout.cell_count+ {+ return Err(invalid("invalid grid counts"));+ }++ let countries = range(layout.countries, layout.country_count, 2)+ .ok_or(invalid("country section overflow"))?;+ let blocks = range(layout.blocks, layout.block_count, BLOCK_LEN)+ .ok_or(invalid("block section overflow"))?;+ let offsets = range(+ layout.cell_offsets,+ layout+ .nonempty_count+ .checked_add(1)+ .ok_or(invalid("cell offset overflow"))?,+ 4,+ )+ .ok_or(invalid("cell offset overflow"))?;+ if layout.countries != HEADER_LEN+ || layout.blocks != countries.end+ || layout.cell_offsets != blocks.end+ || layout.cells != offsets.end+ || layout.cells > bytes.len()+ || declared_length != bytes.len()+ {+ return Err(invalid("sections are not contiguous"));+ }++ let mut seen_codes = [false; 26 * 26];+ for index in 0..layout.country_count {+ let offset = layout.countries + index * 2;+ let code = [bytes[offset], bytes[offset + 1]];+ if !code.iter().all(u8::is_ascii_uppercase) {+ return Err(invalid("invalid country code"));+ }+ let slot = usize::from(code[0] - b'A') * 26 + usize::from(code[1] - b'A');+ if seen_codes[slot] {+ return Err(invalid("duplicate country code"));+ }+ seen_codes[slot] = true;+ }++ let mut expected_dense = 0;+ for block in 0..layout.block_count {+ let offset = layout.blocks + block * BLOCK_LEN;+ let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));+ let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;+ if first != expected_dense {+ return Err(invalid("invalid block dense offset"));+ }+ if block + 1 == layout.block_count {+ let used_bits = layout.cell_count - block * BLOCK_SIZE;+ if used_bits < BLOCK_SIZE && mask >> used_bits != 0 {+ return Err(invalid("last block references nonexistent cells"));+ }+ }+ expected_dense += mask.count_ones() as usize;+ }+ if expected_dense != layout.nonempty_count {+ return Err(invalid("wrong nonempty cell count"));+ }++ let cells_length = bytes.len() - layout.cells;+ let mut previous = 0;+ for index in 0..=layout.nonempty_count {+ let offset = u32_at(bytes, layout.cell_offsets + index * 4).expect("validated offset range")+ as usize;+ if offset < previous || offset > cells_length {+ return Err(invalid("invalid cell offset"));+ }+ if index > 0 {+ validate_cell(+ &bytes[layout.cells + previous..layout.cells + offset],+ layout.country_count,+ )?;+ }+ previous = offset;+ }+ if previous != cells_length {+ return Err(invalid("last cell offset is not terminal"));+ }+ Ok(())+}++fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {+ let mut reader = Reader::new(bytes);+ for _ in 0..reader.byte()? {+ validate_country(reader.byte()?, country_count)?;+ }+ for _ in 0..reader.byte()? {+ validate_country(reader.byte()?, country_count)?;+ skip_rings(&mut reader)?;+ skip_rings(&mut reader)?;+ }+ if reader.position != bytes.len() {+ return Err(invalid("cell has trailing bytes"));+ }+ Ok(())+}++fn skip_rings(reader: &mut Reader<'_>) -> crate::Result<()> {+ for _ in 0..reader.byte()? {+ let count = usize::from(u16::from_le_bytes(+ reader.take(2)?.try_into().expect("two-byte slice"),+ ));+ reader.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;+ }+ Ok(())+}++fn validate_country(index: u8, country_count: usize) -> crate::Result<()> {+ if usize::from(index) < country_count {+ Ok(())+ } else {+ Err(invalid("cell references unknown country"))+ }+}++fn point_at(ring: &[u8], index: usize) -> Point {+ let offset = index * 4;+ Point {+ x: u16::from_le_bytes([ring[offset], ring[offset + 1]]),+ y: u16::from_le_bytes([ring[offset + 2], ring[offset + 3]]),+ }+}++fn is_left(start: Point, end: Point, point: Point) -> i64 {+ (i64::from(end.x) - i64::from(start.x)) * (i64::from(point.y) - i64::from(start.y))+ - (i64::from(point.x) - i64::from(start.x)) * (i64::from(end.y) - i64::from(start.y))+}++fn normalize(value: f64, minimum: f64, width: f64) -> f64 {+ (value - minimum).rem_euclid(width) + minimum+}++fn push_unique(countries: &mut Vec<CountryCode>, country: CountryCode) {+ if !countries.contains(&country) {+ countries.push(country);+ }+}++fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {+ Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)+}++fn invalid(reason: &'static str) -> Error {+ Error::invalid(SECTION, reason)+}diff --git a/rust/crates/location/src/dispute.rs b/rust/crates/location/src/dispute.rsnew file mode 100644index 0000000000..c69bda9519--- /dev/null+++ b/rust/crates/location/src/dispute.rs@@ -0,0 +1,449 @@+use std::str;++use crate::binary::{range, u16_at, u32_at};+use crate::country::{CountryGeometry, PreparedCell};+use crate::{CountryCode, Error};++const MAGIC: &[u8; 4] = b"DSPT";+const VERSION: u16 = 1;+const HEADER_LEN: usize = 64;+const TERRITORY_LEN: usize = 12;+const MISSING_COUNTRY: u8 = u8::MAX;+const SECTION: &str = "dispute index";++#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]+pub struct TerritoryId(u16);++impl TerritoryId {+ pub const ARUNACHAL_PRADESH: Self = Self(1);+ pub const TIRPANI_VALLEYS: Self = Self(2);+ pub const BARA_HOTII_VALLEYS: Self = Self(3);+ pub const DEMCHOK: Self = Self(4);+ pub const SAMDU_VALLEYS: Self = Self(5);+ pub const KASHMIR: Self = Self(6);+ pub const TRANS_KARAKORAM_TRACT: Self = Self(7);+ pub const AKSAI_CHIN: Self = Self(8);+ pub const GILGIT_BALTISTAN: Self = Self(9);+ pub const AZAD_KASHMIR: Self = Self(10);+ pub const SIACHEN_GLACIER: Self = Self(11);+ pub const GAZA_STRIP: Self = Self(12);+ pub const WEST_BANK: Self = Self(13);+ pub const GOLAN_HEIGHTS: Self = Self(14);+ pub const SHEBAA_FARMS: Self = Self(15);+ pub const EAST_JERUSALEM: Self = Self(16);+ pub const LATRUN_FORT: Self = Self(17);+ pub const JERUSALEM_NO_MANS_LAND: Self = Self(18);+ pub const MOUNT_SCOPUS: Self = Self(19);+ pub const TAIWAN: Self = Self(20);+ pub const CRIMEA: Self = Self(21);+ pub const DONETSK_REGION: Self = Self(22);+ pub const LUHANSK_REGION: Self = Self(23);+ pub const KHERSON_REGION: Self = Self(24);+ pub const ZAPORIZHZHIA_REGION: Self = Self(25);+ pub const KOSOVO: Self = Self(26);+ pub const WESTERN_SAHARA_MOROCCAN_AREA: Self = Self(27);+ pub const WESTERN_SAHARA_SELF_ADMINISTERED_AREA: Self = Self(28);+ pub const NORTHERN_CYPRUS: Self = Self(29);+ pub const CYPRUS_BUFFER_ZONE: Self = Self(30);+ pub const FALKLAND_ISLANDS: Self = Self(31);++ pub const fn get(self) -> u16 {+ self.0+ }+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum CountryView {+ Neutral,+ SourceDefault,+ Region(CountryCode),+}++#[derive(Clone, Copy, Debug)]+struct Layout {+ worldview_count: usize,+ territory_count: usize,+ country_count: usize,+ countries: usize,+ worldviews: usize,+ territories: usize,+ candidates: usize,+ assignments: usize,+ strings: usize,+ geometry: usize,+}++#[derive(Debug)]+pub(crate) struct DisputeIndex {+ bytes: Box<[u8]>,+ layout: Layout,+ geometry: CountryGeometry,+}++impl DisputeIndex {+ pub(crate) fn from_bytes(bytes: impl Into<Box<[u8]>>) -> crate::Result<Self> {+ let bytes = bytes.into();+ let layout = validate(&bytes)?;+ let geometry = CountryGeometry::from_bytes(bytes[layout.geometry..].to_vec())?;+ validate_geometry(&geometry, &bytes, layout)?;+ Ok(Self {+ bytes,+ layout,+ geometry,+ })+ }++ pub(crate) fn lookup_prepared(+ &self,+ location: PreparedCell,+ ) -> crate::Result<Vec<DisputeMatch<'_>>> {+ Ok(self+ .geometry+ .lookup_prepared(location)?+ .into_iter()+ .map(|code| DisputeMatch {+ index: self,+ territory: territory_index_for_code(&self.bytes, self.layout, code)+ .expect("geometry was validated against the catalog"),+ })+ .collect())+ }++ fn country_code(&self, index: u8) -> CountryCode {+ country_code(&self.bytes, self.layout, index)+ }++ fn worldview_index(&self, code: CountryCode) -> Option<usize> {+ worldview_index(&self.bytes, self.layout, code)+ }+}++#[derive(Clone, Copy, Debug)]+pub struct DisputeMatch<'a> {+ index: &'a DisputeIndex,+ territory: usize,+}++impl<'a> DisputeMatch<'a> {+ pub fn territory_id(self) -> TerritoryId {+ TerritoryId(self.record().id)+ }++ pub fn name(self) -> &'a str {+ let record = self.record();+ self.string(record.name_start, usize::from(record.name_length))+ }++ fn source_default(self) -> Option<CountryCode> {+ decode_optional_country(+ &self.index.bytes,+ self.index.layout,+ self.record().default_country,+ )+ }++ pub fn possible_countries(self) -> impl ExactSizeIterator<Item = CountryCode> + 'a {+ let record = self.record();+ let start = self.index.layout.candidates + usize::from(record.candidate_start);+ self.index.bytes[start..start + usize::from(record.candidate_count)]+ .iter()+ .map(|&country| self.index.country_code(country))+ }++ pub fn resolve(self, view: CountryView) -> Option<CountryCode> {+ match view {+ CountryView::Neutral => None,+ CountryView::SourceDefault => self.source_default(),+ CountryView::Region(region) => {+ if let Some(worldview) = self.index.worldview_index(region) {+ let assignment = self.index.layout.assignments+ + self.territory * self.index.layout.worldview_count+ + worldview;+ decode_optional_country(+ &self.index.bytes,+ self.index.layout,+ self.index.bytes[assignment],+ )+ } else if self.possible_countries().any(|country| country == region) {+ Some(region)+ } else {+ self.source_default()+ }+ }+ }+ }++ fn record(self) -> TerritoryRecord {+ let offset = self.record_offset();+ TerritoryRecord {+ id: u16_at(&self.index.bytes, offset).expect("validated territory"),+ default_country: self.index.bytes[offset + 4],+ candidate_count: self.index.bytes[offset + 5],+ candidate_start: u16_at(&self.index.bytes, offset + 6).expect("validated territory"),+ name_start: u16_at(&self.index.bytes, offset + 8).expect("validated territory"),+ name_length: self.index.bytes[offset + 10],+ }+ }++ fn record_offset(self) -> usize {+ self.index.layout.territories + self.territory * TERRITORY_LEN+ }++ fn string(self, start: u16, length: usize) -> &'a str {+ let start = self.index.layout.strings + usize::from(start);+ str::from_utf8(&self.index.bytes[start..start + length]).expect("validated UTF-8")+ }+}++#[derive(Clone, Copy)]+struct TerritoryRecord {+ id: u16,+ default_country: u8,+ candidate_count: u8,+ candidate_start: u16,+ name_start: u16,+ name_length: u8,+}++fn validate(bytes: &[u8]) -> crate::Result<Layout> {+ if bytes.len() < HEADER_LEN {+ return Err(invalid("truncated"));+ }+ if &bytes[..4] != MAGIC {+ return Err(invalid("invalid magic"));+ }+ let version = read_u16(bytes, 4)?;+ if version != VERSION {+ return Err(Error::invalid(+ SECTION,+ format!("unsupported format version {version}"),+ ));+ }+ if usize::from(read_u16(bytes, 6)?) != HEADER_LEN+ || usize::from(read_u16(bytes, 14)?) != TERRITORY_LEN+ {+ return Err(invalid("unexpected header or territory length"));+ }+ if bytes[48..HEADER_LEN].iter().any(|&byte| byte != 0) {+ return Err(invalid("nonzero reserved header bytes"));+ }+ let layout = Layout {+ worldview_count: usize::from(read_u16(bytes, 8)?),+ territory_count: usize::from(read_u16(bytes, 10)?),+ country_count: usize::from(read_u16(bytes, 12)?),+ countries: read_usize(bytes, 16)?,+ worldviews: read_usize(bytes, 20)?,+ territories: read_usize(bytes, 24)?,+ candidates: read_usize(bytes, 28)?,+ assignments: read_usize(bytes, 32)?,+ strings: read_usize(bytes, 36)?,+ geometry: read_usize(bytes, 40)?,+ };+ validate_layout(bytes, layout, read_usize(bytes, 44)?)?;+ Ok(layout)+}++fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {+ if layout.worldview_count == 0+ || layout.territory_count == 0+ || layout.country_count == 0+ || layout.country_count >= usize::from(MISSING_COUNTRY)+ {+ return Err(invalid("invalid catalog counts"));+ }+ let countries = range(layout.countries, layout.country_count, 2)+ .ok_or(invalid("country table overflow"))?;+ let worldviews = range(layout.worldviews, layout.worldview_count, 1)+ .ok_or(invalid("worldview table overflow"))?;+ let territories = range(layout.territories, layout.territory_count, TERRITORY_LEN)+ .ok_or(invalid("territory table overflow"))?;+ let assignment_count = layout+ .territory_count+ .checked_mul(layout.worldview_count)+ .ok_or(invalid("assignment table overflow"))?;+ let assignments = range(layout.assignments, assignment_count, 1)+ .ok_or(invalid("assignment table overflow"))?;+ if layout.countries != HEADER_LEN+ || layout.worldviews != countries.end+ || layout.territories != worldviews.end+ || layout.candidates != territories.end+ || layout.assignments < layout.candidates+ || layout.strings != assignments.end+ || layout.geometry < layout.strings+ || layout.geometry >= bytes.len()+ || declared_length != bytes.len()+ {+ return Err(invalid("sections are not contiguous"));+ }++ let mut seen_codes = [false; 26 * 26];+ for index in 0..layout.country_count {+ let code = code_at(bytes, layout.countries + index * 2)?;+ let slot = usize::from(code[0] - b'A') * 26 + usize::from(code[1] - b'A');+ if seen_codes[slot] {+ return Err(invalid("duplicate catalog country code"));+ }+ seen_codes[slot] = true;+ }++ let mut seen_worldviews = [false; 255];+ for &country in &bytes[layout.worldviews..worldviews.end] {+ validate_country(country, layout.country_count)?;+ if seen_worldviews[usize::from(country)] {+ return Err(invalid("duplicate worldview"));+ }+ seen_worldviews[usize::from(country)] = true;+ }++ let candidate_length = layout.assignments - layout.candidates;+ let string_length = layout.geometry - layout.strings;+ let mut ids = Vec::with_capacity(layout.territory_count);+ let mut geometry_codes = Vec::with_capacity(layout.territory_count);+ for territory in 0..layout.territory_count {+ let offset = layout.territories + territory * TERRITORY_LEN;+ let id = read_u16_infallible(bytes, offset);+ if id == 0 || ids.contains(&id) {+ return Err(invalid("invalid or duplicate territory ID"));+ }+ ids.push(id);+ let geometry_code = code_at(bytes, offset + 2)?;+ if geometry_codes.contains(&geometry_code) {+ return Err(invalid("duplicate territory geometry code"));+ }+ geometry_codes.push(geometry_code);+ validate_optional_country(bytes[offset + 4], layout.country_count)?;++ let candidate_start = usize::from(read_u16_infallible(bytes, offset + 6));+ let candidate_end = candidate_start+ .checked_add(usize::from(bytes[offset + 5]))+ .ok_or(invalid("candidate range overflow"))?;+ if candidate_end > candidate_length {+ return Err(invalid("candidate range exceeds table"));+ }+ for &country in+ &bytes[layout.candidates + candidate_start..layout.candidates + candidate_end]+ {+ validate_country(country, layout.country_count)?;+ }+ for &country in &bytes[layout.assignments + territory * layout.worldview_count+ ..layout.assignments + (territory + 1) * layout.worldview_count]+ {+ validate_optional_country(country, layout.country_count)?;+ }++ let name_start = usize::from(read_u16_infallible(bytes, offset + 8));+ let name_length = usize::from(bytes[offset + 10]);+ if name_length == 0 {+ return Err(invalid("empty territory name"));+ }+ validate_string(+ bytes,+ layout.strings,+ string_length,+ name_start,+ name_length,+ )?;+ if bytes[offset + 11] != 0 {+ return Err(invalid("nonzero reserved territory byte"));+ }+ }+ Ok(())+}++fn validate_geometry(+ geometry: &CountryGeometry,+ bytes: &[u8],+ layout: Layout,+) -> crate::Result<()> {+ if geometry.columns() != 360+ || geometry.rows() != 180+ || geometry.country_count() != layout.territory_count+ || geometry+ .country_codes()+ .any(|code| territory_index_for_code(bytes, layout, code).is_none())+ {+ return Err(invalid("geometry does not match the territory catalog"));+ }+ Ok(())+}++fn validate_string(+ bytes: &[u8],+ strings: usize,+ string_length: usize,+ start: usize,+ length: usize,+) -> crate::Result<()> {+ let end = start+ .checked_add(length)+ .ok_or(invalid("string range overflow"))?;+ if end > string_length {+ return Err(invalid("string exceeds string table"));+ }+ str::from_utf8(&bytes[strings + start..strings + end]).map_err(|_| invalid("invalid UTF-8"))?;+ Ok(())+}++fn territory_index_for_code(bytes: &[u8], layout: Layout, code: CountryCode) -> Option<usize> {+ (0..layout.territory_count).find(|&index| {+ let offset = layout.territories + index * TERRITORY_LEN + 2;+ bytes[offset..offset + 2] == code.as_bytes()+ })+}++fn worldview_index(bytes: &[u8], layout: Layout, code: CountryCode) -> Option<usize> {+ (0..layout.worldview_count).find(|&index| {+ let country = bytes[layout.worldviews + index];+ country_code(bytes, layout, country) == code+ })+}++fn country_code(bytes: &[u8], layout: Layout, index: u8) -> CountryCode {+ let offset = layout.countries + usize::from(index) * 2;+ CountryCode::from_validated([bytes[offset], bytes[offset + 1]])+}++fn decode_optional_country(bytes: &[u8], layout: Layout, index: u8) -> Option<CountryCode> {+ (index != MISSING_COUNTRY).then(|| country_code(bytes, layout, index))+}++fn validate_country(index: u8, count: usize) -> crate::Result<()> {+ if usize::from(index) < count {+ Ok(())+ } else {+ Err(invalid("catalog references unknown country"))+ }+}++fn validate_optional_country(index: u8, count: usize) -> crate::Result<()> {+ if index == MISSING_COUNTRY {+ Ok(())+ } else {+ validate_country(index, count)+ }+}++fn code_at(bytes: &[u8], offset: usize) -> crate::Result<[u8; 2]> {+ let code = [bytes[offset], bytes[offset + 1]];+ if code.iter().all(u8::is_ascii_uppercase) {+ Ok(code)+ } else {+ Err(invalid("invalid country code"))+ }+}++fn read_u16(bytes: &[u8], offset: usize) -> crate::Result<u16> {+ u16_at(bytes, offset).ok_or(invalid("truncated"))+}++fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {+ Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)+}++fn read_u16_infallible(bytes: &[u8], offset: usize) -> u16 {+ u16_at(bytes, offset).expect("validated territory field")+}++fn invalid(reason: &'static str) -> Error {+ Error::invalid(SECTION, reason)+}diff --git a/rust/crates/location/src/error.rs b/rust/crates/location/src/error.rsindex 91f65d9be2..479df1e66c 100644--- a/rust/crates/location/src/error.rs+++ b/rust/crates/location/src/error.rs@@ -13,6 +13,8 @@ pub enum Error {section: &'static str,reason: Cow<'static, str>,},+ #[error("coordinate is outside the valid latitude/longitude range")]+ InvalidCoordinate,#[error("map point {0} has an invalid coordinate")]InvalidMapPoint(usize),#[error("minimum marker distance must be finite and positive")]diff --git a/rust/crates/location/src/lib.rs b/rust/crates/location/src/lib.rsindex 412c4a1205..1fa2202830 100644--- a/rust/crates/location/src/lib.rs+++ b/rust/crates/location/src/lib.rs@@ -1,14 +1,23 @@+use std::collections::BTreeMap;use std::fmt;+use std::fs;+use std::path::Path;use std::str::FromStr;mod binary;mod city;pub mod cluster;+mod country;+mod dispute;mod error;pub use city::{City, CityIndex, CityMatch};+pub use dispute::{CountryView, DisputeMatch, TerritoryId};pub use error::{Error, Result};+use country::CountryGeometry;+use dispute::DisputeIndex;+#[derive(Clone, Copy, Debug, PartialEq)]pub struct Coordinate {pub latitude: f64,@@ -77,3 +86,126 @@ impl FromStr for CountryCode {#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]#[error("country codes must contain two uppercase ASCII letters")]pub struct InvalidCountryCode;++pub struct CountryIndex {+ countries: CountryGeometry,+ disputes: DisputeIndex,+}++impl CountryIndex {+ pub fn from_paths(countries: impl AsRef<Path>, disputes: impl AsRef<Path>) -> Result<Self> {+ Self::from_bytes(fs::read(countries)?, fs::read(disputes)?)+ }++ pub fn from_bytes(+ countries: impl Into<Box<[u8]>>,+ disputes: impl Into<Box<[u8]>>,+ ) -> Result<Self> {+ let countries = CountryGeometry::from_bytes(countries)?;+ let disputes = DisputeIndex::from_bytes(disputes)?;+ if countries.columns() != 360 || countries.rows() != 180 {+ return Err(Error::invalid(+ "country index",+ "country grid must be 360 by 180",+ ));+ }+ Ok(Self {+ countries,+ disputes,+ })+ }++ pub fn lookup(&self, coordinate: Coordinate) -> Result<CountryClassification<'_>> {+ let cell = self+ .countries+ .prepare_cell(coordinate.latitude, coordinate.longitude)?;+ Ok(CountryClassification {+ countries: self.countries.lookup_prepared(cell)?,+ disputes: self.disputes.lookup_prepared(cell)?,+ })+ }++ pub fn group(&self, coordinates: &[Coordinate], view: CountryView) -> Result<CountryGrouping> {+ let mut countries = BTreeMap::<CountryCode, Vec<u32>>::new();+ let mut disputes = BTreeMap::<TerritoryId, DisputeGroup>::new();+ let mut unclassified_coordinate_indices = Vec::new();++ for (coordinate_index, &coordinate) in coordinates.iter().enumerate() {+ let classification = self.lookup(coordinate)?;+ let coordinate_index = coordinate_index as u32;+ if classification.disputes.is_empty() {+ if classification.countries.is_empty() {+ unclassified_coordinate_indices.push(coordinate_index);+ }+ for country in classification.countries {+ countries.entry(country).or_default().push(coordinate_index);+ }+ continue;+ }++ let mut resolved = Vec::new();+ for dispute in classification.disputes {+ let country = dispute.resolve(view);+ disputes+ .entry(dispute.territory_id())+ .or_insert_with(|| DisputeGroup {+ territory: dispute.territory_id(),+ name: dispute.name().to_owned(),+ possible_countries: dispute.possible_countries().collect(),+ resolved_country: country,+ coordinate_indices: Vec::new(),+ })+ .coordinate_indices+ .push(coordinate_index);+ if let Some(country) = country+ && !resolved.contains(&country)+ {+ resolved.push(country);+ }+ }+ for country in resolved {+ countries.entry(country).or_default().push(coordinate_index);+ }+ }++ Ok(CountryGrouping {+ countries: countries+ .into_iter()+ .map(|(country, coordinate_indices)| CountryGroup {+ country,+ coordinate_indices,+ })+ .collect(),+ disputes: disputes.into_values().collect(),+ unclassified_coordinate_indices,+ })+ }+}++#[derive(Debug)]+pub struct CountryClassification<'a> {+ pub countries: Vec<CountryCode>,+ pub disputes: Vec<DisputeMatch<'a>>,+}++#[derive(Debug, Eq, PartialEq)]+pub struct CountryGrouping {+ pub countries: Vec<CountryGroup>,+ pub disputes: Vec<DisputeGroup>,+ pub unclassified_coordinate_indices: Vec<u32>,+}++#[derive(Debug, Eq, PartialEq)]+pub struct CountryGroup {+ pub country: CountryCode,+ pub coordinate_indices: Vec<u32>,+}++#[derive(Debug, Eq, PartialEq)]+pub struct DisputeGroup {+ pub territory: TerritoryId,+ pub name: String,+ pub possible_countries: Vec<CountryCode>,+ pub resolved_country: Option<CountryCode>,+ pub coordinate_indices: Vec<u32>,+}