Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Add offline country lookup

Neeraj Gupta86bda9ceebe4
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 1e6f782d65..aec02142c2 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -880,6 +880,12 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "bufrw"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "394c22b416e3981206df472c3bcbbf9ae130883619aefc2199e6087517af1bba"
+
[[package]]
name = "build-target"
version = "0.4.0"
@@ -1884,6 +1890,17 @@ dependencies = [
"parking_lot_core",
]
+[[package]]
+name = "dbase"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abde818b4b82df61e01348c2b4610c221bcecb97cb7f567da086edf295d8d126"
+dependencies = [
+ "bufrw",
+ "byteorder",
+ "time",
+]
+
[[package]]
name = "dbus"
version = "0.9.11"
@@ -2641,8 +2658,10 @@ name = "ente-location-dataset"
version = "0.0.0"
dependencies = [
"ente-location",
+ "geo",
"reqwest 0.12.28",
"sha2 0.11.0",
+ "shapefile",
"thiserror 2.0.18",
"zip 8.6.0",
]
@@ -3208,6 +3227,12 @@ dependencies = [
"zlib-rs",
]
+[[package]]
+name = "float_next_after"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37007738a80ea34f969af54a3390dd72cacdef654974cfd449c9f6f72dbaac10"
+
[[package]]
name = "flutter_rust_bridge"
version = "2.12.0"
@@ -3604,6 +3629,46 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "geo"
+version = "0.33.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30eb1fdc57c1e5cfd11826fe0caec4b9dc7901f3758263bb506228d88c8d9e9a"
+dependencies = [
+ "float_next_after",
+ "geo-types",
+ "geographiclib-rs",
+ "i_overlay",
+ "log",
+ "num-traits",
+ "rand 0.10.1",
+ "rand_pcg",
+ "robust",
+ "rstar",
+ "sif-itree",
+]
+
+[[package]]
+name = "geo-types"
+version = "0.7.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5"
+dependencies = [
+ "approx",
+ "num-traits",
+ "rstar",
+ "serde",
+]
+
+[[package]]
+name = "geographiclib-rs"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7"
+dependencies = [
+ "libm",
+]
+
[[package]]
name = "getrandom"
version = "0.2.17"
@@ -3862,6 +3927,15 @@ dependencies = [
"zerocopy 0.8.48",
]
+[[package]]
+name = "hash32"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -3900,6 +3974,16 @@ dependencies = [
"hashbrown 0.16.1",
]
+[[package]]
+name = "heapless"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad"
+dependencies = [
+ "hash32",
+ "stable_deref_trait",
+]
+
[[package]]
name = "heck"
version = "0.4.1"
@@ -4167,6 +4251,48 @@ dependencies = [
"windows-registry",
]
+[[package]]
+name = "i_float"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "813145bb0ad5b60f55cbbf3c74cdceda1c0a9d253b35c4cc36ae0df7887cb78f"
+dependencies = [
+ "libm",
+]
+
+[[package]]
+name = "i_key_sort"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d73d122b937fca067feb0ad74f62388920272b27c356d4df2d0cfdd59e044cf0"
+
+[[package]]
+name = "i_overlay"
+version = "4.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8dd314b4668e2b3a12508f2e125558c82a6c0a8636fa5107a900f79ce414e450"
+dependencies = [
+ "i_float",
+ "i_key_sort",
+ "i_shape",
+ "i_tree",
+]
+
+[[package]]
+name = "i_shape"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa9eac533d7509a8ab87672b60ac610c17240f9ea4851d26227689fdfe349c8"
+dependencies = [
+ "i_float",
+]
+
+[[package]]
+name = "i_tree"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4804bdc1dc124eb7e1aa9e144ecc04096bcf787a10a15fa44af682b51f0f6cce"
+
[[package]]
name = "iana-time-zone"
version = "0.1.65"
@@ -7322,6 +7448,15 @@ dependencies = [
"rand 0.8.6",
]
+[[package]]
+name = "rand_pcg"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
+dependencies = [
+ "rand_core 0.10.1",
+]
+
[[package]]
name = "rav1d"
version = "1.1.0"
@@ -7741,6 +7876,12 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "robust"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839"
+
[[package]]
name = "rsqlite-vfs"
version = "0.1.0"
@@ -7751,6 +7892,17 @@ dependencies = [
"thiserror 2.0.18",
]
+[[package]]
+name = "rstar"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb"
+dependencies = [
+ "heapless",
+ "num-traits",
+ "smallvec",
+]
+
[[package]]
name = "rubato"
version = "0.16.2"
@@ -8478,6 +8630,17 @@ dependencies = [
"sponge-cursor",
]
+[[package]]
+name = "shapefile"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92fedc0c44dc53a02655cd8dccd2fdf3159c372c0f6eb59115e889f11325e4f0"
+dependencies = [
+ "byteorder",
+ "dbase",
+ "geo-types",
+]
+
[[package]]
name = "shell-words"
version = "1.1.1"
@@ -8490,6 +8653,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+[[package]]
+name = "sif-itree"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7f45b8998ced5134fb1d75732c77842a3e888f19c1ff98481822e8fbfbf930b"
+
[[package]]
name = "signal-hook-registry"
version = "1.4.8"
diff --git a/rust/crates/location-dataset/Cargo.toml b/rust/crates/location-dataset/Cargo.toml
index 89c5542fcc..85141b006a 100644
--- a/rust/crates/location-dataset/Cargo.toml
+++ b/rust/crates/location-dataset/Cargo.toml
@@ -5,7 +5,9 @@ publish = false
[dependencies]
ente-location.workspace = true
+geo = { version = "0.33.1", default-features = false }
reqwest = { workspace = true, features = ["blocking", "rustls-tls-webpki-roots"] }
sha2.workspace = true
+shapefile = { version = "0.9.0", features = ["geo-types"] }
thiserror.workspace = true
zip = { version = "8.6.0", default-features = false, features = ["deflate-flate2-zlib-rs"] }
diff --git a/rust/crates/location-dataset/README.md b/rust/crates/location-dataset/README.md
index 217695fc11..9c81f92152 100644
--- a/rust/crates/location-dataset/README.md
+++ b/rust/crates/location-dataset/README.md
@@ -1,14 +1,14 @@
# ente-location-dataset
-Host-side generator for the city asset consumed by `ente-location`.
+Host-side generator for the assets consumed by `ente-location`.
```sh
cargo run --release -p ente-location-dataset -- \
build --output /tmp/ente-location
```
-The command downloads and caches its sources, writes `cities.bin`, and prints
-its size and SHA-256 hash.
+The command downloads and caches its sources, writes `cities.bin`,
+`countries.bin`, and `disputes.bin`, and prints their sizes and SHA-256 hashes.
## Source
@@ -16,3 +16,6 @@ City data is derived from [GeoNames](https://www.geonames.org/) `cities5000`
and `countryInfo`, licensed under
[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Ente filters and
converts the data, which is provided without warranty.
+
+Country boundaries are derived from [Natural Earth](https://www.naturalearthdata.com/),
+which is in the public domain.
diff --git a/rust/crates/location-dataset/src/catalog.rs b/rust/crates/location-dataset/src/catalog.rs
new file mode 100644
index 0000000000..5086b12bd4
--- /dev/null
+++ b/rust/crates/location-dataset/src/catalog.rs
@@ -0,0 +1,128 @@
+use ente_location::TerritoryId;
+
+pub(crate) struct DisputedAreaSource {
+ pub source_id: &'static str,
+ pub geometry_code: [u8; 2],
+ pub territory: TerritoryId,
+ pub additional_countries: &'static [[u8; 2]],
+}
+
+pub(crate) struct UkrainianRegionSource {
+ pub iso_3166_2: &'static str,
+ pub geometry_code: [u8; 2],
+ pub territory: TerritoryId,
+}
+
+pub(crate) const WORLDVIEWS: &[([u8; 2], &str)] = &[
+ (*b"AR", "ADM0_A3_AR"),
+ (*b"BD", "ADM0_A3_BD"),
+ (*b"BR", "ADM0_A3_BR"),
+ (*b"CN", "ADM0_A3_CN"),
+ (*b"DE", "ADM0_A3_DE"),
+ (*b"EG", "ADM0_A3_EG"),
+ (*b"ES", "ADM0_A3_ES"),
+ (*b"FR", "ADM0_A3_FR"),
+ (*b"GB", "ADM0_A3_GB"),
+ (*b"GR", "ADM0_A3_GR"),
+ (*b"IN", "ADM0_A3_IN"),
+ (*b"ID", "ADM0_A3_ID"),
+ (*b"IL", "ADM0_A3_IL"),
+ (*b"IT", "ADM0_A3_IT"),
+ (*b"JP", "ADM0_A3_JP"),
+ (*b"KR", "ADM0_A3_KO"),
+ (*b"MA", "ADM0_A3_MA"),
+ (*b"NP", "ADM0_A3_NP"),
+ (*b"NL", "ADM0_A3_NL"),
+ (*b"PK", "ADM0_A3_PK"),
+ (*b"PL", "ADM0_A3_PL"),
+ (*b"PT", "ADM0_A3_PT"),
+ (*b"PS", "ADM0_A3_PS"),
+ (*b"RU", "ADM0_A3_RU"),
+ (*b"SA", "ADM0_A3_SA"),
+ (*b"SE", "ADM0_A3_SE"),
+ (*b"TR", "ADM0_A3_TR"),
+ (*b"TW", "ADM0_A3_TW"),
+ (*b"UA", "ADM0_A3_UA"),
+ (*b"US", "ADM0_A3_US"),
+ (*b"VN", "ADM0_A3_VN"),
+];
+
+const NONE: &[[u8; 2]] = &[];
+const PALESTINE: &[[u8; 2]] = &[*b"PS"];
+
+pub(crate) const DISPUTED_AREAS: &[DisputedAreaSource] = &[
+ area("B00", *b"AP", TerritoryId::ARUNACHAL_PRADESH, NONE),
+ area("B01", *b"TV", TerritoryId::TIRPANI_VALLEYS, NONE),
+ area("B02", *b"BH", TerritoryId::BARA_HOTII_VALLEYS, NONE),
+ area("B03", *b"DM", TerritoryId::DEMCHOK, NONE),
+ area("B04", *b"SV", TerritoryId::SAMDU_VALLEYS, NONE),
+ area("B05", *b"KM", TerritoryId::KASHMIR, NONE),
+ area("B06", *b"TK", TerritoryId::TRANS_KARAKORAM_TRACT, NONE),
+ area("B07", *b"AC", TerritoryId::AKSAI_CHIN, NONE),
+ area("B08", *b"GB", TerritoryId::GILGIT_BALTISTAN, NONE),
+ area("B09", *b"AK", TerritoryId::AZAD_KASHMIR, NONE),
+ area("B45", *b"SC", TerritoryId::SIACHEN_GLACIER, NONE),
+ area("B53", *b"GZ", TerritoryId::GAZA_STRIP, NONE),
+ area("B54", *b"WB", TerritoryId::WEST_BANK, NONE),
+ area("B16", *b"GH", TerritoryId::GOLAN_HEIGHTS, NONE),
+ area("B58", *b"SF", TerritoryId::SHEBAA_FARMS, NONE),
+ area("B98", *b"EJ", TerritoryId::EAST_JERUSALEM, NONE),
+ area("B78", *b"LF", TerritoryId::LATRUN_FORT, NONE),
+ area("B79", *b"JN", TerritoryId::JERUSALEM_NO_MANS_LAND, NONE),
+ area("B99", *b"MS", TerritoryId::MOUNT_SCOPUS, PALESTINE),
+ area("B77", *b"TW", TerritoryId::TAIWAN, NONE),
+ area("B89", *b"CR", TerritoryId::CRIMEA, NONE),
+ area("B57", *b"KO", TerritoryId::KOSOVO, NONE),
+ area(
+ "B19",
+ *b"WM",
+ TerritoryId::WESTERN_SAHARA_MOROCCAN_AREA,
+ NONE,
+ ),
+ area(
+ "B28",
+ *b"WS",
+ TerritoryId::WESTERN_SAHARA_SELF_ADMINISTERED_AREA,
+ NONE,
+ ),
+ area("B20", *b"NC", TerritoryId::NORTHERN_CYPRUS, NONE),
+ area("B43", *b"CB", TerritoryId::CYPRUS_BUFFER_ZONE, NONE),
+ area("B12", *b"FK", TerritoryId::FALKLAND_ISLANDS, NONE),
+];
+
+pub(crate) const UKRAINIAN_REGIONS: &[UkrainianRegionSource] = &[
+ UkrainianRegionSource {
+ iso_3166_2: "UA-14",
+ geometry_code: *b"DO",
+ territory: TerritoryId::DONETSK_REGION,
+ },
+ UkrainianRegionSource {
+ iso_3166_2: "UA-09",
+ geometry_code: *b"LU",
+ territory: TerritoryId::LUHANSK_REGION,
+ },
+ UkrainianRegionSource {
+ iso_3166_2: "UA-65",
+ geometry_code: *b"KH",
+ territory: TerritoryId::KHERSON_REGION,
+ },
+ UkrainianRegionSource {
+ iso_3166_2: "UA-23",
+ geometry_code: *b"ZP",
+ territory: TerritoryId::ZAPORIZHZHIA_REGION,
+ },
+];
+
+const fn area(
+ source_id: &'static str,
+ geometry_code: [u8; 2],
+ territory: TerritoryId,
+ additional_countries: &'static [[u8; 2]],
+) -> DisputedAreaSource {
+ DisputedAreaSource {
+ source_id,
+ geometry_code,
+ territory,
+ additional_countries,
+ }
+}
diff --git a/rust/crates/location-dataset/src/country.rs b/rust/crates/location-dataset/src/country.rs
new file mode 100644
index 0000000000..321293c52e
--- /dev/null
+++ b/rust/crates/location-dataset/src/country.rs
@@ -0,0 +1,383 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+
+use geo::{BooleanOps, BoundingRect, Contains, Coord, LineString, MultiPolygon, Rect};
+use shapefile::dbase::{FieldValue, Record};
+use shapefile::{Reader, Shape};
+
+use crate::{Result, invalid};
+
+const COLUMNS: u16 = 360;
+const ROWS: u16 = 180;
+const HEADER_LEN: usize = 48;
+const BLOCK_SIZE: usize = 64;
+const BLOCK_LEN: usize = 12;
+
+pub(crate) struct Area {
+ pub code: [u8; 2],
+ pub geometry: MultiPolygon<f64>,
+}
+
+#[derive(Default)]
+struct Cell {
+ containing: BTreeSet<[u8; 2]>,
+ intersecting: BTreeMap<[u8; 2], CellAreas>,
+}
+
+#[derive(Default)]
+struct CellAreas {
+ outer: Vec<Vec<[u16; 2]>>,
+ inner: Vec<Vec<[u16; 2]>>,
+}
+
+pub(crate) fn build_countries(path: &Path) -> Result<Vec<u8>> {
+ let mut reader = Reader::from_path(path)?;
+ let mut areas = Vec::new();
+ for shape_record in reader.iter_shapes_and_records() {
+ let (shape, record) = shape_record?;
+ let Some(code) = natural_earth_country_code(&record) else {
+ continue;
+ };
+ if &code == b"AQ" {
+ continue;
+ }
+ areas.push(Area {
+ code,
+ geometry: polygon_geometry(shape, "country")?,
+ });
+ }
+ encode(&areas)
+}
+
+pub(crate) fn encode(areas: &[Area]) -> Result<Vec<u8>> {
+ if areas.is_empty() {
+ return Err(invalid("country geometry is empty"));
+ }
+ let mut cells: Vec<Cell> = (0..usize::from(COLUMNS) * usize::from(ROWS))
+ .map(|_| Cell::default())
+ .collect();
+ for area in areas {
+ validate_code(area.code)?;
+ add_area(&mut cells, area)?;
+ }
+ encode_cells(&cells)
+}
+
+fn add_area(cells: &mut [Cell], area: &Area) -> Result<()> {
+ for polygon in &area.geometry {
+ add_polygon(cells, area.code, polygon)?;
+ }
+ Ok(())
+}
+
+fn add_polygon(cells: &mut [Cell], code: [u8; 2], polygon: &geo::Polygon<f64>) -> Result<()> {
+ let bounds = polygon
+ .bounding_rect()
+ .ok_or_else(|| invalid("country polygon has no bounds"))?;
+ let minimum_column = longitude_column(bounds.min().x);
+ let maximum_column = longitude_column(bounds.max().x);
+ let northern_row = latitude_row(bounds.max().y);
+ let southern_row = latitude_row(bounds.min().y);
+
+ for row in northern_row..=southern_row {
+ for column in minimum_column..=maximum_column {
+ let bounds = cell_rect(column, row);
+ let cell = &mut cells[row * usize::from(COLUMNS) + column];
+ if polygon.contains(&bounds) {
+ cell.containing.insert(code);
+ cell.intersecting.remove(&code);
+ continue;
+ }
+ if cell.containing.contains(&code) {
+ continue;
+ }
+ let intersection = polygon.intersection(&bounds.to_polygon());
+ if intersection.0.is_empty() {
+ continue;
+ }
+ let areas = cell.intersecting.entry(code).or_default();
+ for polygon in intersection {
+ if let Some(ring) = encode_ring(polygon.exterior(), bounds) {
+ areas.outer.push(ring);
+ }
+ for ring in polygon.interiors() {
+ if let Some(ring) = encode_ring(ring, bounds) {
+ areas.inner.push(ring);
+ }
+ }
+ }
+ if areas.outer.is_empty() {
+ cell.intersecting.remove(&code);
+ }
+ }
+ }
+ Ok(())
+}
+
+fn encode_cells(cells: &[Cell]) -> Result<Vec<u8>> {
+ let countries: Vec<[u8; 2]> = cells
+ .iter()
+ .flat_map(|cell| {
+ cell.containing
+ .iter()
+ .copied()
+ .chain(cell.intersecting.keys().copied())
+ })
+ .collect::<BTreeSet<_>>()
+ .into_iter()
+ .collect();
+ if countries.is_empty() || countries.len() > 256 {
+ return Err(invalid("country table does not fit one-byte indices"));
+ }
+ let country_indices: BTreeMap<[u8; 2], u8> = countries
+ .iter()
+ .copied()
+ .enumerate()
+ .map(|(index, code)| (code, index as u8))
+ .collect();
+
+ let encoded_cells: Vec<Option<Vec<u8>>> = cells
+ .iter()
+ .map(|cell| encode_cell(cell, &country_indices))
+ .collect::<Result<_>>()?;
+ let cell_count = cells.len();
+ let block_count = cell_count.div_ceil(BLOCK_SIZE);
+ let nonempty_count = encoded_cells.iter().filter(|cell| cell.is_some()).count();
+ let countries_offset = HEADER_LEN;
+ let blocks_offset = countries_offset + countries.len() * 2;
+ let cell_offsets_offset = blocks_offset + block_count * BLOCK_LEN;
+ let cells_offset = cell_offsets_offset + (nonempty_count + 1) * 4;
+ let data_length: usize = encoded_cells.iter().flatten().map(Vec::len).sum();
+ let file_length = cells_offset
+ .checked_add(data_length)
+ .ok_or_else(|| invalid("country index size overflow"))?;
+ let mut output = Vec::with_capacity(file_length);
+
+ output.extend_from_slice(b"CTRY");
+ push_u16(&mut output, 1);
+ push_u16(&mut output, HEADER_LEN as u16);
+ push_u16(&mut output, COLUMNS);
+ push_u16(&mut output, ROWS);
+ push_u16(
+ &mut output,
+ u16::try_from(countries.len()).map_err(|_| invalid("too many countries"))?,
+ );
+ push_u16(&mut output, BLOCK_SIZE as u16);
+ for value in [
+ cell_count,
+ block_count,
+ nonempty_count,
+ countries_offset,
+ blocks_offset,
+ cell_offsets_offset,
+ cells_offset,
+ file_length,
+ ] {
+ push_len(&mut output, value)?;
+ }
+ for code in &countries {
+ output.extend_from_slice(code);
+ }
+
+ let mut dense_start = 0;
+ for block in 0..block_count {
+ let mut mask = 0_u64;
+ for bit in 0..BLOCK_SIZE {
+ let index = block * BLOCK_SIZE + bit;
+ if index < cell_count && encoded_cells[index].is_some() {
+ mask |= 1_u64 << bit;
+ }
+ }
+ output.extend_from_slice(&mask.to_le_bytes());
+ push_len(&mut output, dense_start)?;
+ dense_start += mask.count_ones() as usize;
+ }
+
+ let mut cell_offset = 0;
+ push_len(&mut output, cell_offset)?;
+ for cell in encoded_cells.iter().flatten() {
+ cell_offset += cell.len();
+ push_len(&mut output, cell_offset)?;
+ }
+ for cell in encoded_cells.into_iter().flatten() {
+ output.extend_from_slice(&cell);
+ }
+ debug_assert_eq!(output.len(), file_length);
+ Ok(output)
+}
+
+fn encode_cell(cell: &Cell, countries: &BTreeMap<[u8; 2], u8>) -> Result<Option<Vec<u8>>> {
+ if cell.containing.is_empty() && cell.intersecting.is_empty() {
+ return Ok(None);
+ }
+ let mut output = Vec::new();
+ output.push(
+ u8::try_from(cell.containing.len())
+ .map_err(|_| invalid("too many containing countries in one cell"))?,
+ );
+ output.extend(cell.containing.iter().map(|code| countries[code]));
+ output.push(
+ u8::try_from(cell.intersecting.len())
+ .map_err(|_| invalid("too many intersecting countries in one cell"))?,
+ );
+ for (code, areas) in &cell.intersecting {
+ output.push(countries[code]);
+ encode_rings(&mut output, &areas.outer)?;
+ encode_rings(&mut output, &areas.inner)?;
+ }
+ Ok(Some(output))
+}
+
+fn encode_rings(output: &mut Vec<u8>, rings: &[Vec<[u16; 2]>]) -> Result<()> {
+ output.push(
+ u8::try_from(rings.len()).map_err(|_| invalid("too many clipped rings in one cell"))?,
+ );
+ for ring in rings {
+ push_u16(
+ output,
+ u16::try_from(ring.len()).map_err(|_| invalid("clipped ring is too long"))?,
+ );
+ for &[x, y] in ring {
+ push_u16(output, x);
+ push_u16(output, y);
+ }
+ }
+ Ok(())
+}
+
+fn encode_ring(ring: &LineString<f64>, bounds: Rect<f64>) -> Option<Vec<[u16; 2]>> {
+ let width = bounds.width();
+ let height = bounds.height();
+ let mut points: Vec<[u16; 2]> = ring
+ .0
+ .iter()
+ .map(|coordinate| {
+ [
+ quantize(coordinate.x, bounds.min().x, width),
+ quantize(coordinate.y, bounds.min().y, height),
+ ]
+ })
+ .collect();
+ points.dedup();
+ while points.len() > 3 && points.first() == points.last() {
+ points.pop();
+ }
+ loop {
+ if points.len() <= 3 {
+ break;
+ }
+ let removable: Vec<bool> = (0..points.len())
+ .map(|index| {
+ lies_on_segment(
+ points[(index + points.len() - 1) % points.len()],
+ points[index],
+ points[(index + 1) % points.len()],
+ )
+ })
+ .collect();
+ let retained = removable.iter().filter(|&&remove| !remove).count();
+ if retained < 3 || retained == points.len() {
+ break;
+ }
+ points = points
+ .into_iter()
+ .zip(removable)
+ .filter_map(|(point, remove)| (!remove).then_some(point))
+ .collect();
+ }
+ (points.len() >= 3).then_some(points)
+}
+
+fn cell_rect(column: usize, row: usize) -> Rect<f64> {
+ let minimum_longitude = -180.0 + column as f64 * 360.0 / f64::from(COLUMNS);
+ let maximum_latitude = 90.0 - row as f64 * 180.0 / f64::from(ROWS);
+ Rect::new(
+ Coord {
+ x: minimum_longitude,
+ y: maximum_latitude - 180.0 / f64::from(ROWS),
+ },
+ Coord {
+ x: minimum_longitude + 360.0 / f64::from(COLUMNS),
+ y: maximum_latitude,
+ },
+ )
+}
+
+fn longitude_column(longitude: f64) -> usize {
+ (((longitude + 180.0) / 360.0) * f64::from(COLUMNS))
+ .floor()
+ .clamp(0.0, f64::from(COLUMNS - 1)) as usize
+}
+
+fn latitude_row(latitude: f64) -> usize {
+ (((90.0 - latitude) / 180.0) * f64::from(ROWS))
+ .floor()
+ .clamp(0.0, f64::from(ROWS - 1)) as usize
+}
+
+fn quantize(value: f64, minimum: f64, width: f64) -> u16 {
+ (((value - minimum) * f64::from(u16::MAX) / width).clamp(0.0, f64::from(u16::MAX))) as u16
+}
+
+fn lies_on_segment(start: [u16; 2], point: [u16; 2], end: [u16; 2]) -> bool {
+ let cross = (i64::from(point[0]) - i64::from(start[0]))
+ * (i64::from(end[1]) - i64::from(start[1]))
+ - (i64::from(point[1]) - i64::from(start[1])) * (i64::from(end[0]) - i64::from(start[0]));
+ cross == 0
+ && point[0] >= start[0].min(end[0])
+ && point[0] <= start[0].max(end[0])
+ && point[1] >= start[1].min(end[1])
+ && point[1] <= start[1].max(end[1])
+}
+
+pub(crate) fn natural_earth_country_code(record: &Record) -> Option<[u8; 2]> {
+ ["ISO_A2_EH", "ISO_A2"]
+ .into_iter()
+ .filter_map(|field| character_field(record, field))
+ .find_map(|value| {
+ let code: [u8; 2] = value.as_bytes().try_into().ok()?;
+ code.iter().all(u8::is_ascii_uppercase).then_some(code)
+ })
+}
+
+pub(crate) fn character_field<'a>(record: &'a Record, field: &str) -> Option<&'a str> {
+ match record.get(field) {
+ Some(FieldValue::Character(Some(value))) => Some(value.trim()),
+ _ => None,
+ }
+}
+
+pub(crate) fn polygon_geometry(shape: Shape, source: &str) -> Result<MultiPolygon<f64>> {
+ let Shape::Polygon(polygon) = shape else {
+ return Err(invalid(format!("{source} contains a non-polygon shape")));
+ };
+ let geometry: MultiPolygon<f64> = polygon
+ .try_into()
+ .map_err(|error| invalid(format!("cannot convert {source} polygon: {error}")))?;
+ if geometry.0.is_empty() {
+ Err(invalid(format!("{source} has empty geometry")))
+ } else {
+ Ok(geometry)
+ }
+}
+
+fn validate_code(code: [u8; 2]) -> Result<()> {
+ if code.iter().all(u8::is_ascii_uppercase) {
+ Ok(())
+ } else {
+ Err(invalid("geometry has an invalid country code"))
+ }
+}
+
+fn push_u16(output: &mut Vec<u8>, value: u16) {
+ output.extend_from_slice(&value.to_le_bytes());
+}
+
+fn push_len(output: &mut Vec<u8>, value: usize) -> Result<()> {
+ output.extend_from_slice(
+ &u32::try_from(value)
+ .map_err(|_| invalid("country index exceeds 4 GiB"))?
+ .to_le_bytes(),
+ );
+ Ok(())
+}
diff --git a/rust/crates/location-dataset/src/dispute.rs b/rust/crates/location-dataset/src/dispute.rs
new file mode 100644
index 0000000000..decd885242
--- /dev/null
+++ b/rust/crates/location-dataset/src/dispute.rs
@@ -0,0 +1,447 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+
+use ente_location::TerritoryId;
+use shapefile::Reader;
+use shapefile::dbase::Record;
+
+use crate::catalog::{DISPUTED_AREAS, UKRAINIAN_REGIONS, WORLDVIEWS};
+use crate::country::{
+ Area, character_field, encode as encode_geometry, natural_earth_country_code, polygon_geometry,
+};
+use crate::{Result, invalid};
+
+const HEADER_LEN: usize = 64;
+const TERRITORY_LEN: usize = 12;
+const MISSING_COUNTRY: u8 = u8::MAX;
+
+#[derive(Clone)]
+struct SourceDispute {
+ territory: TerritoryId,
+ geometry_code: [u8; 2],
+ name: String,
+ source_default: Option<[u8; 2]>,
+ possible_countries: Vec<[u8; 2]>,
+ worldview_assignments: Vec<Option<[u8; 2]>>,
+}
+
+struct EncodedRecord {
+ territory: TerritoryId,
+ geometry_code: [u8; 2],
+ default_country: u8,
+ candidate_start: u16,
+ candidate_count: u8,
+ name_start: u16,
+ name_length: u8,
+}
+
+pub(crate) fn build(
+ countries_path: &Path,
+ disputes_path: &Path,
+ admin1_path: &Path,
+) -> Result<Vec<u8>> {
+ let country_codes = read_country_codes(countries_path)?;
+ let (mut metadata, mut areas) = read_disputed_areas(disputes_path, &country_codes)?;
+ let (ukraine_metadata, ukraine_areas) = read_ukrainian_regions(admin1_path)?;
+ metadata.extend(ukraine_metadata);
+ areas.extend(ukraine_areas);
+ if metadata.len() != DISPUTED_AREAS.len() + UKRAINIAN_REGIONS.len() {
+ return Err(invalid("Priority-1 dispute catalog is incomplete"));
+ }
+ let geometry = encode_geometry(&areas)?;
+ encode(&geometry, &metadata)
+}
+
+fn read_country_codes(path: &Path) -> Result<BTreeMap<String, [u8; 2]>> {
+ let mut reader = Reader::from_path(path)?;
+ let mut codes = BTreeMap::new();
+ for shape_record in reader.iter_shapes_and_records() {
+ let (_, record) = shape_record?;
+ let Some(code) = natural_earth_country_code(&record) else {
+ continue;
+ };
+ for field in [
+ "ADM0_A3",
+ "SOV_A3",
+ "ISO_A3_EH",
+ "ISO_A3",
+ "GU_A3",
+ "SU_A3",
+ "ADM0_TLC",
+ ] {
+ let Some(value) = character_field(&record, field) else {
+ continue;
+ };
+ if value.len() == 3 && value.bytes().all(|byte| byte.is_ascii_uppercase()) {
+ codes.entry(value.to_owned()).or_insert(code);
+ }
+ }
+ }
+ Ok(codes)
+}
+
+fn read_disputed_areas(
+ path: &Path,
+ country_codes: &BTreeMap<String, [u8; 2]>,
+) -> Result<(Vec<SourceDispute>, Vec<Area>)> {
+ let selections: BTreeMap<&str, _> = DISPUTED_AREAS
+ .iter()
+ .map(|source| (source.source_id, source))
+ .collect();
+ let mut seen = BTreeSet::new();
+ let mut metadata = Vec::new();
+ let mut areas = Vec::new();
+ let mut reader = Reader::from_path(path)?;
+ for shape_record in reader.iter_shapes_and_records() {
+ let (shape, record) = shape_record?;
+ let Some(source_id) = character_field(&record, "BRK_A3") else {
+ continue;
+ };
+ let Some(selection) = selections.get(source_id) else {
+ continue;
+ };
+ if !seen.insert(source_id.to_owned()) {
+ return Err(invalid(format!(
+ "duplicate disputed-area source {source_id}"
+ )));
+ }
+ let name = required_field(&record, "NAME_EN", source_id)?.to_owned();
+ let geometry = polygon_geometry(shape, source_id)?;
+ metadata.push(SourceDispute {
+ territory: selection.territory,
+ geometry_code: selection.geometry_code,
+ name,
+ source_default: assigned_country(&record, "ADM0_A3", country_codes),
+ possible_countries: selection.additional_countries.to_vec(),
+ worldview_assignments: WORLDVIEWS
+ .iter()
+ .map(|&(_, field)| assigned_country(&record, field, country_codes))
+ .collect(),
+ });
+ areas.push(Area {
+ code: selection.geometry_code,
+ geometry,
+ });
+ }
+ let missing: Vec<&str> = selections
+ .keys()
+ .copied()
+ .filter(|source| !seen.contains(*source))
+ .collect();
+ if !missing.is_empty() {
+ return Err(invalid(format!(
+ "disputed-area source is missing {missing:?}"
+ )));
+ }
+ Ok((metadata, areas))
+}
+
+fn read_ukrainian_regions(path: &Path) -> Result<(Vec<SourceDispute>, Vec<Area>)> {
+ let selections: BTreeMap<&str, _> = UKRAINIAN_REGIONS
+ .iter()
+ .map(|source| (source.iso_3166_2, source))
+ .collect();
+ let mut seen = BTreeSet::new();
+ let mut metadata = Vec::new();
+ let mut areas = Vec::new();
+ let mut reader = Reader::from_path(path)?;
+ for shape_record in reader.iter_shapes_and_records() {
+ let (shape, record) = shape_record?;
+ let Some(iso_code) = character_field(&record, "iso_3166_2") else {
+ continue;
+ };
+ let Some(selection) = selections.get(iso_code) else {
+ continue;
+ };
+ if !seen.insert(iso_code.to_owned()) {
+ return Err(invalid(format!("duplicate Admin-1 source {iso_code}")));
+ }
+ let name = required_field(&record, "name_en", iso_code)?;
+ let geometry = polygon_geometry(shape, iso_code)?;
+ metadata.push(SourceDispute {
+ territory: selection.territory,
+ geometry_code: selection.geometry_code,
+ name: format!("{name} Region"),
+ source_default: Some(*b"UA"),
+ possible_countries: vec![*b"RU", *b"UA"],
+ worldview_assignments: WORLDVIEWS
+ .iter()
+ .map(|&(viewpoint, _)| Some(if viewpoint == *b"RU" { *b"RU" } else { *b"UA" }))
+ .collect(),
+ });
+ areas.push(Area {
+ code: selection.geometry_code,
+ geometry,
+ });
+ }
+ let missing: Vec<&str> = selections
+ .keys()
+ .copied()
+ .filter(|source| !seen.contains(*source))
+ .collect();
+ if !missing.is_empty() {
+ return Err(invalid(format!("Admin-1 source is missing {missing:?}")));
+ }
+ Ok((metadata, areas))
+}
+
+fn encode(geometry: &[u8], source: &[SourceDispute]) -> Result<Vec<u8>> {
+ let mut disputes = source.to_vec();
+ disputes.sort_unstable_by_key(|dispute| dispute.territory);
+ validate_source(&disputes)?;
+ let countries = collect_countries(&disputes)?;
+ let country_indices: BTreeMap<[u8; 2], u8> = countries
+ .iter()
+ .copied()
+ .enumerate()
+ .map(|(index, code)| (code, index as u8))
+ .collect();
+
+ let mut candidates = Vec::new();
+ let mut assignments = Vec::with_capacity(disputes.len() * WORLDVIEWS.len());
+ let mut strings = Vec::new();
+ let mut records = Vec::with_capacity(disputes.len());
+ for dispute in &disputes {
+ let candidate_start = u16_length(candidates.len(), "too many dispute candidates")?;
+ let possible = complete_candidates(dispute);
+ let candidate_count = u8::try_from(possible.len())
+ .map_err(|_| invalid("too many candidates for one territory"))?;
+ candidates.extend(possible.into_iter().map(|code| country_indices[&code]));
+
+ let name_start = u16_length(strings.len(), "dispute strings exceed 64 KiB")?;
+ let name_length = u8::try_from(dispute.name.len())
+ .map_err(|_| invalid("territory name exceeds 255 bytes"))?;
+ strings.extend_from_slice(dispute.name.as_bytes());
+
+ records.push(EncodedRecord {
+ territory: dispute.territory,
+ geometry_code: dispute.geometry_code,
+ default_country: encode_optional(dispute.source_default, &country_indices),
+ candidate_start,
+ candidate_count,
+ name_start,
+ name_length,
+ });
+ assignments.extend(
+ dispute
+ .worldview_assignments
+ .iter()
+ .copied()
+ .map(|country| encode_optional(country, &country_indices)),
+ );
+ }
+ u16_length(strings.len(), "dispute strings exceed 64 KiB")?;
+
+ let countries_offset = HEADER_LEN;
+ let worldviews_offset = countries_offset + countries.len() * 2;
+ let territories_offset = worldviews_offset + WORLDVIEWS.len();
+ let candidates_offset = territories_offset + records.len() * TERRITORY_LEN;
+ let assignments_offset = candidates_offset + candidates.len();
+ let strings_offset = assignments_offset + assignments.len();
+ let geometry_offset = strings_offset + strings.len();
+ let file_length = geometry_offset
+ .checked_add(geometry.len())
+ .ok_or_else(|| invalid("dispute index size overflow"))?;
+ let mut output = Vec::with_capacity(file_length);
+
+ output.extend_from_slice(b"DSPT");
+ push_u16(&mut output, 1);
+ push_u16(&mut output, HEADER_LEN as u16);
+ push_u16(
+ &mut output,
+ u16_length(WORLDVIEWS.len(), "too many worldviews")?,
+ );
+ push_u16(
+ &mut output,
+ u16_length(records.len(), "too many territories")?,
+ );
+ push_u16(
+ &mut output,
+ u16_length(countries.len(), "too many countries")?,
+ );
+ push_u16(&mut output, TERRITORY_LEN as u16);
+ for value in [
+ countries_offset,
+ worldviews_offset,
+ territories_offset,
+ candidates_offset,
+ assignments_offset,
+ strings_offset,
+ geometry_offset,
+ file_length,
+ ] {
+ push_length(&mut output, value)?;
+ }
+ output.resize(HEADER_LEN, 0);
+ for country in &countries {
+ output.extend_from_slice(country);
+ }
+ for &(worldview, _) in WORLDVIEWS {
+ output.push(country_indices[&worldview]);
+ }
+ for record in records {
+ push_u16(&mut output, record.territory.get());
+ output.extend_from_slice(&record.geometry_code);
+ output.push(record.default_country);
+ output.push(record.candidate_count);
+ push_u16(&mut output, record.candidate_start);
+ push_u16(&mut output, record.name_start);
+ output.push(record.name_length);
+ output.push(0);
+ }
+ output.extend_from_slice(&candidates);
+ output.extend_from_slice(&assignments);
+ output.extend_from_slice(&strings);
+ output.extend_from_slice(geometry);
+ debug_assert_eq!(output.len(), file_length);
+ Ok(output)
+}
+
+fn validate_source(disputes: &[SourceDispute]) -> Result<()> {
+ let mut ids = BTreeSet::new();
+ let mut geometry_codes = BTreeSet::new();
+ for dispute in disputes {
+ if !ids.insert(dispute.territory)
+ || !geometry_codes.insert(dispute.geometry_code)
+ || dispute.name.is_empty()
+ || dispute.worldview_assignments.len() != WORLDVIEWS.len()
+ {
+ return Err(invalid("invalid or duplicate dispute metadata"));
+ }
+ }
+ Ok(())
+}
+
+fn collect_countries(disputes: &[SourceDispute]) -> Result<Vec<[u8; 2]>> {
+ let mut countries: BTreeSet<[u8; 2]> = WORLDVIEWS.iter().map(|&(code, _)| code).collect();
+ for dispute in disputes {
+ countries.extend(complete_candidates(dispute));
+ }
+ if countries.len() >= usize::from(MISSING_COUNTRY) {
+ return Err(invalid("dispute country table exceeds one-byte indices"));
+ }
+ if countries
+ .iter()
+ .flatten()
+ .any(|byte| !byte.is_ascii_uppercase())
+ {
+ return Err(invalid("dispute catalog contains an invalid country code"));
+ }
+ Ok(countries.into_iter().collect())
+}
+
+fn complete_candidates(dispute: &SourceDispute) -> BTreeSet<[u8; 2]> {
+ let mut countries: BTreeSet<[u8; 2]> = dispute.possible_countries.iter().copied().collect();
+ countries.extend(dispute.source_default);
+ countries.extend(dispute.worldview_assignments.iter().flatten().copied());
+ countries
+}
+
+fn assigned_country(
+ record: &Record,
+ field: &str,
+ country_codes: &BTreeMap<String, [u8; 2]>,
+) -> Option<[u8; 2]> {
+ character_field(record, field).and_then(|code| country_codes.get(code).copied())
+}
+
+fn required_field<'a>(record: &'a Record, field: &str, source: &str) -> Result<&'a str> {
+ character_field(record, field)
+ .ok_or_else(|| invalid(format!("source {source} is missing {field}")))
+}
+
+fn encode_optional(country: Option<[u8; 2]>, country_indices: &BTreeMap<[u8; 2], u8>) -> u8 {
+ country.map_or(MISSING_COUNTRY, |code| country_indices[&code])
+}
+
+fn u16_length(value: usize, message: &'static str) -> Result<u16> {
+ u16::try_from(value).map_err(|_| invalid(message))
+}
+
+fn push_u16(output: &mut Vec<u8>, value: u16) {
+ output.extend_from_slice(&value.to_le_bytes());
+}
+
+fn push_length(output: &mut Vec<u8>, value: usize) -> Result<()> {
+ output.extend_from_slice(
+ &u32::try_from(value)
+ .map_err(|_| invalid("dispute index exceeds 4 GiB"))?
+ .to_le_bytes(),
+ );
+ Ok(())
+}
+
+#[cfg(test)]
+pub(crate) fn test_bytes() -> Vec<u8> {
+ use geo::{LineString, MultiPolygon, Polygon};
+
+ let geometry = encode_geometry(&[Area {
+ code: *b"AC",
+ geometry: MultiPolygon(vec![Polygon::new(
+ LineString::from(vec![
+ (79.0, 35.0),
+ (80.0, 35.0),
+ (80.0, 36.0),
+ (79.0, 36.0),
+ (79.0, 35.0),
+ ]),
+ vec![],
+ )]),
+ }])
+ .unwrap();
+ encode(
+ &geometry,
+ &[SourceDispute {
+ territory: TerritoryId::AKSAI_CHIN,
+ geometry_code: *b"AC",
+ name: "Aksai Chin".to_owned(),
+ source_default: Some(*b"CN"),
+ possible_countries: vec![*b"CN", *b"IN"],
+ worldview_assignments: WORLDVIEWS
+ .iter()
+ .map(|&(region, _)| Some(if region == *b"IN" { *b"IN" } else { *b"CN" }))
+ .collect(),
+ }],
+ )
+ .unwrap()
+}
+
+#[cfg(test)]
+mod tests {
+ use ente_location::{Coordinate, CountryCode, CountryIndex, CountryView, TerritoryId};
+ use geo::{LineString, MultiPolygon, Polygon};
+
+ use crate::country::{Area, encode as encode_geometry};
+
+ use super::test_bytes;
+
+ #[test]
+ fn builds_and_resolves_a_dispute_overlay() {
+ let countries = encode_geometry(&[Area {
+ code: *b"AA",
+ geometry: MultiPolygon(vec![Polygon::new(
+ LineString::from(vec![
+ (79.0, 35.0),
+ (80.0, 35.0),
+ (80.0, 36.0),
+ (79.0, 36.0),
+ (79.0, 35.0),
+ ]),
+ vec![],
+ )]),
+ }])
+ .unwrap();
+ let index = CountryIndex::from_bytes(countries, test_bytes()).unwrap();
+ let disputed = index.lookup(Coordinate::new(35.5, 79.5)).unwrap().disputes[0];
+ let india = CountryCode::from_bytes(*b"IN").unwrap();
+
+ assert_eq!(disputed.territory_id(), TerritoryId::AKSAI_CHIN);
+ assert_eq!(disputed.name(), "Aksai Chin");
+ assert_eq!(disputed.resolve(CountryView::Neutral), None);
+ assert_eq!(
+ disputed.resolve(CountryView::SourceDefault),
+ Some(CountryCode::from_bytes(*b"CN").unwrap())
+ );
+ assert_eq!(disputed.resolve(CountryView::Region(india)), Some(india));
+ }
+}
diff --git a/rust/crates/location-dataset/src/lib.rs b/rust/crates/location-dataset/src/lib.rs
index 2b660a2ffe..417e669103 100644
--- a/rust/crates/location-dataset/src/lib.rs
+++ b/rust/crates/location-dataset/src/lib.rs
@@ -1,17 +1,22 @@
use std::fmt::Write as _;
use std::path::PathBuf;
-use ente_location::CityIndex;
+use ente_location::{CityIndex, CountryIndex};
use sha2::{Digest, Sha256};
use thiserror::Error;
+mod catalog;
mod city;
+mod country;
+mod dispute;
mod download;
mod sources;
pub use sources::{RemoteSources, SourcePaths};
pub const CITY_FILE: &str = "cities.bin";
+pub const COUNTRY_FILE: &str = "countries.bin";
+pub const DISPUTE_FILE: &str = "disputes.bin";
pub type Result<T> = std::result::Result<T, Error>;
@@ -23,6 +28,8 @@ pub enum Error {
Http(#[from] reqwest::Error),
#[error("ZIP archive is invalid: {0}")]
Zip(#[from] zip::result::ZipError),
+ #[error("shapefile is invalid: {0}")]
+ Shapefile(#[from] shapefile::Error),
#[error("invalid location data: {0}")]
InvalidData(String),
#[error(transparent)]
@@ -36,23 +43,57 @@ pub struct BuildOptions {
pub struct BuildOutput {
pub city_count: usize,
+ pub territory_count: usize,
+ pub byte_length: usize,
+ pub files: [BuildFile; 3],
+}
+
+pub struct BuildFile {
+ pub name: &'static str,
pub byte_length: usize,
pub sha256: String,
}
pub fn build(options: &BuildOptions) -> Result<BuildOutput> {
- let bytes = city::build(&options.sources.cities, &options.sources.country_info)?;
- let city_count = CityIndex::from_bytes(bytes.as_slice())?.len();
+ let cities = city::build(&options.sources.cities, &options.sources.country_info)?;
+ let countries = country::build_countries(&options.sources.countries)?;
+ let disputes = dispute::build(
+ &options.sources.countries,
+ &options.sources.disputes,
+ &options.sources.admin1,
+ )?;
+ let city_count = CityIndex::from_bytes(cities.as_slice())?.len();
+ CountryIndex::from_bytes(countries.as_slice(), disputes.as_slice())?;
+ let files = [
+ build_file(CITY_FILE, &cities),
+ build_file(COUNTRY_FILE, &countries),
+ build_file(DISPUTE_FILE, &disputes),
+ ];
let output = BuildOutput {
city_count,
- byte_length: bytes.len(),
- sha256: hex(Sha256::digest(&bytes)),
+ territory_count: catalog::DISPUTED_AREAS.len() + catalog::UKRAINIAN_REGIONS.len(),
+ byte_length: files.iter().map(|file| file.byte_length).sum(),
+ files,
};
std::fs::create_dir_all(&options.output)?;
- std::fs::write(options.output.join(CITY_FILE), bytes)?;
+ for (name, bytes) in [
+ (CITY_FILE, cities.as_slice()),
+ (COUNTRY_FILE, countries.as_slice()),
+ (DISPUTE_FILE, disputes.as_slice()),
+ ] {
+ std::fs::write(options.output.join(name), bytes)?;
+ }
Ok(output)
}
+fn build_file(name: &'static str, bytes: &[u8]) -> BuildFile {
+ BuildFile {
+ name,
+ byte_length: bytes.len(),
+ sha256: hex(Sha256::digest(bytes)),
+ }
+}
+
fn invalid(message: impl Into<String>) -> Error {
Error::InvalidData(message.into())
}
diff --git a/rust/crates/location-dataset/src/main.rs b/rust/crates/location-dataset/src/main.rs
index 4ed8dad10b..1981fbe424 100644
--- a/rust/crates/location-dataset/src/main.rs
+++ b/rust/crates/location-dataset/src/main.rs
@@ -2,11 +2,12 @@ use std::env;
use std::error::Error;
use std::path::PathBuf;
-use ente_location_dataset::{BuildOptions, CITY_FILE, RemoteSources, SourcePaths, build};
+use ente_location_dataset::{BuildOptions, RemoteSources, SourcePaths, build};
const USAGE: &str = concat!(
"usage: ente-location-dataset build --output <directory> [--cache <directory>] ",
- "[--cities <file> --country-info <file>]"
+ "[--cities <file> --country-info <file> ",
+ "--countries <shp> --disputes <shp> --admin1 <shp>]"
);
fn main() {
@@ -22,21 +23,33 @@ fn run() -> Result<(), Box<dyn Error>> {
cache,
cities,
country_info,
+ countries,
+ disputes,
+ admin1,
} = Arguments::parse()?;
- let sources = match (cities, country_info) {
- (None, None) => RemoteSources::new(cache).fetch()?,
- (Some(cities), Some(country_info)) => SourcePaths {
- cities,
- country_info,
- },
- _ => return Err("local generation requires both source paths".into()),
+ let sources = match (cities, country_info, countries, disputes, admin1) {
+ (None, None, None, None, None) => RemoteSources::new(cache).fetch()?,
+ (Some(cities), Some(country_info), Some(countries), Some(disputes), Some(admin1)) => {
+ SourcePaths {
+ cities,
+ country_info,
+ countries,
+ disputes,
+ admin1,
+ }
+ }
+ _ => return Err("local generation requires all five source paths".into()),
};
let result = build(&BuildOptions { sources, output })?;
println!("cities: {}", result.city_count);
- println!(
- "{CITY_FILE}: {} bytes, sha256 {}",
- result.byte_length, result.sha256
- );
+ println!("Priority-1 territories: {}", result.territory_count);
+ println!("bytes: {}", result.byte_length);
+ for file in result.files {
+ println!(
+ "{}: {} bytes, sha256 {}",
+ file.name, file.byte_length, file.sha256
+ );
+ }
Ok(())
}
@@ -45,6 +58,9 @@ struct Arguments {
cache: PathBuf,
cities: Option<PathBuf>,
country_info: Option<PathBuf>,
+ countries: Option<PathBuf>,
+ disputes: Option<PathBuf>,
+ admin1: Option<PathBuf>,
}
impl Arguments {
@@ -57,6 +73,9 @@ impl Arguments {
let mut cache = None;
let mut cities = None;
let mut country_info = None;
+ let mut countries = None;
+ let mut disputes = None;
+ let mut admin1 = None;
while let Some(argument) = args.next() {
let value = args.next().ok_or(USAGE)?;
match argument.as_str() {
@@ -64,6 +83,9 @@ impl Arguments {
"--cache" => cache = Some(value.into()),
"--cities" => cities = Some(value.into()),
"--country-info" => country_info = Some(value.into()),
+ "--countries" => countries = Some(value.into()),
+ "--disputes" => disputes = Some(value.into()),
+ "--admin1" => admin1 = Some(value.into()),
_ => return Err(USAGE.into()),
}
}
@@ -72,6 +94,9 @@ impl Arguments {
cache: cache.unwrap_or_else(default_cache),
cities,
country_info,
+ countries,
+ disputes,
+ admin1,
})
}
}
diff --git a/rust/crates/location-dataset/src/sources.rs b/rust/crates/location-dataset/src/sources.rs
index 75b0ef90da..828f1ace49 100644
--- a/rust/crates/location-dataset/src/sources.rs
+++ b/rust/crates/location-dataset/src/sources.rs
@@ -7,11 +7,23 @@ use crate::download::{download, extract};
const CITIES_URL: &str = "https://download.geonames.org/export/dump/cities5000.zip";
const COUNTRY_INFO_URL: &str = "https://download.geonames.org/export/dump/countryInfo.txt";
+const COUNTRIES: &str = "ne_10m_admin_0_countries";
+const COUNTRIES_URL: &str =
+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_countries.zip";
+const DISPUTES: &str = "ne_10m_admin_0_disputed_areas";
+const DISPUTES_URL: &str =
+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_disputed_areas.zip";
+const ADMIN1: &str = "ne_10m_admin_1_states_provinces";
+const ADMIN1_URL: &str =
+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_1_states_provinces.zip";
#[derive(Clone, Debug)]
pub struct SourcePaths {
pub cities: PathBuf,
pub country_info: PathBuf,
+ pub countries: PathBuf,
+ pub disputes: PathBuf,
+ pub admin1: PathBuf,
}
#[derive(Clone, Debug)]
@@ -37,9 +49,25 @@ impl RemoteSources {
download(&client, CITIES_URL, &cities_archive)?;
download(&client, COUNTRY_INFO_URL, &country_info)?;
extract(&cities_archive, &cities, "cities5000.txt")?;
+ let countries = self.fetch_shapefile(&client, COUNTRIES, COUNTRIES_URL)?;
+ let disputes = self.fetch_shapefile(&client, DISPUTES, DISPUTES_URL)?;
+ let admin1 = self.fetch_shapefile(&client, ADMIN1, ADMIN1_URL)?;
Ok(SourcePaths {
cities,
country_info,
+ countries,
+ disputes,
+ admin1,
})
}
+
+ fn fetch_shapefile(&self, client: &Client, name: &str, url: &str) -> Result<PathBuf> {
+ let archive = self.cache.join(format!("{name}.zip"));
+ download(client, url, &archive)?;
+ for extension in ["shp", "shx", "dbf"] {
+ let file = format!("{name}.{extension}");
+ extract(&archive, &self.cache.join(&file), &file)?;
+ }
+ Ok(self.cache.join(format!("{name}.shp")))
+ }
}
diff --git a/rust/crates/location/src/country.rs b/rust/crates/location/src/country.rs
new file mode 100644
index 0000000000..9f2492f1b1
--- /dev/null
+++ b/rust/crates/location/src/country.rs
@@ -0,0 +1,431 @@
+use crate::binary::{array, range, u16_at, u32_at};
+use crate::{CountryCode, Error};
+
+const MAGIC: &[u8; 4] = b"CTRY";
+const VERSION: u16 = 1;
+const HEADER_LEN: usize = 48;
+const BLOCK_SIZE: usize = 64;
+const BLOCK_LEN: usize = 12;
+const SECTION: &str = "country index";
+
+#[derive(Clone, Copy, Debug)]
+struct Layout {
+ columns: usize,
+ rows: usize,
+ country_count: usize,
+ cell_count: usize,
+ block_count: usize,
+ nonempty_count: usize,
+ countries: usize,
+ blocks: usize,
+ cell_offsets: usize,
+ cells: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct CountryGeometry {
+ bytes: Box<[u8]>,
+ layout: Layout,
+}
+
+impl CountryGeometry {
+ pub(crate) fn from_bytes(bytes: impl Into<Box<[u8]>>) -> crate::Result<Self> {
+ let bytes = bytes.into();
+ let layout = validate(&bytes)?;
+ Ok(Self { bytes, layout })
+ }
+
+ pub(crate) const fn country_count(&self) -> usize {
+ self.layout.country_count
+ }
+
+ pub(crate) fn country_codes(&self) -> impl ExactSizeIterator<Item = CountryCode> + '_ {
+ (0..self.layout.country_count).map(|index| self.country_code(index as u8))
+ }
+
+ pub(crate) const fn columns(&self) -> usize {
+ self.layout.columns
+ }
+
+ pub(crate) const fn rows(&self) -> usize {
+ self.layout.rows
+ }
+
+ pub(crate) fn prepare_cell(
+ &self,
+ latitude: f64,
+ longitude: f64,
+ ) -> crate::Result<PreparedCell> {
+ if !latitude.is_finite()
+ || !longitude.is_finite()
+ || !(-90.0..=90.0).contains(&latitude)
+ || !(-180.0..=180.0).contains(&longitude)
+ {
+ return Err(Error::InvalidCoordinate);
+ }
+
+ let longitude = normalize(longitude, -180.0, 360.0);
+ let cell_x = (((longitude + 180.0) * self.layout.columns as f64 / 360.0).floor() as usize)
+ .min(self.layout.columns - 1);
+ let cell_y = ((self.layout.rows as f64 * (90.0 - latitude) / 180.0).ceil() as usize)
+ .saturating_sub(1);
+ Ok(PreparedCell {
+ latitude,
+ longitude,
+ x: cell_x,
+ y: cell_y,
+ index: cell_y * self.layout.columns + cell_x,
+ })
+ }
+
+ pub(crate) fn lookup_prepared(
+ &self,
+ location: PreparedCell,
+ ) -> crate::Result<Vec<CountryCode>> {
+ let Some(dense_index) = self.dense_cell_index(location.index) else {
+ return Ok(Vec::new());
+ };
+ let point = Point {
+ x: {
+ let cell_longitude =
+ -180.0 + 360.0 * location.x as f64 / self.layout.columns as f64;
+ ((location.longitude - cell_longitude)
+ * self.layout.columns as f64
+ * f64::from(u16::MAX)
+ / 360.0) as u16
+ },
+ y: {
+ let cell_latitude =
+ 90.0 - 180.0 * (location.y + 1) as f64 / self.layout.rows as f64;
+ ((location.latitude - cell_latitude)
+ * self.layout.rows as f64
+ * f64::from(u16::MAX)
+ / 180.0) as u16
+ },
+ };
+ let start = self.cell_offset(dense_index);
+ let end = self.cell_offset(dense_index + 1);
+ self.lookup_cell(
+ &self.bytes[self.layout.cells + start..self.layout.cells + end],
+ point,
+ )
+ }
+
+ fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {
+ let block_index = cell_index / BLOCK_SIZE;
+ let bit_index = cell_index % BLOCK_SIZE;
+ let offset = self.layout.blocks + block_index * BLOCK_LEN;
+ let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));
+ let bit = 1_u64 << bit_index;
+ if mask & bit == 0 {
+ return None;
+ }
+ let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;
+ Some(first + (mask & bit.wrapping_sub(1)).count_ones() as usize)
+ }
+
+ fn cell_offset(&self, dense_index: usize) -> usize {
+ u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)
+ .expect("validated cell offset") as usize
+ }
+
+ fn country_code(&self, index: u8) -> CountryCode {
+ let offset = self.layout.countries + usize::from(index) * 2;
+ CountryCode::from_validated([self.bytes[offset], self.bytes[offset + 1]])
+ }
+
+ fn lookup_cell(&self, bytes: &[u8], point: Point) -> crate::Result<Vec<CountryCode>> {
+ let mut reader = Reader::new(bytes);
+ let mut matches = Vec::new();
+ for _ in 0..reader.byte()? {
+ push_unique(&mut matches, self.country_code(reader.byte()?));
+ }
+ for _ in 0..reader.byte()? {
+ let country = self.country_code(reader.byte()?);
+ let mut winding = 0_i32;
+ for _ in 0..reader.byte()? {
+ if reader.ring_contains(point)? {
+ winding += 1;
+ }
+ }
+ for _ in 0..reader.byte()? {
+ if reader.ring_contains(point)? {
+ winding -= 1;
+ }
+ }
+ if winding > 0 {
+ push_unique(&mut matches, country);
+ }
+ }
+ debug_assert_eq!(reader.position, bytes.len());
+ Ok(matches)
+ }
+}
+
+#[derive(Clone, Copy)]
+pub(crate) struct PreparedCell {
+ latitude: f64,
+ longitude: f64,
+ x: usize,
+ y: usize,
+ index: usize,
+}
+
+#[derive(Clone, Copy)]
+struct Point {
+ x: u16,
+ y: u16,
+}
+
+struct Reader<'a> {
+ bytes: &'a [u8],
+ position: usize,
+}
+
+impl<'a> Reader<'a> {
+ const fn new(bytes: &'a [u8]) -> Self {
+ Self { bytes, position: 0 }
+ }
+
+ fn byte(&mut self) -> crate::Result<u8> {
+ let value = *self.bytes.get(self.position).ok_or(invalid("truncated"))?;
+ self.position += 1;
+ Ok(value)
+ }
+
+ fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {
+ let end = self
+ .position
+ .checked_add(count)
+ .ok_or(invalid("truncated"))?;
+ let bytes = self
+ .bytes
+ .get(self.position..end)
+ .ok_or(invalid("truncated"))?;
+ self.position = end;
+ Ok(bytes)
+ }
+
+ fn ring_contains(&mut self, point: Point) -> crate::Result<bool> {
+ let count = usize::from(u16::from_le_bytes(
+ self.take(2)?.try_into().expect("two-byte slice"),
+ ));
+ if count == 0 {
+ return Ok(false);
+ }
+ let ring = self.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;
+ let mut winding = 0_i32;
+ let mut previous = point_at(ring, count - 1);
+ for index in 0..count {
+ let current = point_at(ring, index);
+ if previous.y <= point.y {
+ if current.y > point.y && is_left(previous, current, point) > 0 {
+ winding += 1;
+ }
+ } else if current.y <= point.y && is_left(previous, current, point) < 0 {
+ winding -= 1;
+ }
+ previous = current;
+ }
+ Ok(winding != 0)
+ }
+}
+
+fn validate(bytes: &[u8]) -> crate::Result<Layout> {
+ if bytes.len() < HEADER_LEN {
+ return Err(invalid("truncated"));
+ }
+ if &bytes[..4] != MAGIC {
+ return Err(invalid("invalid magic"));
+ }
+ let version = u16_at(bytes, 4).ok_or(invalid("truncated"))?;
+ if version != VERSION {
+ return Err(Error::invalid(
+ SECTION,
+ format!("unsupported format version {version}"),
+ ));
+ }
+ if usize::from(u16_at(bytes, 6).ok_or(invalid("truncated"))?) != HEADER_LEN
+ || usize::from(u16_at(bytes, 14).ok_or(invalid("truncated"))?) != BLOCK_SIZE
+ {
+ return Err(invalid("unexpected header or block length"));
+ }
+ let layout = Layout {
+ columns: usize::from(u16_at(bytes, 8).ok_or(invalid("truncated"))?),
+ rows: usize::from(u16_at(bytes, 10).ok_or(invalid("truncated"))?),
+ country_count: usize::from(u16_at(bytes, 12).ok_or(invalid("truncated"))?),
+ cell_count: read_usize(bytes, 16)?,
+ block_count: read_usize(bytes, 20)?,
+ nonempty_count: read_usize(bytes, 24)?,
+ countries: read_usize(bytes, 28)?,
+ blocks: read_usize(bytes, 32)?,
+ cell_offsets: read_usize(bytes, 36)?,
+ cells: read_usize(bytes, 40)?,
+ };
+ let declared_length = read_usize(bytes, 44)?;
+ validate_layout(bytes, layout, declared_length)?;
+ Ok(layout)
+}
+
+fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {
+ if layout.columns == 0
+ || layout.rows == 0
+ || layout.country_count == 0
+ || layout.country_count > 256
+ {
+ return Err(invalid("invalid grid or country count"));
+ }
+ if layout.cell_count
+ != layout
+ .columns
+ .checked_mul(layout.rows)
+ .ok_or(invalid("cell count overflow"))?
+ || layout.block_count != layout.cell_count.div_ceil(BLOCK_SIZE)
+ || layout.nonempty_count > layout.cell_count
+ {
+ return Err(invalid("invalid grid counts"));
+ }
+
+ let countries = range(layout.countries, layout.country_count, 2)
+ .ok_or(invalid("country section overflow"))?;
+ let blocks = range(layout.blocks, layout.block_count, BLOCK_LEN)
+ .ok_or(invalid("block section overflow"))?;
+ let offsets = range(
+ layout.cell_offsets,
+ layout
+ .nonempty_count
+ .checked_add(1)
+ .ok_or(invalid("cell offset overflow"))?,
+ 4,
+ )
+ .ok_or(invalid("cell offset overflow"))?;
+ if layout.countries != HEADER_LEN
+ || layout.blocks != countries.end
+ || layout.cell_offsets != blocks.end
+ || layout.cells != offsets.end
+ || layout.cells > bytes.len()
+ || declared_length != bytes.len()
+ {
+ return Err(invalid("sections are not contiguous"));
+ }
+
+ let mut seen_codes = [false; 26 * 26];
+ for index in 0..layout.country_count {
+ let offset = layout.countries + index * 2;
+ let code = [bytes[offset], bytes[offset + 1]];
+ if !code.iter().all(u8::is_ascii_uppercase) {
+ return Err(invalid("invalid country code"));
+ }
+ let slot = usize::from(code[0] - b'A') * 26 + usize::from(code[1] - b'A');
+ if seen_codes[slot] {
+ return Err(invalid("duplicate country code"));
+ }
+ seen_codes[slot] = true;
+ }
+
+ let mut expected_dense = 0;
+ for block in 0..layout.block_count {
+ let offset = layout.blocks + block * BLOCK_LEN;
+ let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));
+ let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;
+ if first != expected_dense {
+ return Err(invalid("invalid block dense offset"));
+ }
+ if block + 1 == layout.block_count {
+ let used_bits = layout.cell_count - block * BLOCK_SIZE;
+ if used_bits < BLOCK_SIZE && mask >> used_bits != 0 {
+ return Err(invalid("last block references nonexistent cells"));
+ }
+ }
+ expected_dense += mask.count_ones() as usize;
+ }
+ if expected_dense != layout.nonempty_count {
+ return Err(invalid("wrong nonempty cell count"));
+ }
+
+ let cells_length = bytes.len() - layout.cells;
+ let mut previous = 0;
+ for index in 0..=layout.nonempty_count {
+ let offset = u32_at(bytes, layout.cell_offsets + index * 4).expect("validated offset range")
+ as usize;
+ if offset < previous || offset > cells_length {
+ return Err(invalid("invalid cell offset"));
+ }
+ if index > 0 {
+ validate_cell(
+ &bytes[layout.cells + previous..layout.cells + offset],
+ layout.country_count,
+ )?;
+ }
+ previous = offset;
+ }
+ if previous != cells_length {
+ return Err(invalid("last cell offset is not terminal"));
+ }
+ Ok(())
+}
+
+fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {
+ let mut reader = Reader::new(bytes);
+ for _ in 0..reader.byte()? {
+ validate_country(reader.byte()?, country_count)?;
+ }
+ for _ in 0..reader.byte()? {
+ validate_country(reader.byte()?, country_count)?;
+ skip_rings(&mut reader)?;
+ skip_rings(&mut reader)?;
+ }
+ if reader.position != bytes.len() {
+ return Err(invalid("cell has trailing bytes"));
+ }
+ Ok(())
+}
+
+fn skip_rings(reader: &mut Reader<'_>) -> crate::Result<()> {
+ for _ in 0..reader.byte()? {
+ let count = usize::from(u16::from_le_bytes(
+ reader.take(2)?.try_into().expect("two-byte slice"),
+ ));
+ reader.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;
+ }
+ Ok(())
+}
+
+fn validate_country(index: u8, country_count: usize) -> crate::Result<()> {
+ if usize::from(index) < country_count {
+ Ok(())
+ } else {
+ Err(invalid("cell references unknown country"))
+ }
+}
+
+fn point_at(ring: &[u8], index: usize) -> Point {
+ let offset = index * 4;
+ Point {
+ x: u16::from_le_bytes([ring[offset], ring[offset + 1]]),
+ y: u16::from_le_bytes([ring[offset + 2], ring[offset + 3]]),
+ }
+}
+
+fn is_left(start: Point, end: Point, point: Point) -> i64 {
+ (i64::from(end.x) - i64::from(start.x)) * (i64::from(point.y) - i64::from(start.y))
+ - (i64::from(point.x) - i64::from(start.x)) * (i64::from(end.y) - i64::from(start.y))
+}
+
+fn normalize(value: f64, minimum: f64, width: f64) -> f64 {
+ (value - minimum).rem_euclid(width) + minimum
+}
+
+fn push_unique(countries: &mut Vec<CountryCode>, country: CountryCode) {
+ if !countries.contains(&country) {
+ countries.push(country);
+ }
+}
+
+fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {
+ Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)
+}
+
+fn invalid(reason: &'static str) -> Error {
+ Error::invalid(SECTION, reason)
+}
diff --git a/rust/crates/location/src/dispute.rs b/rust/crates/location/src/dispute.rs
new file mode 100644
index 0000000000..c69bda9519
--- /dev/null
+++ b/rust/crates/location/src/dispute.rs
@@ -0,0 +1,449 @@
+use std::str;
+
+use crate::binary::{range, u16_at, u32_at};
+use crate::country::{CountryGeometry, PreparedCell};
+use crate::{CountryCode, Error};
+
+const MAGIC: &[u8; 4] = b"DSPT";
+const VERSION: u16 = 1;
+const HEADER_LEN: usize = 64;
+const TERRITORY_LEN: usize = 12;
+const MISSING_COUNTRY: u8 = u8::MAX;
+const SECTION: &str = "dispute index";
+
+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+pub struct TerritoryId(u16);
+
+impl TerritoryId {
+ pub const ARUNACHAL_PRADESH: Self = Self(1);
+ pub const TIRPANI_VALLEYS: Self = Self(2);
+ pub const BARA_HOTII_VALLEYS: Self = Self(3);
+ pub const DEMCHOK: Self = Self(4);
+ pub const SAMDU_VALLEYS: Self = Self(5);
+ pub const KASHMIR: Self = Self(6);
+ pub const TRANS_KARAKORAM_TRACT: Self = Self(7);
+ pub const AKSAI_CHIN: Self = Self(8);
+ pub const GILGIT_BALTISTAN: Self = Self(9);
+ pub const AZAD_KASHMIR: Self = Self(10);
+ pub const SIACHEN_GLACIER: Self = Self(11);
+ pub const GAZA_STRIP: Self = Self(12);
+ pub const WEST_BANK: Self = Self(13);
+ pub const GOLAN_HEIGHTS: Self = Self(14);
+ pub const SHEBAA_FARMS: Self = Self(15);
+ pub const EAST_JERUSALEM: Self = Self(16);
+ pub const LATRUN_FORT: Self = Self(17);
+ pub const JERUSALEM_NO_MANS_LAND: Self = Self(18);
+ pub const MOUNT_SCOPUS: Self = Self(19);
+ pub const TAIWAN: Self = Self(20);
+ pub const CRIMEA: Self = Self(21);
+ pub const DONETSK_REGION: Self = Self(22);
+ pub const LUHANSK_REGION: Self = Self(23);
+ pub const KHERSON_REGION: Self = Self(24);
+ pub const ZAPORIZHZHIA_REGION: Self = Self(25);
+ pub const KOSOVO: Self = Self(26);
+ pub const WESTERN_SAHARA_MOROCCAN_AREA: Self = Self(27);
+ pub const WESTERN_SAHARA_SELF_ADMINISTERED_AREA: Self = Self(28);
+ pub const NORTHERN_CYPRUS: Self = Self(29);
+ pub const CYPRUS_BUFFER_ZONE: Self = Self(30);
+ pub const FALKLAND_ISLANDS: Self = Self(31);
+
+ pub const fn get(self) -> u16 {
+ self.0
+ }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CountryView {
+ Neutral,
+ SourceDefault,
+ Region(CountryCode),
+}
+
+#[derive(Clone, Copy, Debug)]
+struct Layout {
+ worldview_count: usize,
+ territory_count: usize,
+ country_count: usize,
+ countries: usize,
+ worldviews: usize,
+ territories: usize,
+ candidates: usize,
+ assignments: usize,
+ strings: usize,
+ geometry: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct DisputeIndex {
+ bytes: Box<[u8]>,
+ layout: Layout,
+ geometry: CountryGeometry,
+}
+
+impl DisputeIndex {
+ pub(crate) fn from_bytes(bytes: impl Into<Box<[u8]>>) -> crate::Result<Self> {
+ let bytes = bytes.into();
+ let layout = validate(&bytes)?;
+ let geometry = CountryGeometry::from_bytes(bytes[layout.geometry..].to_vec())?;
+ validate_geometry(&geometry, &bytes, layout)?;
+ Ok(Self {
+ bytes,
+ layout,
+ geometry,
+ })
+ }
+
+ pub(crate) fn lookup_prepared(
+ &self,
+ location: PreparedCell,
+ ) -> crate::Result<Vec<DisputeMatch<'_>>> {
+ Ok(self
+ .geometry
+ .lookup_prepared(location)?
+ .into_iter()
+ .map(|code| DisputeMatch {
+ index: self,
+ territory: territory_index_for_code(&self.bytes, self.layout, code)
+ .expect("geometry was validated against the catalog"),
+ })
+ .collect())
+ }
+
+ fn country_code(&self, index: u8) -> CountryCode {
+ country_code(&self.bytes, self.layout, index)
+ }
+
+ fn worldview_index(&self, code: CountryCode) -> Option<usize> {
+ worldview_index(&self.bytes, self.layout, code)
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct DisputeMatch<'a> {
+ index: &'a DisputeIndex,
+ territory: usize,
+}
+
+impl<'a> DisputeMatch<'a> {
+ pub fn territory_id(self) -> TerritoryId {
+ TerritoryId(self.record().id)
+ }
+
+ pub fn name(self) -> &'a str {
+ let record = self.record();
+ self.string(record.name_start, usize::from(record.name_length))
+ }
+
+ fn source_default(self) -> Option<CountryCode> {
+ decode_optional_country(
+ &self.index.bytes,
+ self.index.layout,
+ self.record().default_country,
+ )
+ }
+
+ pub fn possible_countries(self) -> impl ExactSizeIterator<Item = CountryCode> + 'a {
+ let record = self.record();
+ let start = self.index.layout.candidates + usize::from(record.candidate_start);
+ self.index.bytes[start..start + usize::from(record.candidate_count)]
+ .iter()
+ .map(|&country| self.index.country_code(country))
+ }
+
+ pub fn resolve(self, view: CountryView) -> Option<CountryCode> {
+ match view {
+ CountryView::Neutral => None,
+ CountryView::SourceDefault => self.source_default(),
+ CountryView::Region(region) => {
+ if let Some(worldview) = self.index.worldview_index(region) {
+ let assignment = self.index.layout.assignments
+ + self.territory * self.index.layout.worldview_count
+ + worldview;
+ decode_optional_country(
+ &self.index.bytes,
+ self.index.layout,
+ self.index.bytes[assignment],
+ )
+ } else if self.possible_countries().any(|country| country == region) {
+ Some(region)
+ } else {
+ self.source_default()
+ }
+ }
+ }
+ }
+
+ fn record(self) -> TerritoryRecord {
+ let offset = self.record_offset();
+ TerritoryRecord {
+ id: u16_at(&self.index.bytes, offset).expect("validated territory"),
+ default_country: self.index.bytes[offset + 4],
+ candidate_count: self.index.bytes[offset + 5],
+ candidate_start: u16_at(&self.index.bytes, offset + 6).expect("validated territory"),
+ name_start: u16_at(&self.index.bytes, offset + 8).expect("validated territory"),
+ name_length: self.index.bytes[offset + 10],
+ }
+ }
+
+ fn record_offset(self) -> usize {
+ self.index.layout.territories + self.territory * TERRITORY_LEN
+ }
+
+ fn string(self, start: u16, length: usize) -> &'a str {
+ let start = self.index.layout.strings + usize::from(start);
+ str::from_utf8(&self.index.bytes[start..start + length]).expect("validated UTF-8")
+ }
+}
+
+#[derive(Clone, Copy)]
+struct TerritoryRecord {
+ id: u16,
+ default_country: u8,
+ candidate_count: u8,
+ candidate_start: u16,
+ name_start: u16,
+ name_length: u8,
+}
+
+fn validate(bytes: &[u8]) -> crate::Result<Layout> {
+ if bytes.len() < HEADER_LEN {
+ return Err(invalid("truncated"));
+ }
+ if &bytes[..4] != MAGIC {
+ return Err(invalid("invalid magic"));
+ }
+ let version = read_u16(bytes, 4)?;
+ if version != VERSION {
+ return Err(Error::invalid(
+ SECTION,
+ format!("unsupported format version {version}"),
+ ));
+ }
+ if usize::from(read_u16(bytes, 6)?) != HEADER_LEN
+ || usize::from(read_u16(bytes, 14)?) != TERRITORY_LEN
+ {
+ return Err(invalid("unexpected header or territory length"));
+ }
+ if bytes[48..HEADER_LEN].iter().any(|&byte| byte != 0) {
+ return Err(invalid("nonzero reserved header bytes"));
+ }
+ let layout = Layout {
+ worldview_count: usize::from(read_u16(bytes, 8)?),
+ territory_count: usize::from(read_u16(bytes, 10)?),
+ country_count: usize::from(read_u16(bytes, 12)?),
+ countries: read_usize(bytes, 16)?,
+ worldviews: read_usize(bytes, 20)?,
+ territories: read_usize(bytes, 24)?,
+ candidates: read_usize(bytes, 28)?,
+ assignments: read_usize(bytes, 32)?,
+ strings: read_usize(bytes, 36)?,
+ geometry: read_usize(bytes, 40)?,
+ };
+ validate_layout(bytes, layout, read_usize(bytes, 44)?)?;
+ Ok(layout)
+}
+
+fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {
+ if layout.worldview_count == 0
+ || layout.territory_count == 0
+ || layout.country_count == 0
+ || layout.country_count >= usize::from(MISSING_COUNTRY)
+ {
+ return Err(invalid("invalid catalog counts"));
+ }
+ let countries = range(layout.countries, layout.country_count, 2)
+ .ok_or(invalid("country table overflow"))?;
+ let worldviews = range(layout.worldviews, layout.worldview_count, 1)
+ .ok_or(invalid("worldview table overflow"))?;
+ let territories = range(layout.territories, layout.territory_count, TERRITORY_LEN)
+ .ok_or(invalid("territory table overflow"))?;
+ let assignment_count = layout
+ .territory_count
+ .checked_mul(layout.worldview_count)
+ .ok_or(invalid("assignment table overflow"))?;
+ let assignments = range(layout.assignments, assignment_count, 1)
+ .ok_or(invalid("assignment table overflow"))?;
+ if layout.countries != HEADER_LEN
+ || layout.worldviews != countries.end
+ || layout.territories != worldviews.end
+ || layout.candidates != territories.end
+ || layout.assignments < layout.candidates
+ || layout.strings != assignments.end
+ || layout.geometry < layout.strings
+ || layout.geometry >= bytes.len()
+ || declared_length != bytes.len()
+ {
+ return Err(invalid("sections are not contiguous"));
+ }
+
+ let mut seen_codes = [false; 26 * 26];
+ for index in 0..layout.country_count {
+ let code = code_at(bytes, layout.countries + index * 2)?;
+ let slot = usize::from(code[0] - b'A') * 26 + usize::from(code[1] - b'A');
+ if seen_codes[slot] {
+ return Err(invalid("duplicate catalog country code"));
+ }
+ seen_codes[slot] = true;
+ }
+
+ let mut seen_worldviews = [false; 255];
+ for &country in &bytes[layout.worldviews..worldviews.end] {
+ validate_country(country, layout.country_count)?;
+ if seen_worldviews[usize::from(country)] {
+ return Err(invalid("duplicate worldview"));
+ }
+ seen_worldviews[usize::from(country)] = true;
+ }
+
+ let candidate_length = layout.assignments - layout.candidates;
+ let string_length = layout.geometry - layout.strings;
+ let mut ids = Vec::with_capacity(layout.territory_count);
+ let mut geometry_codes = Vec::with_capacity(layout.territory_count);
+ for territory in 0..layout.territory_count {
+ let offset = layout.territories + territory * TERRITORY_LEN;
+ let id = read_u16_infallible(bytes, offset);
+ if id == 0 || ids.contains(&id) {
+ return Err(invalid("invalid or duplicate territory ID"));
+ }
+ ids.push(id);
+ let geometry_code = code_at(bytes, offset + 2)?;
+ if geometry_codes.contains(&geometry_code) {
+ return Err(invalid("duplicate territory geometry code"));
+ }
+ geometry_codes.push(geometry_code);
+ validate_optional_country(bytes[offset + 4], layout.country_count)?;
+
+ let candidate_start = usize::from(read_u16_infallible(bytes, offset + 6));
+ let candidate_end = candidate_start
+ .checked_add(usize::from(bytes[offset + 5]))
+ .ok_or(invalid("candidate range overflow"))?;
+ if candidate_end > candidate_length {
+ return Err(invalid("candidate range exceeds table"));
+ }
+ for &country in
+ &bytes[layout.candidates + candidate_start..layout.candidates + candidate_end]
+ {
+ validate_country(country, layout.country_count)?;
+ }
+ for &country in &bytes[layout.assignments + territory * layout.worldview_count
+ ..layout.assignments + (territory + 1) * layout.worldview_count]
+ {
+ validate_optional_country(country, layout.country_count)?;
+ }
+
+ let name_start = usize::from(read_u16_infallible(bytes, offset + 8));
+ let name_length = usize::from(bytes[offset + 10]);
+ if name_length == 0 {
+ return Err(invalid("empty territory name"));
+ }
+ validate_string(
+ bytes,
+ layout.strings,
+ string_length,
+ name_start,
+ name_length,
+ )?;
+ if bytes[offset + 11] != 0 {
+ return Err(invalid("nonzero reserved territory byte"));
+ }
+ }
+ Ok(())
+}
+
+fn validate_geometry(
+ geometry: &CountryGeometry,
+ bytes: &[u8],
+ layout: Layout,
+) -> crate::Result<()> {
+ if geometry.columns() != 360
+ || geometry.rows() != 180
+ || geometry.country_count() != layout.territory_count
+ || geometry
+ .country_codes()
+ .any(|code| territory_index_for_code(bytes, layout, code).is_none())
+ {
+ return Err(invalid("geometry does not match the territory catalog"));
+ }
+ Ok(())
+}
+
+fn validate_string(
+ bytes: &[u8],
+ strings: usize,
+ string_length: usize,
+ start: usize,
+ length: usize,
+) -> crate::Result<()> {
+ let end = start
+ .checked_add(length)
+ .ok_or(invalid("string range overflow"))?;
+ if end > string_length {
+ return Err(invalid("string exceeds string table"));
+ }
+ str::from_utf8(&bytes[strings + start..strings + end]).map_err(|_| invalid("invalid UTF-8"))?;
+ Ok(())
+}
+
+fn territory_index_for_code(bytes: &[u8], layout: Layout, code: CountryCode) -> Option<usize> {
+ (0..layout.territory_count).find(|&index| {
+ let offset = layout.territories + index * TERRITORY_LEN + 2;
+ bytes[offset..offset + 2] == code.as_bytes()
+ })
+}
+
+fn worldview_index(bytes: &[u8], layout: Layout, code: CountryCode) -> Option<usize> {
+ (0..layout.worldview_count).find(|&index| {
+ let country = bytes[layout.worldviews + index];
+ country_code(bytes, layout, country) == code
+ })
+}
+
+fn country_code(bytes: &[u8], layout: Layout, index: u8) -> CountryCode {
+ let offset = layout.countries + usize::from(index) * 2;
+ CountryCode::from_validated([bytes[offset], bytes[offset + 1]])
+}
+
+fn decode_optional_country(bytes: &[u8], layout: Layout, index: u8) -> Option<CountryCode> {
+ (index != MISSING_COUNTRY).then(|| country_code(bytes, layout, index))
+}
+
+fn validate_country(index: u8, count: usize) -> crate::Result<()> {
+ if usize::from(index) < count {
+ Ok(())
+ } else {
+ Err(invalid("catalog references unknown country"))
+ }
+}
+
+fn validate_optional_country(index: u8, count: usize) -> crate::Result<()> {
+ if index == MISSING_COUNTRY {
+ Ok(())
+ } else {
+ validate_country(index, count)
+ }
+}
+
+fn code_at(bytes: &[u8], offset: usize) -> crate::Result<[u8; 2]> {
+ let code = [bytes[offset], bytes[offset + 1]];
+ if code.iter().all(u8::is_ascii_uppercase) {
+ Ok(code)
+ } else {
+ Err(invalid("invalid country code"))
+ }
+}
+
+fn read_u16(bytes: &[u8], offset: usize) -> crate::Result<u16> {
+ u16_at(bytes, offset).ok_or(invalid("truncated"))
+}
+
+fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {
+ Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)
+}
+
+fn read_u16_infallible(bytes: &[u8], offset: usize) -> u16 {
+ u16_at(bytes, offset).expect("validated territory field")
+}
+
+fn invalid(reason: &'static str) -> Error {
+ Error::invalid(SECTION, reason)
+}
diff --git a/rust/crates/location/src/error.rs b/rust/crates/location/src/error.rs
index 91f65d9be2..479df1e66c 100644
--- a/rust/crates/location/src/error.rs
+++ b/rust/crates/location/src/error.rs
@@ -13,6 +13,8 @@ pub enum Error {
section: &'static str,
reason: Cow<'static, str>,
},
+ #[error("coordinate is outside the valid latitude/longitude range")]
+ InvalidCoordinate,
#[error("map point {0} has an invalid coordinate")]
InvalidMapPoint(usize),
#[error("minimum marker distance must be finite and positive")]
diff --git a/rust/crates/location/src/lib.rs b/rust/crates/location/src/lib.rs
index 412c4a1205..1fa2202830 100644
--- a/rust/crates/location/src/lib.rs
+++ b/rust/crates/location/src/lib.rs
@@ -1,14 +1,23 @@
+use std::collections::BTreeMap;
use std::fmt;
+use std::fs;
+use std::path::Path;
use std::str::FromStr;
mod binary;
mod city;
pub mod cluster;
+mod country;
+mod dispute;
mod error;
pub use city::{City, CityIndex, CityMatch};
+pub use dispute::{CountryView, DisputeMatch, TerritoryId};
pub use error::{Error, Result};
+use country::CountryGeometry;
+use dispute::DisputeIndex;
+
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Coordinate {
pub latitude: f64,
@@ -77,3 +86,126 @@ impl FromStr for CountryCode {
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
#[error("country codes must contain two uppercase ASCII letters")]
pub struct InvalidCountryCode;
+
+pub struct CountryIndex {
+ countries: CountryGeometry,
+ disputes: DisputeIndex,
+}
+
+impl CountryIndex {
+ pub fn from_paths(countries: impl AsRef<Path>, disputes: impl AsRef<Path>) -> Result<Self> {
+ Self::from_bytes(fs::read(countries)?, fs::read(disputes)?)
+ }
+
+ pub fn from_bytes(
+ countries: impl Into<Box<[u8]>>,
+ disputes: impl Into<Box<[u8]>>,
+ ) -> Result<Self> {
+ let countries = CountryGeometry::from_bytes(countries)?;
+ let disputes = DisputeIndex::from_bytes(disputes)?;
+ if countries.columns() != 360 || countries.rows() != 180 {
+ return Err(Error::invalid(
+ "country index",
+ "country grid must be 360 by 180",
+ ));
+ }
+ Ok(Self {
+ countries,
+ disputes,
+ })
+ }
+
+ pub fn lookup(&self, coordinate: Coordinate) -> Result<CountryClassification<'_>> {
+ let cell = self
+ .countries
+ .prepare_cell(coordinate.latitude, coordinate.longitude)?;
+ Ok(CountryClassification {
+ countries: self.countries.lookup_prepared(cell)?,
+ disputes: self.disputes.lookup_prepared(cell)?,
+ })
+ }
+
+ pub fn group(&self, coordinates: &[Coordinate], view: CountryView) -> Result<CountryGrouping> {
+ let mut countries = BTreeMap::<CountryCode, Vec<u32>>::new();
+ let mut disputes = BTreeMap::<TerritoryId, DisputeGroup>::new();
+ let mut unclassified_coordinate_indices = Vec::new();
+
+ for (coordinate_index, &coordinate) in coordinates.iter().enumerate() {
+ let classification = self.lookup(coordinate)?;
+ let coordinate_index = coordinate_index as u32;
+ if classification.disputes.is_empty() {
+ if classification.countries.is_empty() {
+ unclassified_coordinate_indices.push(coordinate_index);
+ }
+ for country in classification.countries {
+ countries.entry(country).or_default().push(coordinate_index);
+ }
+ continue;
+ }
+
+ let mut resolved = Vec::new();
+ for dispute in classification.disputes {
+ let country = dispute.resolve(view);
+ disputes
+ .entry(dispute.territory_id())
+ .or_insert_with(|| DisputeGroup {
+ territory: dispute.territory_id(),
+ name: dispute.name().to_owned(),
+ possible_countries: dispute.possible_countries().collect(),
+ resolved_country: country,
+ coordinate_indices: Vec::new(),
+ })
+ .coordinate_indices
+ .push(coordinate_index);
+ if let Some(country) = country
+ && !resolved.contains(&country)
+ {
+ resolved.push(country);
+ }
+ }
+ for country in resolved {
+ countries.entry(country).or_default().push(coordinate_index);
+ }
+ }
+
+ Ok(CountryGrouping {
+ countries: countries
+ .into_iter()
+ .map(|(country, coordinate_indices)| CountryGroup {
+ country,
+ coordinate_indices,
+ })
+ .collect(),
+ disputes: disputes.into_values().collect(),
+ unclassified_coordinate_indices,
+ })
+ }
+}
+
+#[derive(Debug)]
+pub struct CountryClassification<'a> {
+ pub countries: Vec<CountryCode>,
+ pub disputes: Vec<DisputeMatch<'a>>,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct CountryGrouping {
+ pub countries: Vec<CountryGroup>,
+ pub disputes: Vec<DisputeGroup>,
+ pub unclassified_coordinate_indices: Vec<u32>,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct CountryGroup {
+ pub country: CountryCode,
+ pub coordinate_indices: Vec<u32>,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct DisputeGroup {
+ pub territory: TerritoryId,
+ pub name: String,
+ pub possible_countries: Vec<CountryCode>,
+ pub resolved_country: Option<CountryCode>,
+ pub coordinate_indices: Vec<u32>,
+}