💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Use urban-centre polygons for city matching
Neeraj Guptaa5cd748594ae
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 7f60798efe..2155ff8f3a 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2662,6 +2662,7 @@ dependencies = ["ente-location","geo","reqwest 0.12.28",+ "rusqlite","sha2 0.11.0","shapefile","thiserror 2.0.18",diff --git a/rust/crates/location-dataset/Cargo.toml b/rust/crates/location-dataset/Cargo.tomlindex 85141b006a..a71cd3deae 100644--- a/rust/crates/location-dataset/Cargo.toml+++ b/rust/crates/location-dataset/Cargo.toml@@ -7,6 +7,7 @@ publish = falseente-location.workspace = truegeo = { version = "0.33.1", default-features = false }reqwest = { workspace = true, features = ["blocking", "rustls-tls-webpki-roots"] }+rusqlite = { version = "0.38", default-features = false, features = ["bundled"] }sha2.workspace = trueshapefile = { version = "0.9.0", features = ["geo-types"] }thiserror.workspace = truediff --git a/rust/crates/location-dataset/README.md b/rust/crates/location-dataset/README.mdindex 9c81f92152..ef90680d61 100644--- a/rust/crates/location-dataset/README.md+++ b/rust/crates/location-dataset/README.md@@ -8,14 +8,16 @@ cargo run --release -p ente-location-dataset -- \```The command downloads and caches its sources, writes `cities.bin`,-`countries.bin`, and `disputes.bin`, and prints their sizes and SHA-256 hashes.+`urban-centres.bin`, `countries.bin`, and `disputes.bin`, and prints their+sizes and SHA-256 hashes.-## Source+## Attribution-City data is derived from [GeoNames](https://www.geonames.org/) `cities5000`-and `countryInfo`, licensed under-[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Ente filters and-converts the data, which is provided without warranty.--Country boundaries are derived from [Natural Earth](https://www.naturalearthdata.com/),-which is in the public domain.+- [GeoNames](https://www.geonames.org/) `cities5000`, `countryInfo`, and+ `alternateNamesV2`: [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/),+ adapted by Ente.+- European Commission Joint Research Centre+ [GHS-UCDB R2024A](https://doi.org/10.2905/1a338be6-7eaf-480c-9664-3a8ade88cbcd):+ [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/), adapted by Ente.+- [Natural Earth](https://www.naturalearthdata.com/) country boundaries and+ populated places: public domain.diff --git a/rust/crates/location-dataset/src/city.rs b/rust/crates/location-dataset/src/city.rsindex 26a2c7fc37..1229695cd2 100644--- a/rust/crates/location-dataset/src/city.rs+++ b/rust/crates/location-dataset/src/city.rs@@ -1,23 +1,27 @@use std::cmp::Ordering;-use std::collections::{BTreeMap, HashMap, HashSet};+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};use std::fs::File;use std::io::{BufRead, BufReader};use std::path::Path;-use ente_location::CountryCode;+use ente_location::{CountryCode, normalize_search_text};+use shapefile::dbase::{FieldValue, Reader};+use zip::ZipArchive;use crate::{Result, invalid};-const HEADER_LEN: usize = 72;+const HEADER_LEN: usize = 76;const NODE_LEN: usize = 3;const POINT_LEN: usize = 15;const NAME_OFFSET_LEN: usize = 3;const MAX_U24: u32 = 0x00ff_ffff;+const ALTERNATE_NAMES_FILE: &str = "alternateNamesV2.txt";struct SourceCity {source_id: u32,name: String,ascii_name: String,+ aliases: Vec<String>,country_code: [u8; 2],latitude: f64,longitude: f64,@@ -41,14 +45,42 @@ impl SourceCity {}}-pub(crate) fn build(cities_path: &Path, country_info_path: &Path) -> Result<Vec<u8>> {+pub(crate) fn build(+ cities_path: &Path,+ country_info_path: &Path,+ alternate_names_path: &Path,+ populated_places_path: &Path,+) -> Result<Vec<u8>> {let countries = read_countries(country_info_path)?;+ let languages = read_country_languages(country_info_path)?;let mut cities = read_cities(cities_path, &countries)?;+ read_alternate_names(alternate_names_path, &mut cities, &languages)?;+ let override_ids = read_override_ids(populated_places_path)?;cities.sort_by(compare_priority);- encode(&cities, &countries)+ encode(&cities, &countries, &override_ids)}-fn read_countries(path: &Path) -> Result<BTreeMap<[u8; 2], String>> {+fn read_override_ids(path: &Path) -> Result<HashSet<u32>> {+ let mut reader = Reader::from_path(path).map_err(shapefile::Error::from)?;+ let mut ids = HashSet::new();+ for record in reader.iter_records() {+ let record = record.map_err(shapefile::Error::from)?;+ if let Some(FieldValue::Numeric(Some(id))) = record.get("GEONAMESID")+ && *id > 0.0+ && id.fract() == 0.0+ {+ ids.insert(*id as u32);+ }+ }+ if ids.is_empty() {+ return Err(invalid(+ "Natural Earth populated places contain no GeoNames IDs",+ ));+ }+ Ok(ids)+}++pub(crate) fn read_countries(path: &Path) -> Result<BTreeMap<[u8; 2], String>> {let reader = BufReader::new(File::open(path)?);let mut countries = BTreeMap::new();for line in reader.lines() {@@ -127,6 +159,7 @@ fn read_cities(path: &Path, countries: &BTreeMap<[u8; 2], String>) -> Result<Vecsource_id,name: name.to_owned(),ascii_name: ascii_name.to_owned(),+ aliases: Vec::new(),country_code,latitude,longitude,@@ -140,6 +173,145 @@ fn read_cities(path: &Path, countries: &BTreeMap<[u8; 2], String>) -> Result<VecOk(cities)}+fn read_country_languages(path: &Path) -> Result<BTreeMap<[u8; 2], BTreeSet<String>>> {+ let reader = BufReader::new(File::open(path)?);+ let mut countries = BTreeMap::new();+ for line in reader.lines() {+ let line = line?;+ if line.is_empty() || line.starts_with('#') {+ continue;+ }+ let fields: Vec<&str> = line.split('\t').collect();+ if fields.len() < 16 {+ return Err(invalid("GeoNames countryInfo row has too few fields"));+ }+ let code = code(fields[0])?;+ let languages = fields[15]+ .split(',')+ .filter(|language| !language.is_empty())+ .map(language_base)+ .map(str::to_owned)+ .collect();+ countries.insert(code, languages);+ }+ Ok(countries)+}++fn read_alternate_names(+ path: &Path,+ cities: &mut [SourceCity],+ country_languages: &BTreeMap<[u8; 2], BTreeSet<String>>,+) -> Result<()> {+ let positions: HashMap<u32, usize> = cities+ .iter()+ .enumerate()+ .map(|(index, city)| (city.source_id, index))+ .collect();+ let mut archive = ZipArchive::new(File::open(path)?)?;+ let source = archive.by_name(ALTERNATE_NAMES_FILE)?;+ apply_alternate_names(+ BufReader::new(source),+ cities,+ &positions,+ country_languages,+ )?;+ finalize_aliases(cities);+ Ok(())+}++fn finalize_aliases(cities: &mut [SourceCity]) {+ for city in cities {+ city.aliases.sort_unstable();+ city.aliases.dedup();+ let name = normalize_search_text(&city.name);+ let ascii_name = normalize_search_text(&city.ascii_name);+ city.aliases+ .retain(|alias| alias != &name && alias != &ascii_name);+ }+}++fn apply_alternate_names(+ reader: impl BufRead,+ cities: &mut [SourceCity],+ positions: &HashMap<u32, usize>,+ country_languages: &BTreeMap<[u8; 2], BTreeSet<String>>,+) -> Result<()> {+ for (line_index, line) in reader.lines().enumerate() {+ let line = line?;+ let mut fields = line.split('\t');+ fields.next();+ let source_id: u32 = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?+ .parse()+ .map_err(|_| {+ invalid(format!(+ "GeoNames alternate-name row {} has invalid ID",+ line_index + 1+ ))+ })?;+ let Some(&city_index) = positions.get(&source_id) else {+ continue;+ };+ let language = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;+ let name = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?+ .trim();+ let preferred = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;+ fields.next();+ let colloquial = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;+ let historic = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;+ fields.next();+ let to = fields+ .next()+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;+ let city = &mut cities[city_index];+ if !name.is_empty()+ && !name.contains('\0')+ && colloquial != "1"+ && historic != "1"+ && to.is_empty()+ && is_search_language(+ language,+ preferred == "1",+ &country_languages[&city.country_code],+ )+ {+ let alias = normalize_search_text(name);+ if !alias.is_empty() {+ city.aliases.push(alias);+ }+ }+ }+ Ok(())+}++fn is_search_language(+ language: &str,+ preferred: bool,+ country_languages: &BTreeSet<String>,+) -> bool {+ !matches!(+ language,+ "abbr" | "faac" | "iata" | "icao" | "link" | "post" | "unlc" | "wkdt"+ ) && (preferred+ || language_base(language) == "en"+ || country_languages.contains(language_base(language)))+}++fn language_base(language: &str) -> &str {+ language.split_once('-').map_or(language, |(base, _)| base)+}+fn is_current_place(feature_code: &str) -> bool {!matches!(feature_code, "PPLH" | "PPLQ" | "PPLW")}@@ -154,11 +326,26 @@ fn compare_priority(left: &SourceCity, right: &SourceCity) -> Ordering {.then_with(|| left.source_id.cmp(&right.source_id))}-fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Result<Vec<u8>> {+fn encode(+ cities: &[SourceCity],+ countries: &BTreeMap<[u8; 2], String>,+ override_ids: &HashSet<u32>,+) -> Result<Vec<u8>> {if cities.len() > MAX_U24 as usize {return Err(invalid("city index exceeds 24-bit point indices"));}- let nodes = build_tree(cities);+ let rank_ends = [4, 3, 2, 1].map(|rank| cities.partition_point(|city| city.rank() >= rank));+ let nodes = build_tree(cities, (0..cities.len()).collect());+ let override_nodes = build_tree(+ cities,+ cities+ .iter()+ .enumerate()+ .filter_map(|(index, city)| {+ (city.rank() >= 2 && override_ids.contains(&city.source_id)).then_some(index)+ })+ .collect(),+ );let mut names = Vec::<String>::new();let mut name_indices = HashMap::<String, u32>::new();let mut point_names = Vec::with_capacity(cities.len());@@ -168,6 +355,10 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resulkey.push('\0');key.push_str(&city.ascii_name.to_lowercase());}+ for alias in &city.aliases {+ key.push('\0');+ key.push_str(alias);+ }let next = names.len() as u32;let index = *name_indices.entry(key.clone()).or_insert_with(|| {names.push(key);@@ -188,10 +379,9 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resul.collect();let name_table = string_table_u24(names.iter().map(String::as_str))?;let country_table = string_table_u32(countries.values().map(String::as_str))?;- let rank_ends = [4, 3, 2, 1].map(|rank| cities.partition_point(|city| city.rank() >= rank));-let nodes_offset = HEADER_LEN;- let points_offset = nodes_offset + nodes.len() * NODE_LEN;+ let override_nodes_offset = nodes_offset + nodes.len() * NODE_LEN;+ let points_offset = override_nodes_offset + override_nodes.len() * NODE_LEN;let name_offsets_offset = points_offset + cities.len() * POINT_LEN;let names_offset = name_offsets_offset + name_table.0.len();let country_offsets_offset = names_offset + name_table.1.len();@@ -201,7 +391,7 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resullet mut output = vec![0; file_length];output[..4].copy_from_slice(b"CITY");- put_u16(&mut output, 4, 1);+ put_u16(&mut output, 4, 2);put_u16(&mut output, 6, HEADER_LEN as u16);put_len(&mut output, 8, cities.len())?;put_len(&mut output, 12, names.len())?;@@ -228,11 +418,16 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resulfor (field, value) in [56, 60, 64, 68].into_iter().zip(rank_ends) {put_len(&mut output, field, value)?;}+ put_len(&mut output, 72, override_nodes.len())?;for (index, &point) in nodes.iter().enumerate() {let offset = nodes_offset + index * NODE_LEN;put_u24(&mut output, offset, point as u32);}+ for (index, &point) in override_nodes.iter().enumerate() {+ let offset = override_nodes_offset + index * NODE_LEN;+ put_u24(&mut output, offset, point as u32);+ }for (index, city) in cities.iter().enumerate() {let offset = points_offset + index * POINT_LEN;output[offset..offset + 4].copy_from_slice(&(city.latitude as f32).to_le_bytes());@@ -251,7 +446,7 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> ResulOk(output)}-fn build_tree(cities: &[SourceCity]) -> Vec<usize> {+fn build_tree(cities: &[SourceCity], mut points: Vec<usize>) -> Vec<usize> {fn build(cities: &[SourceCity], nodes: &mut Vec<usize>, points: &mut [usize], depth: usize) {if points.is_empty() {return;@@ -276,8 +471,7 @@ fn build_tree(cities: &[SourceCity]) -> Vec<usize> {build(cities, nodes, right, depth + 1);}- let mut points: Vec<usize> = (0..cities.len()).collect();- let mut nodes = Vec::with_capacity(cities.len());+ let mut nodes = Vec::with_capacity(points.len());build(cities, &mut nodes, &mut points, 0);nodes}@@ -295,7 +489,7 @@ fn string_table_u24<'a>(Ok((offsets, blob))}-fn string_table_u32<'a>(+pub(crate) fn string_table_u32<'a>(values: impl ExactSizeIterator<Item = &'a str>,) -> Result<(Vec<u8>, Vec<u8>)> {let mut offsets = Vec::with_capacity((values.len() + 1) * 4);@@ -369,35 +563,83 @@ fn copy(output: &mut [u8], offset: usize, bytes: &[u8]) {#[cfg(test)]mod tests {- use std::collections::BTreeMap;+ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};+ use std::io::Cursor;use ente_location::{CityIndex, Coordinate, CountryCode};- use super::{SourceCity, encode};+ use super::{SourceCity, apply_alternate_names, encode, finalize_aliases};#[test]fn round_trips_city_search_and_lookup() {+ let mut munich = city(3, "Munich", "Munich", *b"DE", 48.1374, 11.5755, 3);+ munich.aliases = vec!["garching bei munchen".to_owned(), "munchen".to_owned()];let cities = vec![city(1, "Delhi", "Delhi", *b"IN", 28.6139, 77.2090, 4),city(2, "Zürich", "Zuerich", *b"CH", 47.3667, 8.5500, 3),+ munich,+ city(4, "Small Town", "Small Town", *b"IN", 0.0, 0.0, 0),];let countries = BTreeMap::from([(*b"CH", "Switzerland".to_owned()),+ (*b"DE", "Germany".to_owned()),(*b"IN", "India".to_owned()),]);- let bytes = encode(&cities, &countries).unwrap();+ let bytes = encode(&cities, &countries, &HashSet::from([1, 2, 3])).unwrap();let index = CityIndex::from_bytes(bytes).unwrap();assert_eq!(index.search("zurich", 5)[0].name, "Zürich");assert_eq!(index.search("ZUERICH", 5)[0].name, "Zürich");+ assert_eq!(index.search("München", 5)[0].name, "Munich");+ assert_eq!(index.search("Garching bei München", 5)[0].name, "Munich");+ assert_eq!(+ index.match_coordinates(&[Coordinate::new(0.0, 0.0)], "")[0]+ .city+ .name,+ "Small Town"+ );+ assert!(+ index+ .match_override_coordinates(&[Coordinate::new(0.0, 0.0)], "")+ .is_empty()+ );+ assert!(+ index+ .match_override_coordinates(&[Coordinate::new(28.65, 77.209)], "")+ .is_empty()+ );+ assert_eq!(+ index.match_coordinates(&[Coordinate::new(28.71, 77.209)], "")[0]+ .city+ .name,+ "Delhi"+ );assert_eq!(- index.match_coordinates(&[Coordinate::new(28.61, 77.21)], "")[0]+ index.match_override_coordinates(&[Coordinate::new(28.61, 77.21)], "")[0].city.country_code,CountryCode::from_bytes(*b"IN").unwrap());}+ #[test]+ fn keeps_current_preferred_and_local_aliases() {+ let mut cities = vec![city(1, "Munich", "Munich", *b"DE", 48.1, 11.5, 3)];+ let positions = HashMap::from([(1, 0)]);+ let languages = BTreeMap::from([(*b"DE", BTreeSet::from(["de".to_owned()]))]);+ let source = concat!(+ "1\t1\tde\tMünchen\t1\t\t\t\t\t\n",+ "2\t1\tde\tMuenchen\t\t\t\t\t\t\n",+ "3\t1\ten\tOld Munich\t\t\t\t1\t\t\n",+ "4\t1\tlink\thttps://example.com\t1\t\t\t\t\t\n"+ );++ apply_alternate_names(Cursor::new(source), &mut cities, &positions, &languages).unwrap();+ finalize_aliases(&mut cities);++ assert_eq!(cities[0].aliases, ["muenchen", "munchen"]);+ }+fn city(source_id: u32,name: &str,@@ -411,6 +653,7 @@ mod tests {source_id,name: name.to_owned(),ascii_name: ascii_name.to_owned(),+ aliases: Vec::new(),country_code,latitude,longitude,diff --git a/rust/crates/location-dataset/src/lib.rs b/rust/crates/location-dataset/src/lib.rsindex 417e669103..c3b6f3cf97 100644--- a/rust/crates/location-dataset/src/lib.rs+++ b/rust/crates/location-dataset/src/lib.rs@@ -1,7 +1,7 @@use std::fmt::Write as _;use std::path::PathBuf;-use ente_location::{CityIndex, CountryIndex};+use ente_location::{CityIndex, CountryIndex, UrbanCenterIndex};use sha2::{Digest, Sha256};use thiserror::Error;@@ -11,10 +11,12 @@ mod country;mod dispute;mod download;mod sources;+mod urban;pub use sources::{RemoteSources, SourcePaths};pub const CITY_FILE: &str = "cities.bin";+pub const URBAN_CENTER_FILE: &str = "urban-centres.bin";pub const COUNTRY_FILE: &str = "countries.bin";pub const DISPUTE_FILE: &str = "disputes.bin";@@ -30,6 +32,8 @@ pub enum Error {Zip(#[from] zip::result::ZipError),#[error("shapefile is invalid: {0}")]Shapefile(#[from] shapefile::Error),+ #[error("GeoPackage is invalid: {0}")]+ Sqlite(#[from] rusqlite::Error),#[error("invalid location data: {0}")]InvalidData(String),#[error(transparent)]@@ -43,9 +47,10 @@ pub struct BuildOptions {pub struct BuildOutput {pub city_count: usize,+ pub urban_center_count: usize,pub territory_count: usize,pub byte_length: usize,- pub files: [BuildFile; 3],+ pub files: [BuildFile; 4],}pub struct BuildFile {@@ -55,22 +60,36 @@ pub struct BuildFile {}pub fn build(options: &BuildOptions) -> Result<BuildOutput> {- let cities = city::build(&options.sources.cities, &options.sources.country_info)?;+ let cities = city::build(+ &options.sources.cities,+ &options.sources.country_info,+ &options.sources.alternate_names,+ &options.sources.populated_places,+ )?;+ let city_index = CityIndex::from_bytes(cities.as_slice())?;+ let urban_centers = urban::build(+ &options.sources.urban_centres,+ &options.sources.country_info,+ &city_index,+ )?;let countries = country::build_countries(&options.sources.countries)?;let disputes = dispute::build(&options.sources.countries,&options.sources.disputes,&options.sources.admin1,)?;- let city_count = CityIndex::from_bytes(cities.as_slice())?.len();+ let city_count = city_index.len();+ let urban_center_count = UrbanCenterIndex::from_bytes(urban_centers.as_slice())?.len();CountryIndex::from_bytes(countries.as_slice(), disputes.as_slice())?;let files = [build_file(CITY_FILE, &cities),+ build_file(URBAN_CENTER_FILE, &urban_centers),build_file(COUNTRY_FILE, &countries),build_file(DISPUTE_FILE, &disputes),];let output = BuildOutput {city_count,+ urban_center_count,territory_count: catalog::DISPUTED_AREAS.len() + catalog::UKRAINIAN_REGIONS.len(),byte_length: files.iter().map(|file| file.byte_length).sum(),files,@@ -78,6 +97,7 @@ pub fn build(options: &BuildOptions) -> Result<BuildOutput> {std::fs::create_dir_all(&options.output)?;for (name, bytes) in [(CITY_FILE, cities.as_slice()),+ (URBAN_CENTER_FILE, urban_centers.as_slice()),(COUNTRY_FILE, countries.as_slice()),(DISPUTE_FILE, disputes.as_slice()),] {diff --git a/rust/crates/location-dataset/src/main.rs b/rust/crates/location-dataset/src/main.rsindex 1981fbe424..6e0200b232 100644--- a/rust/crates/location-dataset/src/main.rs+++ b/rust/crates/location-dataset/src/main.rs@@ -6,8 +6,9 @@ use ente_location_dataset::{BuildOptions, RemoteSources, SourcePaths, build};const USAGE: &str = concat!("usage: ente-location-dataset build --output <directory> [--cache <directory>] ",- "[--cities <file> --country-info <file> ",- "--countries <shp> --disputes <shp> --admin1 <shp>]"+ "[--cities <file> --country-info <file> --alternate-names <zip> ",+ "--urban-centres <gpkg> --populated-places <dbf> --countries <shp> ",+ "--disputes <shp> --admin1 <shp>]");fn main() {@@ -23,25 +24,48 @@ fn run() -> Result<(), Box<dyn Error>> {cache,cities,country_info,+ alternate_names,+ urban_centres,+ populated_places,countries,disputes,admin1,} = Arguments::parse()?;- let sources = match (cities, country_info, countries, disputes, admin1) {- (None, None, None, None, None) => RemoteSources::new(cache).fetch()?,- (Some(cities), Some(country_info), Some(countries), Some(disputes), Some(admin1)) => {- SourcePaths {- cities,- country_info,- countries,- disputes,- admin1,- }- }- _ => return Err("local generation requires all five source paths".into()),+ let sources = match (+ cities,+ country_info,+ alternate_names,+ urban_centres,+ populated_places,+ countries,+ disputes,+ admin1,+ ) {+ (None, None, None, None, None, None, None, None) => RemoteSources::new(cache).fetch()?,+ (+ Some(cities),+ Some(country_info),+ Some(alternate_names),+ Some(urban_centres),+ Some(populated_places),+ Some(countries),+ Some(disputes),+ Some(admin1),+ ) => SourcePaths {+ cities,+ country_info,+ alternate_names,+ urban_centres,+ populated_places,+ countries,+ disputes,+ admin1,+ },+ _ => return Err("local generation requires all eight source paths".into()),};let result = build(&BuildOptions { sources, output })?;println!("cities: {}", result.city_count);+ println!("urban centers: {}", result.urban_center_count);println!("Priority-1 territories: {}", result.territory_count);println!("bytes: {}", result.byte_length);for file in result.files {@@ -58,6 +82,9 @@ struct Arguments {cache: PathBuf,cities: Option<PathBuf>,country_info: Option<PathBuf>,+ alternate_names: Option<PathBuf>,+ urban_centres: Option<PathBuf>,+ populated_places: Option<PathBuf>,countries: Option<PathBuf>,disputes: Option<PathBuf>,admin1: Option<PathBuf>,@@ -73,6 +100,9 @@ impl Arguments {let mut cache = None;let mut cities = None;let mut country_info = None;+ let mut alternate_names = None;+ let mut urban_centres = None;+ let mut populated_places = None;let mut countries = None;let mut disputes = None;let mut admin1 = None;@@ -83,6 +113,9 @@ impl Arguments {"--cache" => cache = Some(value.into()),"--cities" => cities = Some(value.into()),"--country-info" => country_info = Some(value.into()),+ "--alternate-names" => alternate_names = Some(value.into()),+ "--urban-centres" => urban_centres = Some(value.into()),+ "--populated-places" => populated_places = Some(value.into()),"--countries" => countries = Some(value.into()),"--disputes" => disputes = Some(value.into()),"--admin1" => admin1 = Some(value.into()),@@ -94,6 +127,9 @@ impl Arguments {cache: cache.unwrap_or_else(default_cache),cities,country_info,+ alternate_names,+ urban_centres,+ populated_places,countries,disputes,admin1,diff --git a/rust/crates/location-dataset/src/sources.rs b/rust/crates/location-dataset/src/sources.rsindex 828f1ace49..6ca280850d 100644--- a/rust/crates/location-dataset/src/sources.rs+++ b/rust/crates/location-dataset/src/sources.rs@@ -7,6 +7,17 @@ use crate::download::{download, extract};const CITIES_URL: &str = "https://download.geonames.org/export/dump/cities5000.zip";const COUNTRY_INFO_URL: &str = "https://download.geonames.org/export/dump/countryInfo.txt";+const ALTERNATE_NAMES_URL: &str = "https://download.geonames.org/export/dump/alternateNamesV2.zip";+const URBAN_CENTRES_FILE: &str = "GHS_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A.gpkg";+const URBAN_CENTRES_URL: &str = concat!(+ "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/",+ "GHS_UCDB_GLOBE_R2024A/GHS_UCDB_THEME_GLOBE_R2024A/",+ "GHS_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A/V1-2/",+ "GHS_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A_V1_2.zip"+);+const POPULATED_PLACES: &str = "ne_10m_populated_places";+const POPULATED_PLACES_URL: &str =+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_populated_places.zip";const COUNTRIES: &str = "ne_10m_admin_0_countries";const COUNTRIES_URL: &str ="https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_countries.zip";@@ -21,6 +32,9 @@ const ADMIN1_URL: &str =pub struct SourcePaths {pub cities: PathBuf,pub country_info: PathBuf,+ pub alternate_names: PathBuf,+ pub urban_centres: PathBuf,+ pub populated_places: PathBuf,pub countries: PathBuf,pub disputes: PathBuf,pub admin1: PathBuf,@@ -46,21 +60,40 @@ impl RemoteSources {let cities_archive = self.cache.join("cities5000.zip");let cities = self.cache.join("cities5000.txt");let country_info = self.cache.join("countryInfo.txt");+ let alternate_names = self.cache.join("alternateNamesV2.zip");download(&client, CITIES_URL, &cities_archive)?;download(&client, COUNTRY_INFO_URL, &country_info)?;+ download(&client, ALTERNATE_NAMES_URL, &alternate_names)?;extract(&cities_archive, &cities, "cities5000.txt")?;+ let urban_centres_archive = self.cache.join("ghsl-urban-centres-v1.2.zip");+ let urban_centres = self.cache.join(URBAN_CENTRES_FILE);+ download(&client, URBAN_CENTRES_URL, &urban_centres_archive)?;+ extract(&urban_centres_archive, &urban_centres, URBAN_CENTRES_FILE)?;+ let populated_places = self.fetch_dbf(&client, POPULATED_PLACES, POPULATED_PLACES_URL)?;let countries = self.fetch_shapefile(&client, COUNTRIES, COUNTRIES_URL)?;let disputes = self.fetch_shapefile(&client, DISPUTES, DISPUTES_URL)?;let admin1 = self.fetch_shapefile(&client, ADMIN1, ADMIN1_URL)?;Ok(SourcePaths {cities,country_info,+ alternate_names,+ urban_centres,+ populated_places,countries,disputes,admin1,})}+ fn fetch_dbf(&self, client: &Client, name: &str, url: &str) -> Result<PathBuf> {+ let archive = self.cache.join(format!("{name}.zip"));+ download(client, url, &archive)?;+ let file = format!("{name}.dbf");+ let path = self.cache.join(&file);+ extract(&archive, &path, &file)?;+ Ok(path)+ }+fn fetch_shapefile(&self, client: &Client, name: &str, url: &str) -> Result<PathBuf> {let archive = self.cache.join(format!("{name}.zip"));download(client, url, &archive)?;diff --git a/rust/crates/location-dataset/src/urban.rs b/rust/crates/location-dataset/src/urban.rsnew file mode 100644index 0000000000..1d0922bd43--- /dev/null+++ b/rust/crates/location-dataset/src/urban.rs@@ -0,0 +1,694 @@+use std::collections::BTreeMap;+use std::f64::consts::{PI, SQRT_2};+use std::path::Path;++use ente_location::{City, CityIndex, Coordinate, UrbanCenterIndex, normalize_search_text};+use geo::{BoundingRect, Coord, Distance, Haversine, LineString, MultiPolygon, Point, Polygon};+use rusqlite::{Connection, OpenFlags};++use crate::city::{read_countries, string_table_u32};+use crate::{Result, invalid};++const QUERY: &str = concat!(+ "SELECT u.ID_UC_G0, u.GC_UCN_MAI_2025, u.GC_CNT_UNN_2025, ",+ "c.GC_UCC_LON_2025, c.GC_UCC_LAT_2025, u.geom ",+ "FROM GHSL_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A u ",+ "JOIN UC_centroids c USING (ID_UC_G0) ",+ "WHERE u.GC_PLS_SCR_2025 IN ('High', 'Medium') ",+ "AND trim(u.GC_UCN_MAI_2025) != '' ORDER BY u.ID_UC_G0"+);+const COLUMNS: u16 = 360;+const ROWS: u16 = 180;+const HEADER_LEN: usize = 72;+const RECORD_LEN: usize = 41;+const BLOCK_SIZE: usize = 64;+const BLOCK_LEN: usize = 12;+const BOUNDS_PADDING: f64 = 0.0001;+const MOLLWEIDE_RADIUS: f64 = 6_378_137.0;+const GHSL_SOURCE_BIT: u32 = 1 << 31;+const MAX_NAME_MATCH_DISTANCE_METERS: f64 = 30_000.0;++struct SourceCenter {+ source_id: u32,+ name: String,+ country_code: [u8; 2],+ centroid: Coordinate,+ geometry: MultiPolygon<f64>,+}++#[derive(Clone, Copy)]+struct Bounds {+ minimum_longitude: f64,+ minimum_latitude: f64,+ maximum_longitude: f64,+ maximum_latitude: f64,+}++pub(crate) fn build(+ path: &Path,+ country_info_path: &Path,+ city_index: &CityIndex,+) -> Result<Vec<u8>> {+ let countries = read_countries(country_info_path)?;+ let mut cities_by_name = BTreeMap::<([u8; 2], String), Vec<City>>::new();+ for city in city_index.search("", usize::MAX) {+ cities_by_name+ .entry((+ city.country_code.as_bytes(),+ normalize_search_text(&city.name),+ ))+ .or_default()+ .push(city);+ }+ let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;+ let mut statement = connection.prepare(QUERY)?;+ let rows = statement.query_map([], |row| {+ Ok((+ row.get::<_, u32>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, String>(2)?,+ row.get::<_, f64>(3)?,+ row.get::<_, f64>(4)?,+ row.get::<_, Vec<u8>>(5)?,+ ))+ })?;+ let mut centers = Vec::new();+ for row in rows {+ let (source_id, name, country, centroid_x, centroid_y, geometry) = row?;+ if source_id & GHSL_SOURCE_BIT != 0 {+ return Err(invalid("GHSL urban center ID exceeds 31 bits"));+ }+ let country_code = resolve_country(&country, &countries)+ .ok_or_else(|| invalid(format!("unknown GHSL country {country}")))?;+ let centroid = mollweide_to_wgs84(centroid_x, centroid_y)?;+ let geometry = decode_geopackage_geometry(&geometry)?;+ let key = (country_code, normalize_search_text(&name));+ let named = canonical_name_city(+ cities_by_name+ .get(&key)+ .map(Vec::as_slice)+ .unwrap_or_default(),+ centroid,+ );+ centers.push(match named {+ Some(city) => SourceCenter {+ source_id: city.source_id,+ name: city.name.clone(),+ country_code,+ centroid: Coordinate::new(city.latitude, city.longitude),+ geometry,+ },+ None => SourceCenter {+ source_id: source_id | GHSL_SOURCE_BIT,+ name,+ country_code,+ centroid,+ geometry,+ },+ });+ }+ encode(¢ers, &countries)+}++fn canonical_name_city(cities: &[City], centroid: Coordinate) -> Option<&City> {+ let centroid = Point::new(centroid.longitude, centroid.latitude);+ cities+ .iter()+ .map(|city| {+ let distance = Haversine.distance(centroid, Point::new(city.longitude, city.latitude));+ (city, distance)+ })+ .filter(|(_, distance)| *distance <= MAX_NAME_MATCH_DISTANCE_METERS)+ .min_by(|left, right| left.1.total_cmp(&right.1))+ .map(|(city, _)| city)+}++fn encode(+ centers: &[SourceCenter],+ source_countries: &BTreeMap<[u8; 2], String>,+) -> Result<Vec<u8>> {+ if centers.is_empty() || centers.len() > usize::from(u16::MAX) {+ return Err(invalid("urban center count does not fit the index"));+ }++ let countries: BTreeMap<[u8; 2], &str> = centers+ .iter()+ .map(|center| {+ (+ center.country_code,+ source_countries[¢er.country_code].as_str(),+ )+ })+ .collect();+ if countries.len() > 256 {+ return Err(invalid("urban center country count exceeds 256"));+ }+ let country_indices: BTreeMap<[u8; 2], u8> = countries+ .keys()+ .copied()+ .enumerate()+ .map(|(index, code)| (code, index as u8))+ .collect();+ let bounds: Vec<Bounds> = centers+ .iter()+ .map(|center| geometry_bounds(¢er.geometry))+ .collect::<Result<_>>()?;++ let mut names = Vec::new();+ let mut name_offsets = Vec::with_capacity(centers.len());+ for center in centers {+ if center.name.contains('\0') {+ return Err(invalid("urban center name contains NUL"));+ }+ name_offsets.push(names.len());+ names.extend_from_slice(center.name.as_bytes());+ names.push(0);+ }++ let mut geometry = Vec::new();+ let mut geometry_offsets = Vec::with_capacity(centers.len() + 1);+ geometry_offsets.push(0);+ for (center, &bounds) in centers.iter().zip(&bounds) {+ encode_geometry(&mut geometry, ¢er.geometry, bounds)?;+ geometry_offsets.push(geometry.len());+ }++ let (country_offsets, country_names) = string_table_u32(countries.values().copied())?;+ let country_codes: Vec<u8> = countries.keys().flatten().copied().collect();+ let cells = grid(&bounds)?;+ let block_count = cells.len().div_ceil(BLOCK_SIZE);+ let nonempty_count = cells.iter().filter(|cell| !cell.is_empty()).count();+ let reference_count: usize = cells.iter().map(Vec::len).sum();++ let records_offset = HEADER_LEN;+ let names_offset = records_offset + centers.len() * RECORD_LEN;+ let geometry_offset = names_offset + names.len();+ let country_offsets_offset = geometry_offset + geometry.len();+ let countries_offset = country_offsets_offset + country_offsets.len();+ let country_codes_offset = countries_offset + country_names.len();+ let blocks_offset = country_codes_offset + country_codes.len();+ let cell_offsets_offset = blocks_offset + block_count * BLOCK_LEN;+ let references_offset = cell_offsets_offset + (nonempty_count + 1) * 4;+ let file_length = references_offset + reference_count * 2;++ let mut output = Vec::with_capacity(file_length);+ output.extend_from_slice(b"URBN");+ push_u16(&mut output, 1);+ push_u16(&mut output, HEADER_LEN as u16);+ push_len(&mut output, centers.len())?;+ push_u16(&mut output, countries.len() as u16);+ push_u16(&mut output, COLUMNS);+ push_u16(&mut output, ROWS);+ push_u16(&mut output, BLOCK_SIZE as u16);+ for value in [+ block_count,+ nonempty_count,+ records_offset,+ names_offset,+ geometry_offset,+ country_offsets_offset,+ countries_offset,+ country_codes_offset,+ blocks_offset,+ cell_offsets_offset,+ references_offset,+ file_length,+ reference_count,+ ] {+ push_len(&mut output, value)?;+ }+ debug_assert_eq!(output.len(), HEADER_LEN);++ for (index, center) in centers.iter().enumerate() {+ let bounds = bounds[index];+ for value in [+ bounds.minimum_longitude,+ bounds.minimum_latitude,+ bounds.maximum_longitude,+ bounds.maximum_latitude,+ center.centroid.latitude,+ center.centroid.longitude,+ ] {+ output.extend_from_slice(&(value as f32).to_le_bytes());+ }+ push_len(&mut output, names_offset + name_offsets[index])?;+ push_len(&mut output, geometry_offset + geometry_offsets[index])?;+ push_len(&mut output, geometry_offset + geometry_offsets[index + 1])?;+ output.extend_from_slice(¢er.source_id.to_le_bytes());+ output.push(country_indices[¢er.country_code]);+ }+ output.extend_from_slice(&names);+ output.extend_from_slice(&geometry);+ output.extend_from_slice(&country_offsets);+ output.extend_from_slice(&country_names);+ output.extend_from_slice(&country_codes);++ let mut dense_start = 0;+ for block in cells.chunks(BLOCK_SIZE) {+ let mask = block.iter().enumerate().fold(0_u64, |mask, (bit, cell)| {+ mask | ((!cell.is_empty()) as u64) << bit+ });+ output.extend_from_slice(&mask.to_le_bytes());+ push_len(&mut output, dense_start)?;+ dense_start += mask.count_ones() as usize;+ }+ let mut reference_offset = 0;+ push_len(&mut output, reference_offset)?;+ for cell in cells.iter().filter(|cell| !cell.is_empty()) {+ reference_offset += cell.len();+ push_len(&mut output, reference_offset)?;+ }+ for cell in cells.iter().filter(|cell| !cell.is_empty()) {+ for &reference in cell {+ push_u16(&mut output, reference);+ }+ }+ debug_assert_eq!(output.len(), file_length);+ UrbanCenterIndex::from_bytes(output.as_slice())?;+ Ok(output)+}++fn geometry_bounds(geometry: &MultiPolygon<f64>) -> Result<Bounds> {+ let bounds = geometry+ .bounding_rect()+ .ok_or_else(|| invalid("urban center has empty geometry"))?;+ Ok(Bounds {+ minimum_longitude: (bounds.min().x - BOUNDS_PADDING).max(-180.0),+ minimum_latitude: (bounds.min().y - BOUNDS_PADDING).max(-90.0),+ maximum_longitude: (bounds.max().x + BOUNDS_PADDING).min(180.0),+ maximum_latitude: (bounds.max().y + BOUNDS_PADDING).min(90.0),+ })+}++fn encode_geometry(+ output: &mut Vec<u8>,+ geometry: &MultiPolygon<f64>,+ bounds: Bounds,+) -> Result<()> {+ push_count(output, geometry.0.len(), "too many urban center polygons")?;+ for polygon in geometry {+ push_count(+ output,+ polygon.interiors().len() + 1,+ "too many urban center rings",+ )?;+ for ring in std::iter::once(polygon.exterior()).chain(polygon.interiors()) {+ push_count(output, ring.0.len(), "urban center ring is too long")?;+ for coordinate in &ring.0 {+ push_u16(+ output,+ quantize(+ coordinate.x,+ bounds.minimum_longitude,+ bounds.maximum_longitude,+ ),+ );+ push_u16(+ output,+ quantize(+ coordinate.y,+ bounds.minimum_latitude,+ bounds.maximum_latitude,+ ),+ );+ }+ }+ }+ Ok(())+}++fn grid(bounds: &[Bounds]) -> Result<Vec<Vec<u16>>> {+ let mut cells = vec![Vec::new(); usize::from(COLUMNS) * usize::from(ROWS)];+ for (index, bounds) in bounds.iter().enumerate() {+ let index = u16::try_from(index).map_err(|_| invalid("too many urban centers"))?;+ for row in latitude_row(bounds.maximum_latitude)..=latitude_row(bounds.minimum_latitude) {+ for column in longitude_column(bounds.minimum_longitude)+ ..=longitude_column(bounds.maximum_longitude)+ {+ cells[row * usize::from(COLUMNS) + column].push(index);+ }+ }+ }+ Ok(cells)+}++fn longitude_column(longitude: f64) -> usize {+ (((longitude + 180.0) / 360.0) * f64::from(COLUMNS))+ .floor()+ .clamp(0.0, f64::from(COLUMNS - 1)) as usize+}++fn latitude_row(latitude: f64) -> usize {+ (((90.0 - latitude) / 180.0) * f64::from(ROWS))+ .floor()+ .clamp(0.0, f64::from(ROWS - 1)) as usize+}++fn quantize(value: f64, minimum: f64, maximum: f64) -> u16 {+ (((value - minimum) * f64::from(u16::MAX) / (maximum - minimum))+ .clamp(0.0, f64::from(u16::MAX))) as u16+}++fn resolve_country(name: &str, countries: &BTreeMap<[u8; 2], String>) -> Option<[u8; 2]> {+ countries+ .iter()+ .find_map(|(&code, country)| (country == name).then_some(code))+ .or_else(|| country_alias(name).filter(|code| countries.contains_key(code)))+}++fn country_alias(name: &str) -> Option<[u8; 2]> {+ Some(*match name {+ "Bolivia (Plurinational State of)" => b"BO",+ "Brunei Darussalam" => b"BN",+ "China, Taiwan Province of China" => b"TW",+ "Congo" => b"CG",+ "Curaçao" => b"CW",+ "Côte d'Ivoire" => b"CI",+ "Dem. People's Republic of Korea" => b"KP",+ "Iran (Islamic Republic of)" => b"IR",+ "Kosovo (under UNSC res. 1244)" => b"XK",+ "Lao People's Democratic Republic" => b"LA",+ "Netherlands" => b"NL",+ "Republic of Korea" => b"KR",+ "Republic of Moldova" => b"MD",+ "Russian Federation" => b"RU",+ "Réunion" => b"RE",+ "State of Palestine" => b"PS",+ "Syrian Arab Republic" => b"SY",+ "Timor-Leste" => b"TL",+ "United Republic of Tanzania" => b"TZ",+ "United States of America" => b"US",+ "Venezuela (Bolivarian Republic of)" => b"VE",+ "Viet Nam" => b"VN",+ _ => return None,+ })+}++fn decode_geopackage_geometry(bytes: &[u8]) -> Result<MultiPolygon<f64>> {+ if bytes.len() < 8 || &bytes[..2] != b"GP" || bytes[2] != 0 {+ return Err(invalid("invalid GHSL GeoPackage geometry header"));+ }+ let flags = bytes[3];+ if flags & 1 == 0 || flags & 0x10 != 0 {+ return Err(invalid("unsupported GHSL GeoPackage geometry flags"));+ }+ let srs = i32::from_le_bytes(bytes[4..8].try_into().expect("eight-byte header"));+ if srs != 54_009 {+ return Err(invalid(format!("unexpected GHSL spatial reference {srs}")));+ }+ let envelope_length = match (flags >> 1) & 7 {+ 0 => 0,+ 1 => 32,+ 2 | 3 => 48,+ 4 => 64,+ _ => return Err(invalid("invalid GHSL GeoPackage envelope")),+ };+ let mut reader = WkbReader::new(+ bytes+ .get(8 + envelope_length..)+ .ok_or_else(|| invalid("truncated GHSL GeoPackage geometry"))?,+ );+ let geometry = reader.multi_polygon()?;+ if reader.position != reader.bytes.len() {+ return Err(invalid("GHSL geometry has trailing bytes"));+ }+ Ok(geometry)+}++struct WkbReader<'a> {+ bytes: &'a [u8],+ position: usize,+}++impl<'a> WkbReader<'a> {+ const fn new(bytes: &'a [u8]) -> Self {+ Self { bytes, position: 0 }+ }++ fn multi_polygon(&mut self) -> Result<MultiPolygon<f64>> {+ let endian = self.header(6)?;+ let count = self.u32(endian)? as usize;+ let polygons = (0..count)+ .map(|_| self.polygon())+ .collect::<Result<Vec<_>>>()?;+ Ok(MultiPolygon::new(polygons))+ }++ fn polygon(&mut self) -> Result<Polygon<f64>> {+ let endian = self.header(3)?;+ let count = self.u32(endian)? as usize;+ if count == 0 {+ return Err(invalid("GHSL polygon has no rings"));+ }+ let mut rings = (0..count)+ .map(|_| self.ring(endian))+ .collect::<Result<Vec<_>>>()?;+ Ok(Polygon::new(rings.remove(0), rings))+ }++ fn ring(&mut self, endian: Endian) -> Result<LineString<f64>> {+ let count = self.u32(endian)? as usize;+ if count < 4 {+ return Err(invalid("GHSL polygon ring is too short"));+ }+ let coordinates = (0..count)+ .map(|_| {+ let x = self.f64(endian)?;+ let y = self.f64(endian)?;+ let coordinate = mollweide_to_wgs84(x, y)?;+ Ok(Coord {+ x: coordinate.longitude,+ y: coordinate.latitude,+ })+ })+ .collect::<Result<Vec<_>>>()?;+ Ok(LineString::new(coordinates))+ }++ fn header(&mut self, expected_type: u32) -> Result<Endian> {+ let endian = match self.byte()? {+ 0 => Endian::Big,+ 1 => Endian::Little,+ _ => return Err(invalid("invalid GHSL WKB byte order")),+ };+ let geometry_type = self.u32(endian)?;+ if geometry_type != expected_type {+ return Err(invalid(format!(+ "unexpected GHSL WKB geometry type {geometry_type}"+ )));+ }+ Ok(endian)+ }++ fn byte(&mut self) -> Result<u8> {+ let byte = *self+ .bytes+ .get(self.position)+ .ok_or_else(|| invalid("truncated GHSL WKB geometry"))?;+ self.position += 1;+ Ok(byte)+ }++ fn u32(&mut self, endian: Endian) -> Result<u32> {+ let bytes = self.take::<4>()?;+ Ok(match endian {+ Endian::Little => u32::from_le_bytes(bytes),+ Endian::Big => u32::from_be_bytes(bytes),+ })+ }++ fn f64(&mut self, endian: Endian) -> Result<f64> {+ let bytes = self.take::<8>()?;+ Ok(match endian {+ Endian::Little => f64::from_le_bytes(bytes),+ Endian::Big => f64::from_be_bytes(bytes),+ })+ }++ fn take<const N: usize>(&mut self) -> Result<[u8; N]> {+ let end = self+ .position+ .checked_add(N)+ .ok_or_else(|| invalid("truncated GHSL WKB geometry"))?;+ let bytes = self+ .bytes+ .get(self.position..end)+ .ok_or_else(|| invalid("truncated GHSL WKB geometry"))?+ .try_into()+ .expect("fixed-length slice");+ self.position = end;+ Ok(bytes)+ }+}++#[derive(Clone, Copy)]+enum Endian {+ Little,+ Big,+}++fn mollweide_to_wgs84(x: f64, y: f64) -> Result<Coordinate> {+ let theta = (y / (MOLLWEIDE_RADIUS * SQRT_2)).asin();+ let latitude = ((2.0 * theta + (2.0 * theta).sin()) / PI).asin();+ let longitude = PI * x / (2.0 * MOLLWEIDE_RADIUS * SQRT_2 * theta.cos());+ let coordinate = Coordinate::new(latitude.to_degrees(), longitude.to_degrees());+ coordinate+ .is_valid()+ .then_some(coordinate)+ .ok_or_else(|| invalid("GHSL geometry has an invalid coordinate"))+}++fn push_count(output: &mut Vec<u8>, value: usize, message: &str) -> Result<()> {+ push_u16(output, u16::try_from(value).map_err(|_| invalid(message))?);+ Ok(())+}++fn push_u16(output: &mut Vec<u8>, value: u16) {+ output.extend_from_slice(&value.to_le_bytes());+}++fn push_len(output: &mut Vec<u8>, value: usize) -> Result<()> {+ output.extend_from_slice(+ &u32::try_from(value)+ .map_err(|_| invalid("urban center index exceeds 4 GiB"))?+ .to_le_bytes(),+ );+ Ok(())+}++#[cfg(test)]+mod tests {+ use ente_location::CountryCode;++ use super::*;++ #[test]+ fn encoded_geometry_preserves_polygons_holes_and_metadata() {+ let centers = vec![+ center(+ 7,+ "Donut",+ vec![+ Polygon::new(square(0.0, 0.0, 2.0, 2.0), vec![square(0.5, 0.5, 1.5, 1.5)]),+ Polygon::new(square(5.0, 5.0, 6.0, 6.0), Vec::new()),+ ],+ ),+ center(+ 8,+ "Neighbour",+ vec![Polygon::new(square(3.0, 3.0, 4.0, 4.0), Vec::new())],+ ),+ center(+ 7,+ "Donut",+ vec![Polygon::new(square(7.0, 7.0, 8.0, 8.0), Vec::new())],+ ),+ ];+ let countries = BTreeMap::from([(*b"AA", "Testland".to_owned())]);+ let index = UrbanCenterIndex::from_bytes(encode(¢ers, &countries).unwrap()).unwrap();+ let coordinates = [+ Coordinate::new(0.25, 0.25),+ Coordinate::new(1.0, 1.0),+ Coordinate::new(3.5, 3.5),+ Coordinate::new(-0.01, -0.01),+ Coordinate::new(5.5, 5.5),+ Coordinate::new(91.0, 0.0),+ Coordinate::new(7.5, 7.5),+ ];++ let matches = index.match_coordinates(&coordinates, "");++ assert_eq!(matches.len(), 2);+ assert_eq!(matches[0].city.name, "Donut");+ assert_eq!(matches[0].city.source_id, 7);+ assert_eq!(+ matches[0].city.country_code,+ CountryCode::from_bytes(*b"AA").unwrap()+ );+ assert_eq!(matches[0].coordinate_indices, [0, 4, 6]);+ assert_eq!(matches[1].city.name, "Neighbour");+ assert_eq!(matches[1].coordinate_indices, [2]);++ let filtered = index.match_coordinates(&coordinates, "NEIGH");+ assert_eq!(filtered.len(), 1);+ assert_eq!(filtered[0].city.name, "Neighbour");+ assert_eq!(filtered[0].coordinate_indices, [2]);+ }++ #[test]+ fn inverse_mollweide_matches_published_apia_centroid() {+ let coordinate = mollweide_to_wgs84(-16_907_128.97, -1_704_473.775).unwrap();+ assert!((coordinate.latitude - -13.836858).abs() < 0.000001);+ assert!((coordinate.longitude - -171.790524).abs() < 0.000001);+ }++ #[test]+ fn canonical_name_city_uses_a_nearby_matching_place() {+ let cities = [city(1, 47.5, 8.5), city(2, 47.3667, 8.55)];++ assert_eq!(+ canonical_name_city(&cities, Coordinate::new(47.37, 8.54))+ .unwrap()+ .source_id,+ 2+ );+ assert!(canonical_name_city(&cities, Coordinate::new(0.0, 0.0)).is_none());+ }++ #[test]+ fn decodes_geopackage_multipolygon_wkb() {+ let mut bytes = b"GP\0\x01".to_vec();+ bytes.extend_from_slice(&54_009_i32.to_le_bytes());+ bytes.push(1);+ bytes.extend_from_slice(&6_u32.to_le_bytes());+ bytes.extend_from_slice(&1_u32.to_le_bytes());+ bytes.push(1);+ bytes.extend_from_slice(&3_u32.to_le_bytes());+ bytes.extend_from_slice(&1_u32.to_le_bytes());+ bytes.extend_from_slice(&4_u32.to_le_bytes());+ for (x, y) in [(0.0_f64, 0.0_f64), (1000.0, 0.0), (0.0, 1000.0), (0.0, 0.0)] {+ bytes.extend_from_slice(&x.to_le_bytes());+ bytes.extend_from_slice(&y.to_le_bytes());+ }++ let geometry = decode_geopackage_geometry(&bytes).unwrap();++ assert_eq!(geometry.0.len(), 1);+ assert_eq!(geometry.0[0].exterior().0.len(), 4);+ assert!(geometry.0[0].exterior().0[1].x > 0.0);+ assert!(geometry.0[0].exterior().0[2].y > 0.0);+ }++ fn center(source_id: u32, name: &str, polygons: Vec<Polygon<f64>>) -> SourceCenter {+ SourceCenter {+ source_id,+ name: name.to_owned(),+ country_code: *b"AA",+ centroid: Coordinate::new(0.25, 0.25),+ geometry: MultiPolygon::new(polygons),+ }+ }++ fn city(source_id: u32, latitude: f64, longitude: f64) -> City {+ City {+ point_index: source_id,+ source_id,+ name: "Example".to_owned(),+ country_name: "Testland".to_owned(),+ country_code: CountryCode::from_bytes(*b"AA").unwrap(),+ latitude,+ longitude,+ rank: 0,+ }+ }++ fn square(minimum_x: f64, minimum_y: f64, maximum_x: f64, maximum_y: f64) -> LineString<f64> {+ LineString::from(vec![+ (minimum_x, minimum_y),+ (maximum_x, minimum_y),+ (maximum_x, maximum_y),+ (minimum_x, maximum_y),+ (minimum_x, minimum_y),+ ])+ }+}diff --git a/rust/crates/location/src/city.rs b/rust/crates/location/src/city.rsindex c472671dc1..b7b4d77a2a 100644--- a/rust/crates/location/src/city.rs+++ b/rust/crates/location/src/city.rs@@ -10,13 +10,14 @@ use crate::binary::{f32_at, range, u16_at, u24_at, u32_at};use crate::{Coordinate, CountryCode, Error};const MAGIC: &[u8; 4] = b"CITY";-const VERSION: u16 = 1;-const HEADER_LEN: usize = 72;+const VERSION: u16 = 2;+const HEADER_LEN: usize = 76;const NODE_LEN: usize = 3;const POINT_LEN: usize = 15;const NAME_OFFSET_LEN: usize = 3;const MAX_U24: usize = 0x00ff_ffff;const MAX_CATCHMENT_KM: f64 = 30.0;+const MAX_OVERRIDE_DISTANCE_KM: f64 = 3.0;const SECTION: &str = "city index";#[derive(Clone, Debug, PartialEq)]@@ -40,6 +41,7 @@ pub struct CityMatch {#[derive(Clone, Copy, Debug)]struct Layout {point_count: usize,+ override_count: usize,name_count: usize,country_count: usize,nodes: usize,@@ -69,6 +71,12 @@ struct SearchBounds {maximum_longitude: f64,}+#[derive(Clone, Copy)]+struct Tree {+ nodes: usize,+ node_count: usize,+}+impl SearchBounds {const fn new(minimum_latitude: f64,@@ -85,13 +93,13 @@ impl SearchBounds {}}-struct SearchPattern {+pub(crate) struct SearchPattern {characters: Vec<char>,fallback: Vec<usize>,}impl SearchPattern {- fn new(query: &str) -> Option<Self> {+ pub(crate) fn new(query: &str) -> Option<Self> {let characters: Vec<char> = search_characters(query).collect();if characters.is_empty() {return None;@@ -113,7 +121,7 @@ impl SearchPattern {})}- fn matches(&self, value: &str) -> bool {+ pub(crate) fn matches(&self, value: &str) -> bool {let mut matched = 0;for character in search_characters(value) {while matched > 0 && character != self.characters[matched] {@@ -137,6 +145,10 @@ fn search_characters(value: &str) -> impl Iterator<Item = char> + '_ {.flat_map(char::to_lowercase)}+pub fn normalize_search_text(value: &str) -> String {+ search_characters(value).collect()+}+pub struct CityIndex {bytes: Box<[u8]>,layout: Layout,@@ -186,6 +198,29 @@ impl CityIndex {}pub fn match_coordinates(&self, coordinates: &[Coordinate], query: &str) -> Vec<CityMatch> {+ self.match_coordinates_in(coordinates, query, self.tree(), MAX_CATCHMENT_KM)+ }++ pub fn match_override_coordinates(+ &self,+ coordinates: &[Coordinate],+ query: &str,+ ) -> Vec<CityMatch> {+ self.match_coordinates_in(+ coordinates,+ query,+ self.override_tree(),+ MAX_OVERRIDE_DISTANCE_KM,+ )+ }++ fn match_coordinates_in(+ &self,+ coordinates: &[Coordinate],+ query: &str,+ tree: Tree,+ maximum_distance_km: f64,+ ) -> Vec<CityMatch> {if coordinates.is_empty() {return Vec::new();}@@ -202,7 +237,13 @@ impl CityIndex {if !coordinate.is_valid() {continue;}- self.range_around(coordinate, MAX_CATCHMENT_KM, &mut candidates, &mut stack);+ self.range_around(+ tree,+ coordinate,+ maximum_distance_km,+ &mut candidates,+ &mut stack,+ );eligible.clear();for &point_index in &candidates {let point = self.point(point_index);@@ -219,7 +260,7 @@ impl CityIndex {f64::from(point.longitude),);let rank = self.rank(point_index);- if distance_km <= catchment_km(rank) {+ if distance_km <= maximum_distance_km.min(catchment_km(rank)) {eligible.push(Candidate {point: point_index,distance_km,@@ -271,6 +312,7 @@ impl CityIndex {fn range_around(&self,+ tree: Tree,coordinate: Coordinate,maximum_distance_km: f64,result: &mut Vec<usize>,@@ -284,6 +326,7 @@ impl CityIndex {let maximum_latitude = (coordinate.latitude + latitude_delta).min(90.0);if longitude_delta >= 180.0 {self.range_into(+ tree,SearchBounds::new(minimum_latitude, maximum_latitude, -180.0, 180.0),result,stack,@@ -295,6 +338,7 @@ impl CityIndex {let maximum_longitude = coordinate.longitude + longitude_delta;if minimum_longitude < -180.0 {self.range_into(+ tree,SearchBounds::new(minimum_latitude,maximum_latitude,@@ -305,6 +349,7 @@ impl CityIndex {stack,);self.range_into(+ tree,SearchBounds::new(minimum_latitude,maximum_latitude,@@ -316,11 +361,13 @@ impl CityIndex {);} else if maximum_longitude > 180.0 {self.range_into(+ tree,SearchBounds::new(minimum_latitude, maximum_latitude, minimum_longitude, 180.0),result,stack,);self.range_into(+ tree,SearchBounds::new(minimum_latitude,maximum_latitude,@@ -332,6 +379,7 @@ impl CityIndex {);} else {self.range_into(+ tree,SearchBounds::new(minimum_latitude,maximum_latitude,@@ -346,17 +394,18 @@ impl CityIndex {fn range_into(&self,+ tree: Tree,bounds: SearchBounds,result: &mut Vec<usize>,stack: &mut Vec<(usize, usize, u8)>,) {- if self.layout.point_count == 0 {+ if tree.node_count == 0 {return;}stack.clear();- stack.push((0, self.layout.point_count, 0));+ stack.push((0, tree.node_count, 0));while let Some((node_index, subtree_size, axis)) = stack.pop() {- let point_index = self.node(node_index);+ let point_index = self.node(tree, node_index);let point = self.point(point_index);let latitude = f64::from(point.latitude);let longitude = f64::from(point.longitude);@@ -401,8 +450,22 @@ impl CityIndex {}}- fn node(&self, index: usize) -> usize {- u24_at(&self.bytes, self.layout.nodes + index * NODE_LEN).expect("validated node") as usize+ fn tree(&self) -> Tree {+ Tree {+ nodes: self.layout.nodes,+ node_count: self.layout.point_count,+ }+ }++ fn override_tree(&self) -> Tree {+ Tree {+ nodes: self.layout.nodes + self.layout.point_count * NODE_LEN,+ node_count: self.layout.override_count,+ }+ }++ fn node(&self, tree: Tree, index: usize) -> usize {+ u24_at(&self.bytes, tree.nodes + index * NODE_LEN).expect("validated node") as usize}fn point(&self, index: usize) -> Point {@@ -495,6 +558,7 @@ fn validate(bytes: &[u8]) -> crate::Result<Layout> {let layout = Layout {point_count: read_usize(bytes, 8)?,+ override_count: read_usize(bytes, 72)?,name_count: read_usize(bytes, 12)?,country_count: usize::from(read_u16(bytes, 16)?),nodes: read_usize(bytes, 24)?,@@ -515,13 +579,29 @@ fn validate(bytes: &[u8]) -> crate::Result<Layout> {validate_sections(bytes, layout, declared_length)?;validate_strings(bytes, layout)?;validate_points(bytes, layout)?;- validate_tree(bytes, layout)?;+ validate_tree(+ bytes,+ layout,+ Tree {+ nodes: layout.nodes,+ node_count: layout.point_count,+ },+ )?;+ validate_tree(+ bytes,+ layout,+ Tree {+ nodes: layout.nodes + layout.point_count * NODE_LEN,+ node_count: layout.override_count,+ },+ )?;Ok(layout)}fn validate_sections(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {if layout.point_count > MAX_U24|| layout.name_count > MAX_U24+ || layout.override_count > layout.point_count|| layout.country_count > 256|| (layout.point_count > 0 && (layout.name_count == 0 || layout.country_count == 0))|| layout.rank_ends[3] > layout.point_count@@ -531,6 +611,8 @@ fn validate_sections(bytes: &[u8], layout: Layout, declared_length: usize) -> cr}let nodes = range(layout.nodes, layout.point_count, NODE_LEN).ok_or(invalid("node section overflow"))?;+ let override_nodes = range(nodes.end, layout.override_count, NODE_LEN)+ .ok_or(invalid("override-node section overflow"))?;let points = range(layout.points, layout.point_count, POINT_LEN).ok_or(invalid("point section overflow"))?;let name_offsets = offset_table_range(layout.name_offsets, layout.name_count, NAME_OFFSET_LEN)?;@@ -554,7 +636,7 @@ fn validate_sections(bytes: &[u8], layout: Layout, declared_length: usize) -> cr.ok_or(invalid("country section overflow"))?;if layout.nodes != HEADER_LEN- || layout.points != nodes.end+ || layout.points != override_nodes.end|| layout.name_offsets != points.end|| layout.names != name_offsets.end|| layout.country_offsets != names_end@@ -587,9 +669,7 @@ fn validate_strings(bytes: &[u8], layout: Layout) -> crate::Result<()> {)?;for index in 0..layout.name_count {let value = table_entry_u24(bytes, layout.name_offsets, layout.names, index);- if let Some((name, alias)) = value.split_once('\0')- && (name.is_empty() || alias.is_empty() || alias.contains('\0'))- {+ if value.split('\0').any(str::is_empty) {return Err(invalid("invalid city name"));}}@@ -630,23 +710,23 @@ fn validate_points(bytes: &[u8], layout: Layout) -> crate::Result<()> {Ok(())}-fn validate_tree(bytes: &[u8], layout: Layout) -> crate::Result<()> {- if layout.point_count == 0 {+fn validate_tree(bytes: &[u8], layout: Layout, tree: Tree) -> crate::Result<()> {+ if tree.node_count == 0 {return Ok(());}let mut visited_points = vec![false; layout.point_count];- let mut stack = vec![(0usize, layout.point_count, 0u8, -90.0, 90.0, -180.0, 180.0)];+ let mut stack = vec![(0usize, tree.node_count, 0u8, -90.0, 90.0, -180.0, 180.0)];while let Some((node_index, size, axis, min_lat, max_lat, min_lng, max_lng)) = stack.pop() {if size == 0|| node_index.checked_add(size)- .is_none_or(|end| end > layout.point_count)+ .is_none_or(|end| end > tree.node_count){return Err(invalid("invalid tree layout"));}- let point = u24_at(bytes, layout.nodes + node_index * NODE_LEN)- .ok_or(invalid("truncated"))? as usize;+ let point =+ u24_at(bytes, tree.nodes + node_index * NODE_LEN).ok_or(invalid("truncated"))? as usize;if point >= layout.point_count || visited_points[point] {return Err(invalid("invalid tree node"));}@@ -709,7 +789,7 @@ fn validate_tree(bytes: &[u8], layout: Layout) -> crate::Result<()> {}}}- if visited_points.iter().any(|&visited| !visited) {+ if tree.node_count == layout.point_count && visited_points.iter().any(|&visited| !visited) {return Err(invalid("tree does not cover every point"));}Ok(())@@ -815,7 +895,7 @@ fn catchment_km(rank: u8) -> f64 {[8.0, 10.0, 14.0, 20.0, 30.0][usize::from(rank)]}-fn distance_km(lat1: f64, lng1: f64, lat2: f64, lng2: f64) -> f64 {+pub(crate) fn distance_km(lat1: f64, lng1: f64, lat2: f64, lng2: f64) -> f64 {let lat1 = lat1.to_radians();let lat2 = lat2.to_radians();let delta_latitude = lat2 - lat1;diff --git a/rust/crates/location/src/city_resolver.rs b/rust/crates/location/src/city_resolver.rsnew file mode 100644index 0000000000..666c924981--- /dev/null+++ b/rust/crates/location/src/city_resolver.rs@@ -0,0 +1,242 @@+use std::collections::BTreeMap;++use crate::city::distance_km;+use crate::{City, CityIndex, CityMatch, Coordinate, UrbanCenterIndex};++const OVERRIDE_DISTANCE_MARGIN_KM: f64 = 1.0;++impl UrbanCenterIndex {+ pub fn match_coordinates_with_cities(+ &self,+ cities: &CityIndex,+ coordinates: &[Coordinate],+ query: &str,+ ) -> Vec<CityMatch> {+ let mut matches = self.match_coordinates(coordinates, query);+ if matches.is_empty() {+ return cities.match_coordinates(coordinates, query);+ }+ let mut urban_groups = vec![usize::MAX; coordinates.len()];+ for (group_index, group) in matches.iter().enumerate() {+ for &index in &group.coordinate_indices {+ urban_groups[index as usize] = group_index;+ }+ }+ let covered_indices: Vec<u32> = urban_groups+ .iter()+ .enumerate()+ .filter_map(|(index, &group)| (group != usize::MAX).then_some(index as u32))+ .collect();+ let mut overrides = cities+ .match_override_coordinates(&coordinates_at(coordinates, &covered_indices), query);+ retain_closer_overrides(+ &mut overrides,+ &covered_indices,+ coordinates,+ &matches,+ &urban_groups,+ );+ replace_matches(&mut matches, &covered_indices, overrides, coordinates.len());+ let missing_indices: Vec<u32> = urban_groups+ .iter()+ .enumerate()+ .filter_map(|(index, &group)| (group == usize::MAX).then_some(index as u32))+ .collect();+ if missing_indices.is_empty() {+ return matches;+ }+ let nearby =+ cities.match_coordinates(&coordinates_at(coordinates, &missing_indices), query);+ append_fallback(&mut matches, &missing_indices, nearby);+ matches+ }+}++fn coordinates_at(coordinates: &[Coordinate], indices: &[u32]) -> Vec<Coordinate> {+ indices+ .iter()+ .map(|&index| coordinates[index as usize])+ .collect()+}++fn retain_closer_overrides(+ overrides: &mut Vec<CityMatch>,+ original_indices: &[u32],+ coordinates: &[Coordinate],+ urban: &[CityMatch],+ urban_groups: &[usize],+) {+ for group in overrides.iter_mut() {+ let city = &group.city;+ group.coordinate_indices.retain(|&relative_index| {+ let index = original_indices[relative_index as usize] as usize;+ distance_to_city(coordinates[index], city) + OVERRIDE_DISTANCE_MARGIN_KM+ < distance_to_city(coordinates[index], &urban[urban_groups[index]].city)+ });+ }+ overrides.retain(|group| !group.coordinate_indices.is_empty());+}++fn distance_to_city(coordinate: Coordinate, city: &City) -> f64 {+ distance_km(+ coordinate.latitude,+ coordinate.longitude,+ city.latitude,+ city.longitude,+ )+}++fn append_fallback(+ matches: &mut Vec<CityMatch>,+ original_indices: &[u32],+ mut fallback: Vec<CityMatch>,+) {+ remap_indices(&mut fallback, original_indices);+ append_matches(matches, fallback);+}++fn replace_matches(+ matches: &mut Vec<CityMatch>,+ original_indices: &[u32],+ mut replacements: Vec<CityMatch>,+ coordinate_count: usize,+) {+ remap_indices(&mut replacements, original_indices);+ let mut replaced = vec![false; coordinate_count];+ for group in &replacements {+ for &index in &group.coordinate_indices {+ replaced[index as usize] = true;+ }+ }+ for group in matches.iter_mut() {+ group+ .coordinate_indices+ .retain(|&index| !replaced[index as usize]);+ }+ matches.retain(|group| !group.coordinate_indices.is_empty());+ append_matches(matches, replacements);+}++fn remap_indices(matches: &mut [CityMatch], original_indices: &[u32]) {+ for group in matches {+ for index in &mut group.coordinate_indices {+ *index = original_indices[*index as usize];+ }+ }+}++fn append_matches(matches: &mut Vec<CityMatch>, additions: Vec<CityMatch>) {+ let mut positions: BTreeMap<u32, usize> = matches+ .iter()+ .enumerate()+ .map(|(index, group)| (group.city.source_id, index))+ .collect();+ for group in additions {+ if let Some(&position) = positions.get(&group.city.source_id) {+ matches[position]+ .coordinate_indices+ .extend(group.coordinate_indices);+ matches[position].coordinate_indices.sort_unstable();+ } else {+ positions.insert(group.city.source_id, matches.len());+ matches.push(group);+ }+ }+}++#[cfg(test)]+mod tests {+ use crate::CountryCode;++ use super::*;++ #[test]+ fn fallback_indices_are_mapped_back_to_the_original_coordinates() {+ let mut matches = vec![city_match(1, "Metropolis", vec![0, 2])];+ let nearby = vec![+ city_match(2, "Small Town", vec![0]),+ city_match(3, "Rural Place", vec![1]),+ ];++ append_fallback(&mut matches, &[1, 3], nearby);++ assert_eq!(label(&matches, 0), "Metropolis");+ assert_eq!(label(&matches, 1), "Small Town");+ assert_eq!(label(&matches, 2), "Metropolis");+ assert_eq!(label(&matches, 3), "Rural Place");+ }++ #[test]+ fn fallback_with_the_same_canonical_city_is_merged() {+ let mut matches = vec![city_match(1, "Zürich", vec![0])];+ let nearby = vec![city_match(1, "Zürich", vec![0])];++ append_fallback(&mut matches, &[1], nearby);++ assert_eq!(matches.len(), 1);+ assert_eq!(matches[0].coordinate_indices, [0, 1]);+ }++ #[test]+ fn curated_city_replaces_the_urban_center_for_its_coordinates() {+ let mut matches = vec![city_match(1, "Metropolis", vec![0, 1, 2])];+ let prominent = vec![city_match(2, "Edge City", vec![1])];++ replace_matches(&mut matches, &[0, 2], prominent, 3);++ assert_eq!(label(&matches, 0), "Metropolis");+ assert_eq!(label(&matches, 1), "Metropolis");+ assert_eq!(label(&matches, 2), "Edge City");+ }++ #[test]+ fn override_must_be_materially_closer_than_the_urban_center() {+ let urban = vec![city_match_at(1, "Metropolis", 0.0, 0.0, vec![0, 1])];+ let mut overrides = vec![city_match_at(2, "Edge City", 0.0, 0.1, vec![0, 1])];++ retain_closer_overrides(+ &mut overrides,+ &[0, 1],+ &[Coordinate::new(0.0, 0.01), Coordinate::new(0.0, 0.09)],+ &urban,+ &[0, 0],+ );++ assert_eq!(overrides[0].coordinate_indices, [1]);+ }++ fn label(matches: &[CityMatch], coordinate_index: u32) -> &str {+ &matches+ .iter()+ .find(|group| group.coordinate_indices.contains(&coordinate_index))+ .unwrap()+ .city+ .name+ }++ fn city_match(source_id: u32, name: &str, coordinate_indices: Vec<u32>) -> CityMatch {+ city_match_at(source_id, name, 0.0, 0.0, coordinate_indices)+ }++ fn city_match_at(+ source_id: u32,+ name: &str,+ latitude: f64,+ longitude: f64,+ coordinate_indices: Vec<u32>,+ ) -> CityMatch {+ CityMatch {+ city: City {+ point_index: 0,+ source_id,+ name: name.to_owned(),+ country_name: String::new(),+ country_code: CountryCode::from_bytes(*b"AA").unwrap(),+ latitude,+ longitude,+ rank: 0,+ },+ coordinate_indices,+ }+ }+}diff --git a/rust/crates/location/src/country.rs b/rust/crates/location/src/country.rsindex 9f2492f1b1..e8cab43f1c 100644--- a/rust/crates/location/src/country.rs+++ b/rust/crates/location/src/country.rs@@ -1,4 +1,5 @@use crate::binary::{array, range, u16_at, u32_at};+use crate::geometry::{Point, Reader};use crate::{CountryCode, Error};const MAGIC: &[u8; 4] = b"CTRY";@@ -135,7 +136,7 @@ impl CountryGeometry {}fn lookup_cell(&self, bytes: &[u8], point: Point) -> crate::Result<Vec<CountryCode>> {- let mut reader = Reader::new(bytes);+ let mut reader = Reader::new(bytes, SECTION);let mut matches = Vec::new();for _ in 0..reader.byte()? {push_unique(&mut matches, self.country_code(reader.byte()?));@@ -157,7 +158,7 @@ impl CountryGeometry {push_unique(&mut matches, country);}}- debug_assert_eq!(reader.position, bytes.len());+ debug_assert!(reader.is_done());Ok(matches)}}@@ -171,66 +172,6 @@ pub(crate) struct PreparedCell {index: usize,}-#[derive(Clone, Copy)]-struct Point {- x: u16,- y: u16,-}--struct Reader<'a> {- bytes: &'a [u8],- position: usize,-}--impl<'a> Reader<'a> {- const fn new(bytes: &'a [u8]) -> Self {- Self { bytes, position: 0 }- }-- fn byte(&mut self) -> crate::Result<u8> {- let value = *self.bytes.get(self.position).ok_or(invalid("truncated"))?;- self.position += 1;- Ok(value)- }-- fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {- let end = self- .position- .checked_add(count)- .ok_or(invalid("truncated"))?;- let bytes = self- .bytes- .get(self.position..end)- .ok_or(invalid("truncated"))?;- self.position = end;- Ok(bytes)- }-- fn ring_contains(&mut self, point: Point) -> crate::Result<bool> {- let count = usize::from(u16::from_le_bytes(- self.take(2)?.try_into().expect("two-byte slice"),- ));- if count == 0 {- return Ok(false);- }- let ring = self.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;- let mut winding = 0_i32;- let mut previous = point_at(ring, count - 1);- for index in 0..count {- let current = point_at(ring, index);- if previous.y <= point.y {- if current.y > point.y && is_left(previous, current, point) > 0 {- winding += 1;- }- } else if current.y <= point.y && is_left(previous, current, point) < 0 {- winding -= 1;- }- previous = current;- }- Ok(winding != 0)- }-}-fn validate(bytes: &[u8]) -> crate::Result<Layout> {if bytes.len() < HEADER_LEN {return Err(invalid("truncated"));@@ -366,7 +307,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat}fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {- let mut reader = Reader::new(bytes);+ let mut reader = Reader::new(bytes, SECTION);for _ in 0..reader.byte()? {validate_country(reader.byte()?, country_count)?;}@@ -375,7 +316,7 @@ fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {skip_rings(&mut reader)?;skip_rings(&mut reader)?;}- if reader.position != bytes.len() {+ if !reader.is_done() {return Err(invalid("cell has trailing bytes"));}Ok(())@@ -383,9 +324,7 @@ fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {fn skip_rings(reader: &mut Reader<'_>) -> crate::Result<()> {for _ in 0..reader.byte()? {- let count = usize::from(u16::from_le_bytes(- reader.take(2)?.try_into().expect("two-byte slice"),- ));+ let count = usize::from(reader.u16()?);reader.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;}Ok(())@@ -399,19 +338,6 @@ fn validate_country(index: u8, country_count: usize) -> crate::Result<()> {}}-fn point_at(ring: &[u8], index: usize) -> Point {- let offset = index * 4;- Point {- x: u16::from_le_bytes([ring[offset], ring[offset + 1]]),- y: u16::from_le_bytes([ring[offset + 2], ring[offset + 3]]),- }-}--fn is_left(start: Point, end: Point, point: Point) -> i64 {- (i64::from(end.x) - i64::from(start.x)) * (i64::from(point.y) - i64::from(start.y))- - (i64::from(point.x) - i64::from(start.x)) * (i64::from(end.y) - i64::from(start.y))-}-fn normalize(value: f64, minimum: f64, width: f64) -> f64 {(value - minimum).rem_euclid(width) + minimum}diff --git a/rust/crates/location/src/geometry.rs b/rust/crates/location/src/geometry.rsnew file mode 100644index 0000000000..ff717bdb5d--- /dev/null+++ b/rust/crates/location/src/geometry.rs@@ -0,0 +1,98 @@+use crate::Error;++#[derive(Clone, Copy)]+pub(crate) struct Point {+ pub x: u16,+ pub y: u16,+}++pub(crate) struct Reader<'a> {+ bytes: &'a [u8],+ position: usize,+ section: &'static str,+}++impl<'a> Reader<'a> {+ pub(crate) const fn new(bytes: &'a [u8], section: &'static str) -> Self {+ Self {+ bytes,+ position: 0,+ section,+ }+ }++ pub(crate) const fn is_done(&self) -> bool {+ self.position == self.bytes.len()+ }++ pub(crate) fn byte(&mut self) -> crate::Result<u8> {+ let value = *self+ .bytes+ .get(self.position)+ .ok_or(self.invalid("truncated"))?;+ self.position += 1;+ Ok(value)+ }++ pub(crate) fn u16(&mut self) -> crate::Result<u16> {+ Ok(u16::from_le_bytes(+ self.take(2)?.try_into().expect("two-byte geometry slice"),+ ))+ }++ pub(crate) fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {+ let end = self+ .position+ .checked_add(count)+ .ok_or(self.invalid("truncated"))?;+ let bytes = self+ .bytes+ .get(self.position..end)+ .ok_or(self.invalid("truncated"))?;+ self.position = end;+ Ok(bytes)+ }++ pub(crate) fn ring_contains(&mut self, point: Point) -> crate::Result<bool> {+ let count = usize::from(self.u16()?);+ if count == 0 {+ return Ok(false);+ }+ let ring = self.take(+ count+ .checked_mul(4)+ .ok_or(self.invalid("ring size overflow"))?,+ )?;+ let mut winding = 0_i32;+ let mut previous = point_at(ring, count - 1);+ for index in 0..count {+ let current = point_at(ring, index);+ if previous.y <= point.y {+ if current.y > point.y && is_left(previous, current, point) > 0 {+ winding += 1;+ }+ } else if current.y <= point.y && is_left(previous, current, point) < 0 {+ winding -= 1;+ }+ previous = current;+ }+ Ok(winding != 0)+ }++ fn invalid(&self, reason: &'static str) -> Error {+ Error::invalid(self.section, reason)+ }+}++fn point_at(ring: &[u8], index: usize) -> Point {+ let offset = index * 4;+ Point {+ x: u16::from_le_bytes([ring[offset], ring[offset + 1]]),+ y: u16::from_le_bytes([ring[offset + 2], ring[offset + 3]]),+ }+}++fn is_left(start: Point, end: Point, point: Point) -> i64 {+ (i64::from(end.x) - i64::from(start.x)) * (i64::from(point.y) - i64::from(start.y))+ - (i64::from(point.x) - i64::from(start.x)) * (i64::from(end.y) - i64::from(start.y))+}diff --git a/rust/crates/location/src/lib.rs b/rust/crates/location/src/lib.rsindex 8b4f69de7b..3451bbb5eb 100644--- a/rust/crates/location/src/lib.rs+++ b/rust/crates/location/src/lib.rs@@ -6,14 +6,18 @@ use std::str::FromStr;mod binary;mod city;+mod city_resolver;pub mod cluster;mod country;mod dispute;mod error;+mod geometry;+mod urban;-pub use city::{City, CityIndex, CityMatch};+pub use city::{City, CityIndex, CityMatch, normalize_search_text};pub use dispute::{CountryView, DisputeMatch, TerritoryId};pub use error::{Error, Result};+pub use urban::UrbanCenterIndex;use country::CountryGeometry;use dispute::DisputeIndex;diff --git a/rust/crates/location/src/urban.rs b/rust/crates/location/src/urban.rsnew file mode 100644index 0000000000..4939b95b95--- /dev/null+++ b/rust/crates/location/src/urban.rs@@ -0,0 +1,513 @@+use std::collections::BTreeMap;+use std::fs;+use std::path::Path;++use crate::binary::{array, f32_at, range, u16_at, u32_at};+use crate::city::SearchPattern;+use crate::geometry::{Point, Reader as GeometryReader};+use crate::{City, CityMatch, Coordinate, CountryCode, Error};++const MAGIC: &[u8; 4] = b"URBN";+const VERSION: u16 = 1;+const HEADER_LEN: usize = 72;+const RECORD_LEN: usize = 41;+const BLOCK_SIZE: usize = 64;+const BLOCK_LEN: usize = 12;+const SECTION: &str = "urban center index";++#[derive(Clone, Copy)]+struct Layout {+ feature_count: usize,+ country_count: usize,+ columns: usize,+ rows: usize,+ block_count: usize,+ nonempty_count: usize,+ reference_count: usize,+ records: usize,+ names: usize,+ geometry: usize,+ country_offsets: usize,+ countries: usize,+ country_codes: usize,+ blocks: usize,+ cell_offsets: usize,+ references: usize,+}++#[derive(Clone, Copy)]+struct Feature {+ minimum_longitude: f32,+ minimum_latitude: f32,+ maximum_longitude: f32,+ maximum_latitude: f32,+ latitude: f32,+ longitude: f32,+ name: usize,+ geometry_start: usize,+ geometry_end: usize,+ source_id: u32,+ country: usize,+}++pub struct UrbanCenterIndex {+ bytes: Box<[u8]>,+ layout: Layout,+}++impl UrbanCenterIndex {+ pub fn from_path(path: impl AsRef<Path>) -> crate::Result<Self> {+ Self::from_bytes(fs::read(path)?)+ }++ pub fn from_bytes(bytes: impl Into<Box<[u8]>>) -> crate::Result<Self> {+ let bytes = bytes.into();+ let layout = validate(&bytes)?;+ Ok(Self { bytes, layout })+ }++ pub const fn len(&self) -> usize {+ self.layout.feature_count+ }++ pub const fn is_empty(&self) -> bool {+ self.len() == 0+ }++ pub fn match_coordinates(&self, coordinates: &[Coordinate], query: &str) -> Vec<CityMatch> {+ let matching: Option<Vec<bool>> = SearchPattern::new(query).map(|pattern| {+ (0..self.layout.feature_count)+ .map(|feature| pattern.matches(self.name(self.feature(feature).name)))+ .collect()+ });+ if matching+ .as_ref()+ .is_some_and(|matching| !matching.iter().any(|&matched| matched))+ {+ return Vec::new();+ }+ let mut matches = BTreeMap::<usize, Vec<u32>>::new();+ for (coordinate_index, &coordinate) in coordinates.iter().enumerate() {+ if let Some(feature) = self.lookup(coordinate, matching.as_deref()) {+ matches+ .entry(feature)+ .or_default()+ .push(coordinate_index as u32);+ }+ }+ let mut groups = BTreeMap::<u32, CityMatch>::new();+ for (feature, coordinate_indices) in matches {+ let city = self.city(feature);+ groups+ .entry(city.source_id)+ .and_modify(|group| {+ group+ .coordinate_indices+ .extend_from_slice(&coordinate_indices);+ group.coordinate_indices.sort_unstable();+ })+ .or_insert(CityMatch {+ city,+ coordinate_indices,+ });+ }+ groups.into_values().collect()+ }++ fn lookup(&self, coordinate: Coordinate, matching: Option<&[bool]>) -> Option<usize> {+ if !coordinate.is_valid() {+ return None;+ }+ let column = (((coordinate.longitude + 180.0) * self.layout.columns as f64 / 360.0).floor()+ as usize)+ .min(self.layout.columns - 1);+ let row = ((self.layout.rows as f64 * (90.0 - coordinate.latitude) / 180.0).ceil()+ as usize)+ .saturating_sub(1);+ let dense = self.dense_cell_index(row * self.layout.columns + column)?;+ let start = self.cell_offset(dense);+ let end = self.cell_offset(dense + 1);+ for reference in start..end {+ let feature = usize::from(+ u16_at(&self.bytes, self.layout.references + reference * 2)+ .expect("validated reference"),+ );+ if matching.is_some_and(|matching| !matching[feature]) {+ continue;+ }+ if self.contains(feature, coordinate) {+ return Some(feature);+ }+ }+ None+ }++ fn contains(&self, index: usize, coordinate: Coordinate) -> bool {+ let feature = self.feature(index);+ let longitude = coordinate.longitude as f32;+ let latitude = coordinate.latitude as f32;+ if longitude < feature.minimum_longitude+ || longitude > feature.maximum_longitude+ || latitude < feature.minimum_latitude+ || latitude > feature.maximum_latitude+ {+ return false;+ }+ let point = Point {+ x: quantize(+ coordinate.longitude,+ f64::from(feature.minimum_longitude),+ f64::from(feature.maximum_longitude),+ ),+ y: quantize(+ coordinate.latitude,+ f64::from(feature.minimum_latitude),+ f64::from(feature.maximum_latitude),+ ),+ };+ let mut reader = GeometryReader::new(+ &self.bytes[feature.geometry_start..feature.geometry_end],+ SECTION,+ );+ for _ in 0..reader.u16().expect("validated polygon count") {+ let ring_count = reader.u16().expect("validated ring count");+ let mut inside = reader.ring_contains(point).expect("validated ring");+ for _ in 1..ring_count {+ if reader.ring_contains(point).expect("validated ring") {+ inside = false;+ }+ }+ if inside {+ return true;+ }+ }+ false+ }++ fn city(&self, index: usize) -> City {+ let feature = self.feature(index);+ City {+ point_index: index as u32,+ source_id: feature.source_id,+ name: self.name(feature.name).to_owned(),+ country_name: self.country(feature.country).to_owned(),+ country_code: self.country_code(feature.country),+ latitude: f64::from(feature.latitude),+ longitude: f64::from(feature.longitude),+ rank: 4,+ }+ }++ fn feature(&self, index: usize) -> Feature {+ let offset = self.layout.records + index * RECORD_LEN;+ Feature {+ minimum_longitude: f32_at(&self.bytes, offset).expect("validated feature"),+ minimum_latitude: f32_at(&self.bytes, offset + 4).expect("validated feature"),+ maximum_longitude: f32_at(&self.bytes, offset + 8).expect("validated feature"),+ maximum_latitude: f32_at(&self.bytes, offset + 12).expect("validated feature"),+ latitude: f32_at(&self.bytes, offset + 16).expect("validated feature"),+ longitude: f32_at(&self.bytes, offset + 20).expect("validated feature"),+ name: u32_at(&self.bytes, offset + 24).expect("validated feature") as usize,+ geometry_start: u32_at(&self.bytes, offset + 28).expect("validated feature") as usize,+ geometry_end: u32_at(&self.bytes, offset + 32).expect("validated feature") as usize,+ source_id: u32_at(&self.bytes, offset + 36).expect("validated feature"),+ country: usize::from(self.bytes[offset + 40]),+ }+ }++ fn name(&self, offset: usize) -> &str {+ let end = self.bytes[offset..self.layout.geometry]+ .iter()+ .position(|&byte| byte == 0)+ .map(|length| offset + length)+ .expect("validated name terminator");+ std::str::from_utf8(&self.bytes[offset..end]).expect("validated name")+ }++ fn country(&self, index: usize) -> &str {+ let start = u32_at(&self.bytes, self.layout.country_offsets + index * 4)+ .expect("validated country offset") as usize;+ let end = u32_at(&self.bytes, self.layout.country_offsets + (index + 1) * 4)+ .expect("validated country offset") as usize;+ std::str::from_utf8(&self.bytes[self.layout.countries + start..self.layout.countries + end])+ .expect("validated country")+ }++ fn country_code(&self, index: usize) -> CountryCode {+ let offset = self.layout.country_codes + index * 2;+ CountryCode::from_validated([self.bytes[offset], self.bytes[offset + 1]])+ }++ fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {+ let block_index = cell_index / BLOCK_SIZE;+ let bit_index = cell_index % BLOCK_SIZE;+ let offset = self.layout.blocks + block_index * BLOCK_LEN;+ let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));+ let bit = 1_u64 << bit_index;+ (mask & bit != 0).then(|| {+ let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;+ first + (mask & bit.wrapping_sub(1)).count_ones() as usize+ })+ }++ fn cell_offset(&self, dense_index: usize) -> usize {+ u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)+ .expect("validated cell offset") as usize+ }+}++fn validate(bytes: &[u8]) -> crate::Result<Layout> {+ if bytes.len() < HEADER_LEN || &bytes[..4] != MAGIC {+ return Err(invalid("truncated or invalid magic"));+ }+ let version = u16_at(bytes, 4).ok_or(invalid("truncated"))?;+ if version != VERSION {+ return Err(Error::invalid(+ SECTION,+ format!("unsupported format version {version}"),+ ));+ }+ if usize::from(u16_at(bytes, 6).ok_or(invalid("truncated"))?) != HEADER_LEN+ || usize::from(u16_at(bytes, 18).ok_or(invalid("truncated"))?) != BLOCK_SIZE+ {+ return Err(invalid("unexpected header or block length"));+ }+ let layout = Layout {+ feature_count: read_usize(bytes, 8)?,+ country_count: usize::from(u16_at(bytes, 12).ok_or(invalid("truncated"))?),+ columns: usize::from(u16_at(bytes, 14).ok_or(invalid("truncated"))?),+ rows: usize::from(u16_at(bytes, 16).ok_or(invalid("truncated"))?),+ block_count: read_usize(bytes, 20)?,+ nonempty_count: read_usize(bytes, 24)?,+ records: read_usize(bytes, 28)?,+ names: read_usize(bytes, 32)?,+ geometry: read_usize(bytes, 36)?,+ country_offsets: read_usize(bytes, 40)?,+ countries: read_usize(bytes, 44)?,+ country_codes: read_usize(bytes, 48)?,+ blocks: read_usize(bytes, 52)?,+ cell_offsets: read_usize(bytes, 56)?,+ references: read_usize(bytes, 60)?,+ reference_count: read_usize(bytes, 68)?,+ };+ let declared_length = read_usize(bytes, 64)?;+ validate_layout(bytes, layout, declared_length)?;+ Ok(layout)+}++fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {+ let cell_count = layout+ .columns+ .checked_mul(layout.rows)+ .ok_or(invalid("cell count overflow"))?;+ if layout.feature_count == 0+ || layout.feature_count > usize::from(u16::MAX)+ || layout.country_count == 0+ || layout.country_count > 256+ || layout.columns == 0+ || layout.rows == 0+ || layout.block_count != cell_count.div_ceil(BLOCK_SIZE)+ || layout.nonempty_count > cell_count+ {+ return Err(invalid("invalid index counts"));+ }+ let records = range(layout.records, layout.feature_count, RECORD_LEN)+ .ok_or(invalid("record section overflow"))?;+ let country_offsets = range(layout.country_offsets, layout.country_count + 1, 4)+ .ok_or(invalid("country offset section overflow"))?;+ let country_codes = range(layout.country_codes, layout.country_count, 2)+ .ok_or(invalid("country code section overflow"))?;+ let blocks = range(layout.blocks, layout.block_count, BLOCK_LEN)+ .ok_or(invalid("block section overflow"))?;+ let cell_offsets = range(layout.cell_offsets, layout.nonempty_count + 1, 4)+ .ok_or(invalid("cell offset section overflow"))?;+ let references = range(layout.references, layout.reference_count, 2)+ .ok_or(invalid("reference section overflow"))?;+ if layout.records != HEADER_LEN+ || layout.names != records.end+ || layout.names > layout.geometry+ || layout.geometry > layout.country_offsets+ || layout.countries != country_offsets.end+ || layout.countries > layout.country_codes+ || layout.blocks != country_codes.end+ || layout.cell_offsets != blocks.end+ || layout.references != cell_offsets.end+ || references.end != bytes.len()+ || declared_length != bytes.len()+ {+ return Err(invalid("sections are not contiguous"));+ }+ validate_countries(bytes, layout)?;+ validate_features(bytes, layout)?;+ validate_grid(bytes, layout, cell_count)?;+ Ok(())+}++fn validate_countries(bytes: &[u8], layout: Layout) -> crate::Result<()> {+ let country_bytes = layout.country_codes - layout.countries;+ let mut previous = 0;+ let mut seen_codes = [false; 26 * 26];+ for index in 0..layout.country_count {+ let offset = u32_at(bytes, layout.country_offsets + index * 4)+ .ok_or(invalid("truncated country offset"))? as usize;+ let end = u32_at(bytes, layout.country_offsets + (index + 1) * 4)+ .ok_or(invalid("truncated country offset"))? as usize;+ if offset != previous || end < offset || end > country_bytes {+ return Err(invalid("invalid country offsets"));+ }+ std::str::from_utf8(&bytes[layout.countries + offset..layout.countries + end])+ .map_err(|_| invalid("country name is not UTF-8"))?;+ previous = end;++ let code_offset = layout.country_codes + index * 2;+ let code = [bytes[code_offset], bytes[code_offset + 1]];+ if !code.iter().all(u8::is_ascii_uppercase) {+ return Err(invalid("invalid country code"));+ }+ let slot = usize::from(code[0] - b'A') * 26 + usize::from(code[1] - b'A');+ if seen_codes[slot] {+ return Err(invalid("duplicate country code"));+ }+ seen_codes[slot] = true;+ }+ if previous != country_bytes {+ return Err(invalid("last country offset is not terminal"));+ }+ Ok(())+}++fn validate_features(bytes: &[u8], layout: Layout) -> crate::Result<()> {+ for index in 0..layout.feature_count {+ let offset = layout.records + index * RECORD_LEN;+ let values = [+ f32_at(bytes, offset).ok_or(invalid("truncated feature"))?,+ f32_at(bytes, offset + 4).ok_or(invalid("truncated feature"))?,+ f32_at(bytes, offset + 8).ok_or(invalid("truncated feature"))?,+ f32_at(bytes, offset + 12).ok_or(invalid("truncated feature"))?,+ f32_at(bytes, offset + 16).ok_or(invalid("truncated feature"))?,+ f32_at(bytes, offset + 20).ok_or(invalid("truncated feature"))?,+ ];+ if values.iter().any(|value| !value.is_finite())+ || !(-180.0..=180.0).contains(&values[0])+ || !(-90.0..=90.0).contains(&values[1])+ || !(-180.0..=180.0).contains(&values[2])+ || !(-90.0..=90.0).contains(&values[3])+ || values[0] >= values[2]+ || values[1] >= values[3]+ || !(-90.0..=90.0).contains(&values[4])+ || !(-180.0..=180.0).contains(&values[5])+ {+ return Err(invalid("feature has invalid coordinates"));+ }+ let name = u32_at(bytes, offset + 24).ok_or(invalid("truncated feature"))? as usize;+ if !(layout.names..layout.geometry).contains(&name) {+ return Err(invalid("feature name is outside the name section"));+ }+ let end = bytes[name..layout.geometry]+ .iter()+ .position(|&byte| byte == 0)+ .ok_or(invalid("unterminated feature name"))?;+ std::str::from_utf8(&bytes[name..name + end])+ .map_err(|_| invalid("feature name is not UTF-8"))?;+ let geometry_start =+ u32_at(bytes, offset + 28).ok_or(invalid("truncated feature"))? as usize;+ let geometry_end = u32_at(bytes, offset + 32).ok_or(invalid("truncated feature"))? as usize;+ if geometry_start < layout.geometry+ || geometry_end <= geometry_start+ || geometry_end > layout.country_offsets+ {+ return Err(invalid("feature geometry is outside the geometry section"));+ }+ validate_geometry(&bytes[geometry_start..geometry_end])?;+ let country = usize::from(bytes[offset + 40]);+ if country >= layout.country_count {+ return Err(invalid("feature references unknown country"));+ }+ }+ Ok(())+}++fn validate_geometry(bytes: &[u8]) -> crate::Result<()> {+ let mut reader = GeometryReader::new(bytes, SECTION);+ let polygon_count = reader.u16()?;+ if polygon_count == 0 {+ return Err(invalid("feature geometry has no polygons"));+ }+ for _ in 0..polygon_count {+ let ring_count = reader.u16()?;+ if ring_count == 0 {+ return Err(invalid("feature polygon has no rings"));+ }+ for _ in 0..ring_count {+ let point_count = usize::from(reader.u16()?);+ if point_count < 4 {+ return Err(invalid("feature ring is too short"));+ }+ reader.take(+ point_count+ .checked_mul(4)+ .ok_or(invalid("ring size overflow"))?,+ )?;+ }+ }+ if !reader.is_done() {+ return Err(invalid("feature geometry has trailing bytes"));+ }+ Ok(())+}++fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Result<()> {+ let mut expected_dense = 0;+ for block in 0..layout.block_count {+ let offset = layout.blocks + block * BLOCK_LEN;+ let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));+ let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;+ if first != expected_dense {+ return Err(invalid("invalid block dense offset"));+ }+ if block + 1 == layout.block_count {+ let used_bits = cell_count - block * BLOCK_SIZE;+ if used_bits < BLOCK_SIZE && mask >> used_bits != 0 {+ return Err(invalid("last block references nonexistent cells"));+ }+ }+ expected_dense += mask.count_ones() as usize;+ }+ if expected_dense != layout.nonempty_count {+ return Err(invalid("wrong nonempty cell count"));+ }+ let mut previous = 0;+ for index in 0..=layout.nonempty_count {+ let offset = u32_at(bytes, layout.cell_offsets + index * 4)+ .expect("validated cell offset range") as usize;+ if offset < previous || offset > layout.reference_count {+ return Err(invalid("invalid cell offset"));+ }+ previous = offset;+ }+ if previous != layout.reference_count {+ return Err(invalid("last cell offset is not terminal"));+ }+ for index in 0..layout.reference_count {+ let feature = usize::from(+ u16_at(bytes, layout.references + index * 2).expect("validated reference range"),+ );+ if feature >= layout.feature_count {+ return Err(invalid("cell references unknown feature"));+ }+ }+ Ok(())+}++fn quantize(value: f64, minimum: f64, maximum: f64) -> u16 {+ (((value - minimum) * f64::from(u16::MAX) / (maximum - minimum))+ .clamp(0.0, f64::from(u16::MAX))) as u16+}++fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {+ Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)+}++fn invalid(reason: &'static str) -> Error {+ Error::invalid(SECTION, reason)+}diff --git a/rust/crates/photos/src/location.rs b/rust/crates/photos/src/location.rsindex 3994da781d..11444df519 100644--- a/rust/crates/photos/src/location.rs+++ b/rust/crates/photos/src/location.rs@@ -2,10 +2,13 @@ use std::path::Path;use ente_assets::download::CancellationToken;use ente_assets::{Asset, AssetFile, AssetStore};-use ente_location::{CityIndex, CityMatch, Coordinate, CountryIndex, CountryView};+use ente_location::{+ CityIndex, CityMatch, Coordinate, CountryIndex, CountryView, UrbanCenterIndex,+};-const ASSET_URL: &str = "https://assets.ente.com";+const ASSET_URL: &str = "https://assets.ente.com/location/v2";const CITY_FILE: &str = "cities.bin";+const URBAN_CENTER_FILE: &str = "urban-centres.bin";const COUNTRY_FILE: &str = "countries.bin";const DISPUTE_FILE: &str = "disputes.bin";@@ -21,6 +24,7 @@ pub enum Error {pub struct LocationIndex {cities: CityIndex,+ urban_centers: UrbanCenterIndex,countries: CountryIndex,}@@ -34,6 +38,7 @@ impl LocationIndex {.await?;Ok(Self {cities: CityIndex::from_path(directory.join(CITY_FILE))?,+ urban_centers: UrbanCenterIndex::from_path(directory.join(URBAN_CENTER_FILE))?,countries: CountryIndex::from_paths(directory.join(COUNTRY_FILE),directory.join(DISPUTE_FILE),@@ -46,7 +51,8 @@ impl LocationIndex {}pub fn group_cities(&self, coordinates: &[Coordinate], query: &str) -> Vec<CityMatch> {- self.cities.match_coordinates(coordinates, query)+ self.urban_centers+ .match_coordinates_with_cities(&self.cities, coordinates, query)}pub fn group_countries(@@ -64,11 +70,15 @@ impl LocationIndex {fn asset() -> Result<Asset, ente_assets::download::Error> {Asset::files(- vec!["location".into(), "v1".into()],+ vec!["location".into(), "v2".into()],vec![file(CITY_FILE,- "929a488aeac782d1cfc9e90b3eeb9536423977aab9f21cb1f86b9324ac53efbc",+ "898dca892a71fd601ae8e75e5c55fd6d4591e4c98de335d9b33e7f076aa668f5",+ ),+ file(+ URBAN_CENTER_FILE,+ "e981b0383e6502ede56b52d5be96cda66f53d7352492e677d8978467c106dab0",),file(COUNTRY_FILE,