Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

Use urban-centre polygons for city matching

Neeraj Guptaa5cd748594ae
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 7f60798efe..2155ff8f3a 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2662,6 +2662,7 @@ dependencies = [
"ente-location",
"geo",
"reqwest 0.12.28",
+ "rusqlite",
"sha2 0.11.0",
"shapefile",
"thiserror 2.0.18",
diff --git a/rust/crates/location-dataset/Cargo.toml b/rust/crates/location-dataset/Cargo.toml
index 85141b006a..a71cd3deae 100644
--- a/rust/crates/location-dataset/Cargo.toml
+++ b/rust/crates/location-dataset/Cargo.toml
@@ -7,6 +7,7 @@ publish = false
ente-location.workspace = true
geo = { version = "0.33.1", default-features = false }
reqwest = { workspace = true, features = ["blocking", "rustls-tls-webpki-roots"] }
+rusqlite = { version = "0.38", default-features = false, features = ["bundled"] }
sha2.workspace = true
shapefile = { version = "0.9.0", features = ["geo-types"] }
thiserror.workspace = true
diff --git a/rust/crates/location-dataset/README.md b/rust/crates/location-dataset/README.md
index 9c81f92152..ef90680d61 100644
--- a/rust/crates/location-dataset/README.md
+++ b/rust/crates/location-dataset/README.md
@@ -8,14 +8,16 @@ cargo run --release -p ente-location-dataset -- \
```
The command downloads and caches its sources, writes `cities.bin`,
-`countries.bin`, and `disputes.bin`, and prints their sizes and SHA-256 hashes.
+`urban-centres.bin`, `countries.bin`, and `disputes.bin`, and prints their
+sizes and SHA-256 hashes.
-## Source
+## Attribution
-City data is derived from [GeoNames](https://www.geonames.org/) `cities5000`
-and `countryInfo`, licensed under
-[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Ente filters and
-converts the data, which is provided without warranty.
-
-Country boundaries are derived from [Natural Earth](https://www.naturalearthdata.com/),
-which is in the public domain.
+- [GeoNames](https://www.geonames.org/) `cities5000`, `countryInfo`, and
+ `alternateNamesV2`: [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/),
+ adapted by Ente.
+- European Commission Joint Research Centre
+ [GHS-UCDB R2024A](https://doi.org/10.2905/1a338be6-7eaf-480c-9664-3a8ade88cbcd):
+ [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/), adapted by Ente.
+- [Natural Earth](https://www.naturalearthdata.com/) country boundaries and
+ populated places: public domain.
diff --git a/rust/crates/location-dataset/src/city.rs b/rust/crates/location-dataset/src/city.rs
index 26a2c7fc37..1229695cd2 100644
--- a/rust/crates/location-dataset/src/city.rs
+++ b/rust/crates/location-dataset/src/city.rs
@@ -1,23 +1,27 @@
use std::cmp::Ordering;
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
-use ente_location::CountryCode;
+use ente_location::{CountryCode, normalize_search_text};
+use shapefile::dbase::{FieldValue, Reader};
+use zip::ZipArchive;
use crate::{Result, invalid};
-const HEADER_LEN: usize = 72;
+const HEADER_LEN: usize = 76;
const NODE_LEN: usize = 3;
const POINT_LEN: usize = 15;
const NAME_OFFSET_LEN: usize = 3;
const MAX_U24: u32 = 0x00ff_ffff;
+const ALTERNATE_NAMES_FILE: &str = "alternateNamesV2.txt";
struct SourceCity {
source_id: u32,
name: String,
ascii_name: String,
+ aliases: Vec<String>,
country_code: [u8; 2],
latitude: f64,
longitude: f64,
@@ -41,14 +45,42 @@ impl SourceCity {
}
}
-pub(crate) fn build(cities_path: &Path, country_info_path: &Path) -> Result<Vec<u8>> {
+pub(crate) fn build(
+ cities_path: &Path,
+ country_info_path: &Path,
+ alternate_names_path: &Path,
+ populated_places_path: &Path,
+) -> Result<Vec<u8>> {
let countries = read_countries(country_info_path)?;
+ let languages = read_country_languages(country_info_path)?;
let mut cities = read_cities(cities_path, &countries)?;
+ read_alternate_names(alternate_names_path, &mut cities, &languages)?;
+ let override_ids = read_override_ids(populated_places_path)?;
cities.sort_by(compare_priority);
- encode(&cities, &countries)
+ encode(&cities, &countries, &override_ids)
}
-fn read_countries(path: &Path) -> Result<BTreeMap<[u8; 2], String>> {
+fn read_override_ids(path: &Path) -> Result<HashSet<u32>> {
+ let mut reader = Reader::from_path(path).map_err(shapefile::Error::from)?;
+ let mut ids = HashSet::new();
+ for record in reader.iter_records() {
+ let record = record.map_err(shapefile::Error::from)?;
+ if let Some(FieldValue::Numeric(Some(id))) = record.get("GEONAMESID")
+ && *id > 0.0
+ && id.fract() == 0.0
+ {
+ ids.insert(*id as u32);
+ }
+ }
+ if ids.is_empty() {
+ return Err(invalid(
+ "Natural Earth populated places contain no GeoNames IDs",
+ ));
+ }
+ Ok(ids)
+}
+
+pub(crate) fn read_countries(path: &Path) -> Result<BTreeMap<[u8; 2], String>> {
let reader = BufReader::new(File::open(path)?);
let mut countries = BTreeMap::new();
for line in reader.lines() {
@@ -127,6 +159,7 @@ fn read_cities(path: &Path, countries: &BTreeMap<[u8; 2], String>) -> Result<Vec
source_id,
name: name.to_owned(),
ascii_name: ascii_name.to_owned(),
+ aliases: Vec::new(),
country_code,
latitude,
longitude,
@@ -140,6 +173,145 @@ fn read_cities(path: &Path, countries: &BTreeMap<[u8; 2], String>) -> Result<Vec
Ok(cities)
}
+fn read_country_languages(path: &Path) -> Result<BTreeMap<[u8; 2], BTreeSet<String>>> {
+ let reader = BufReader::new(File::open(path)?);
+ let mut countries = BTreeMap::new();
+ for line in reader.lines() {
+ let line = line?;
+ if line.is_empty() || line.starts_with('#') {
+ continue;
+ }
+ let fields: Vec<&str> = line.split('\t').collect();
+ if fields.len() < 16 {
+ return Err(invalid("GeoNames countryInfo row has too few fields"));
+ }
+ let code = code(fields[0])?;
+ let languages = fields[15]
+ .split(',')
+ .filter(|language| !language.is_empty())
+ .map(language_base)
+ .map(str::to_owned)
+ .collect();
+ countries.insert(code, languages);
+ }
+ Ok(countries)
+}
+
+fn read_alternate_names(
+ path: &Path,
+ cities: &mut [SourceCity],
+ country_languages: &BTreeMap<[u8; 2], BTreeSet<String>>,
+) -> Result<()> {
+ let positions: HashMap<u32, usize> = cities
+ .iter()
+ .enumerate()
+ .map(|(index, city)| (city.source_id, index))
+ .collect();
+ let mut archive = ZipArchive::new(File::open(path)?)?;
+ let source = archive.by_name(ALTERNATE_NAMES_FILE)?;
+ apply_alternate_names(
+ BufReader::new(source),
+ cities,
+ &positions,
+ country_languages,
+ )?;
+ finalize_aliases(cities);
+ Ok(())
+}
+
+fn finalize_aliases(cities: &mut [SourceCity]) {
+ for city in cities {
+ city.aliases.sort_unstable();
+ city.aliases.dedup();
+ let name = normalize_search_text(&city.name);
+ let ascii_name = normalize_search_text(&city.ascii_name);
+ city.aliases
+ .retain(|alias| alias != &name && alias != &ascii_name);
+ }
+}
+
+fn apply_alternate_names(
+ reader: impl BufRead,
+ cities: &mut [SourceCity],
+ positions: &HashMap<u32, usize>,
+ country_languages: &BTreeMap<[u8; 2], BTreeSet<String>>,
+) -> Result<()> {
+ for (line_index, line) in reader.lines().enumerate() {
+ let line = line?;
+ let mut fields = line.split('\t');
+ fields.next();
+ let source_id: u32 = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?
+ .parse()
+ .map_err(|_| {
+ invalid(format!(
+ "GeoNames alternate-name row {} has invalid ID",
+ line_index + 1
+ ))
+ })?;
+ let Some(&city_index) = positions.get(&source_id) else {
+ continue;
+ };
+ let language = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;
+ let name = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?
+ .trim();
+ let preferred = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;
+ fields.next();
+ let colloquial = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;
+ let historic = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;
+ fields.next();
+ let to = fields
+ .next()
+ .ok_or_else(|| invalid("GeoNames alternate-name row has too few fields"))?;
+ let city = &mut cities[city_index];
+ if !name.is_empty()
+ && !name.contains('\0')
+ && colloquial != "1"
+ && historic != "1"
+ && to.is_empty()
+ && is_search_language(
+ language,
+ preferred == "1",
+ &country_languages[&city.country_code],
+ )
+ {
+ let alias = normalize_search_text(name);
+ if !alias.is_empty() {
+ city.aliases.push(alias);
+ }
+ }
+ }
+ Ok(())
+}
+
+fn is_search_language(
+ language: &str,
+ preferred: bool,
+ country_languages: &BTreeSet<String>,
+) -> bool {
+ !matches!(
+ language,
+ "abbr" | "faac" | "iata" | "icao" | "link" | "post" | "unlc" | "wkdt"
+ ) && (preferred
+ || language_base(language) == "en"
+ || country_languages.contains(language_base(language)))
+}
+
+fn language_base(language: &str) -> &str {
+ language.split_once('-').map_or(language, |(base, _)| base)
+}
+
fn is_current_place(feature_code: &str) -> bool {
!matches!(feature_code, "PPLH" | "PPLQ" | "PPLW")
}
@@ -154,11 +326,26 @@ fn compare_priority(left: &SourceCity, right: &SourceCity) -> Ordering {
.then_with(|| left.source_id.cmp(&right.source_id))
}
-fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Result<Vec<u8>> {
+fn encode(
+ cities: &[SourceCity],
+ countries: &BTreeMap<[u8; 2], String>,
+ override_ids: &HashSet<u32>,
+) -> Result<Vec<u8>> {
if cities.len() > MAX_U24 as usize {
return Err(invalid("city index exceeds 24-bit point indices"));
}
- let nodes = build_tree(cities);
+ let rank_ends = [4, 3, 2, 1].map(|rank| cities.partition_point(|city| city.rank() >= rank));
+ let nodes = build_tree(cities, (0..cities.len()).collect());
+ let override_nodes = build_tree(
+ cities,
+ cities
+ .iter()
+ .enumerate()
+ .filter_map(|(index, city)| {
+ (city.rank() >= 2 && override_ids.contains(&city.source_id)).then_some(index)
+ })
+ .collect(),
+ );
let mut names = Vec::<String>::new();
let mut name_indices = HashMap::<String, u32>::new();
let mut point_names = Vec::with_capacity(cities.len());
@@ -168,6 +355,10 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resul
key.push('\0');
key.push_str(&city.ascii_name.to_lowercase());
}
+ for alias in &city.aliases {
+ key.push('\0');
+ key.push_str(alias);
+ }
let next = names.len() as u32;
let index = *name_indices.entry(key.clone()).or_insert_with(|| {
names.push(key);
@@ -188,10 +379,9 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resul
.collect();
let name_table = string_table_u24(names.iter().map(String::as_str))?;
let country_table = string_table_u32(countries.values().map(String::as_str))?;
- let rank_ends = [4, 3, 2, 1].map(|rank| cities.partition_point(|city| city.rank() >= rank));
-
let nodes_offset = HEADER_LEN;
- let points_offset = nodes_offset + nodes.len() * NODE_LEN;
+ let override_nodes_offset = nodes_offset + nodes.len() * NODE_LEN;
+ let points_offset = override_nodes_offset + override_nodes.len() * NODE_LEN;
let name_offsets_offset = points_offset + cities.len() * POINT_LEN;
let names_offset = name_offsets_offset + name_table.0.len();
let country_offsets_offset = names_offset + name_table.1.len();
@@ -201,7 +391,7 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resul
let mut output = vec![0; file_length];
output[..4].copy_from_slice(b"CITY");
- put_u16(&mut output, 4, 1);
+ put_u16(&mut output, 4, 2);
put_u16(&mut output, 6, HEADER_LEN as u16);
put_len(&mut output, 8, cities.len())?;
put_len(&mut output, 12, names.len())?;
@@ -228,11 +418,16 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resul
for (field, value) in [56, 60, 64, 68].into_iter().zip(rank_ends) {
put_len(&mut output, field, value)?;
}
+ put_len(&mut output, 72, override_nodes.len())?;
for (index, &point) in nodes.iter().enumerate() {
let offset = nodes_offset + index * NODE_LEN;
put_u24(&mut output, offset, point as u32);
}
+ for (index, &point) in override_nodes.iter().enumerate() {
+ let offset = override_nodes_offset + index * NODE_LEN;
+ put_u24(&mut output, offset, point as u32);
+ }
for (index, city) in cities.iter().enumerate() {
let offset = points_offset + index * POINT_LEN;
output[offset..offset + 4].copy_from_slice(&(city.latitude as f32).to_le_bytes());
@@ -251,7 +446,7 @@ fn encode(cities: &[SourceCity], countries: &BTreeMap<[u8; 2], String>) -> Resul
Ok(output)
}
-fn build_tree(cities: &[SourceCity]) -> Vec<usize> {
+fn build_tree(cities: &[SourceCity], mut points: Vec<usize>) -> Vec<usize> {
fn build(cities: &[SourceCity], nodes: &mut Vec<usize>, points: &mut [usize], depth: usize) {
if points.is_empty() {
return;
@@ -276,8 +471,7 @@ fn build_tree(cities: &[SourceCity]) -> Vec<usize> {
build(cities, nodes, right, depth + 1);
}
- let mut points: Vec<usize> = (0..cities.len()).collect();
- let mut nodes = Vec::with_capacity(cities.len());
+ let mut nodes = Vec::with_capacity(points.len());
build(cities, &mut nodes, &mut points, 0);
nodes
}
@@ -295,7 +489,7 @@ fn string_table_u24<'a>(
Ok((offsets, blob))
}
-fn string_table_u32<'a>(
+pub(crate) fn string_table_u32<'a>(
values: impl ExactSizeIterator<Item = &'a str>,
) -> Result<(Vec<u8>, Vec<u8>)> {
let mut offsets = Vec::with_capacity((values.len() + 1) * 4);
@@ -369,35 +563,83 @@ fn copy(output: &mut [u8], offset: usize, bytes: &[u8]) {
#[cfg(test)]
mod tests {
- use std::collections::BTreeMap;
+ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+ use std::io::Cursor;
use ente_location::{CityIndex, Coordinate, CountryCode};
- use super::{SourceCity, encode};
+ use super::{SourceCity, apply_alternate_names, encode, finalize_aliases};
#[test]
fn round_trips_city_search_and_lookup() {
+ let mut munich = city(3, "Munich", "Munich", *b"DE", 48.1374, 11.5755, 3);
+ munich.aliases = vec!["garching bei munchen".to_owned(), "munchen".to_owned()];
let cities = vec![
city(1, "Delhi", "Delhi", *b"IN", 28.6139, 77.2090, 4),
city(2, "Zürich", "Zuerich", *b"CH", 47.3667, 8.5500, 3),
+ munich,
+ city(4, "Small Town", "Small Town", *b"IN", 0.0, 0.0, 0),
];
let countries = BTreeMap::from([
(*b"CH", "Switzerland".to_owned()),
+ (*b"DE", "Germany".to_owned()),
(*b"IN", "India".to_owned()),
]);
- let bytes = encode(&cities, &countries).unwrap();
+ let bytes = encode(&cities, &countries, &HashSet::from([1, 2, 3])).unwrap();
let index = CityIndex::from_bytes(bytes).unwrap();
assert_eq!(index.search("zurich", 5)[0].name, "Zürich");
assert_eq!(index.search("ZUERICH", 5)[0].name, "Zürich");
+ assert_eq!(index.search("München", 5)[0].name, "Munich");
+ assert_eq!(index.search("Garching bei München", 5)[0].name, "Munich");
+ assert_eq!(
+ index.match_coordinates(&[Coordinate::new(0.0, 0.0)], "")[0]
+ .city
+ .name,
+ "Small Town"
+ );
+ assert!(
+ index
+ .match_override_coordinates(&[Coordinate::new(0.0, 0.0)], "")
+ .is_empty()
+ );
+ assert!(
+ index
+ .match_override_coordinates(&[Coordinate::new(28.65, 77.209)], "")
+ .is_empty()
+ );
+ assert_eq!(
+ index.match_coordinates(&[Coordinate::new(28.71, 77.209)], "")[0]
+ .city
+ .name,
+ "Delhi"
+ );
assert_eq!(
- index.match_coordinates(&[Coordinate::new(28.61, 77.21)], "")[0]
+ index.match_override_coordinates(&[Coordinate::new(28.61, 77.21)], "")[0]
.city
.country_code,
CountryCode::from_bytes(*b"IN").unwrap()
);
}
+ #[test]
+ fn keeps_current_preferred_and_local_aliases() {
+ let mut cities = vec![city(1, "Munich", "Munich", *b"DE", 48.1, 11.5, 3)];
+ let positions = HashMap::from([(1, 0)]);
+ let languages = BTreeMap::from([(*b"DE", BTreeSet::from(["de".to_owned()]))]);
+ let source = concat!(
+ "1\t1\tde\tMünchen\t1\t\t\t\t\t\n",
+ "2\t1\tde\tMuenchen\t\t\t\t\t\t\n",
+ "3\t1\ten\tOld Munich\t\t\t\t1\t\t\n",
+ "4\t1\tlink\thttps://example.com\t1\t\t\t\t\t\n"
+ );
+
+ apply_alternate_names(Cursor::new(source), &mut cities, &positions, &languages).unwrap();
+ finalize_aliases(&mut cities);
+
+ assert_eq!(cities[0].aliases, ["muenchen", "munchen"]);
+ }
+
fn city(
source_id: u32,
name: &str,
@@ -411,6 +653,7 @@ mod tests {
source_id,
name: name.to_owned(),
ascii_name: ascii_name.to_owned(),
+ aliases: Vec::new(),
country_code,
latitude,
longitude,
diff --git a/rust/crates/location-dataset/src/lib.rs b/rust/crates/location-dataset/src/lib.rs
index 417e669103..c3b6f3cf97 100644
--- a/rust/crates/location-dataset/src/lib.rs
+++ b/rust/crates/location-dataset/src/lib.rs
@@ -1,7 +1,7 @@
use std::fmt::Write as _;
use std::path::PathBuf;
-use ente_location::{CityIndex, CountryIndex};
+use ente_location::{CityIndex, CountryIndex, UrbanCenterIndex};
use sha2::{Digest, Sha256};
use thiserror::Error;
@@ -11,10 +11,12 @@ mod country;
mod dispute;
mod download;
mod sources;
+mod urban;
pub use sources::{RemoteSources, SourcePaths};
pub const CITY_FILE: &str = "cities.bin";
+pub const URBAN_CENTER_FILE: &str = "urban-centres.bin";
pub const COUNTRY_FILE: &str = "countries.bin";
pub const DISPUTE_FILE: &str = "disputes.bin";
@@ -30,6 +32,8 @@ pub enum Error {
Zip(#[from] zip::result::ZipError),
#[error("shapefile is invalid: {0}")]
Shapefile(#[from] shapefile::Error),
+ #[error("GeoPackage is invalid: {0}")]
+ Sqlite(#[from] rusqlite::Error),
#[error("invalid location data: {0}")]
InvalidData(String),
#[error(transparent)]
@@ -43,9 +47,10 @@ pub struct BuildOptions {
pub struct BuildOutput {
pub city_count: usize,
+ pub urban_center_count: usize,
pub territory_count: usize,
pub byte_length: usize,
- pub files: [BuildFile; 3],
+ pub files: [BuildFile; 4],
}
pub struct BuildFile {
@@ -55,22 +60,36 @@ pub struct BuildFile {
}
pub fn build(options: &BuildOptions) -> Result<BuildOutput> {
- let cities = city::build(&options.sources.cities, &options.sources.country_info)?;
+ let cities = city::build(
+ &options.sources.cities,
+ &options.sources.country_info,
+ &options.sources.alternate_names,
+ &options.sources.populated_places,
+ )?;
+ let city_index = CityIndex::from_bytes(cities.as_slice())?;
+ let urban_centers = urban::build(
+ &options.sources.urban_centres,
+ &options.sources.country_info,
+ &city_index,
+ )?;
let countries = country::build_countries(&options.sources.countries)?;
let disputes = dispute::build(
&options.sources.countries,
&options.sources.disputes,
&options.sources.admin1,
)?;
- let city_count = CityIndex::from_bytes(cities.as_slice())?.len();
+ let city_count = city_index.len();
+ let urban_center_count = UrbanCenterIndex::from_bytes(urban_centers.as_slice())?.len();
CountryIndex::from_bytes(countries.as_slice(), disputes.as_slice())?;
let files = [
build_file(CITY_FILE, &cities),
+ build_file(URBAN_CENTER_FILE, &urban_centers),
build_file(COUNTRY_FILE, &countries),
build_file(DISPUTE_FILE, &disputes),
];
let output = BuildOutput {
city_count,
+ urban_center_count,
territory_count: catalog::DISPUTED_AREAS.len() + catalog::UKRAINIAN_REGIONS.len(),
byte_length: files.iter().map(|file| file.byte_length).sum(),
files,
@@ -78,6 +97,7 @@ pub fn build(options: &BuildOptions) -> Result<BuildOutput> {
std::fs::create_dir_all(&options.output)?;
for (name, bytes) in [
(CITY_FILE, cities.as_slice()),
+ (URBAN_CENTER_FILE, urban_centers.as_slice()),
(COUNTRY_FILE, countries.as_slice()),
(DISPUTE_FILE, disputes.as_slice()),
] {
diff --git a/rust/crates/location-dataset/src/main.rs b/rust/crates/location-dataset/src/main.rs
index 1981fbe424..6e0200b232 100644
--- a/rust/crates/location-dataset/src/main.rs
+++ b/rust/crates/location-dataset/src/main.rs
@@ -6,8 +6,9 @@ use ente_location_dataset::{BuildOptions, RemoteSources, SourcePaths, build};
const USAGE: &str = concat!(
"usage: ente-location-dataset build --output <directory> [--cache <directory>] ",
- "[--cities <file> --country-info <file> ",
- "--countries <shp> --disputes <shp> --admin1 <shp>]"
+ "[--cities <file> --country-info <file> --alternate-names <zip> ",
+ "--urban-centres <gpkg> --populated-places <dbf> --countries <shp> ",
+ "--disputes <shp> --admin1 <shp>]"
);
fn main() {
@@ -23,25 +24,48 @@ fn run() -> Result<(), Box<dyn Error>> {
cache,
cities,
country_info,
+ alternate_names,
+ urban_centres,
+ populated_places,
countries,
disputes,
admin1,
} = Arguments::parse()?;
- let sources = match (cities, country_info, countries, disputes, admin1) {
- (None, None, None, None, None) => RemoteSources::new(cache).fetch()?,
- (Some(cities), Some(country_info), Some(countries), Some(disputes), Some(admin1)) => {
- SourcePaths {
- cities,
- country_info,
- countries,
- disputes,
- admin1,
- }
- }
- _ => return Err("local generation requires all five source paths".into()),
+ let sources = match (
+ cities,
+ country_info,
+ alternate_names,
+ urban_centres,
+ populated_places,
+ countries,
+ disputes,
+ admin1,
+ ) {
+ (None, None, None, None, None, None, None, None) => RemoteSources::new(cache).fetch()?,
+ (
+ Some(cities),
+ Some(country_info),
+ Some(alternate_names),
+ Some(urban_centres),
+ Some(populated_places),
+ Some(countries),
+ Some(disputes),
+ Some(admin1),
+ ) => SourcePaths {
+ cities,
+ country_info,
+ alternate_names,
+ urban_centres,
+ populated_places,
+ countries,
+ disputes,
+ admin1,
+ },
+ _ => return Err("local generation requires all eight source paths".into()),
};
let result = build(&BuildOptions { sources, output })?;
println!("cities: {}", result.city_count);
+ println!("urban centers: {}", result.urban_center_count);
println!("Priority-1 territories: {}", result.territory_count);
println!("bytes: {}", result.byte_length);
for file in result.files {
@@ -58,6 +82,9 @@ struct Arguments {
cache: PathBuf,
cities: Option<PathBuf>,
country_info: Option<PathBuf>,
+ alternate_names: Option<PathBuf>,
+ urban_centres: Option<PathBuf>,
+ populated_places: Option<PathBuf>,
countries: Option<PathBuf>,
disputes: Option<PathBuf>,
admin1: Option<PathBuf>,
@@ -73,6 +100,9 @@ impl Arguments {
let mut cache = None;
let mut cities = None;
let mut country_info = None;
+ let mut alternate_names = None;
+ let mut urban_centres = None;
+ let mut populated_places = None;
let mut countries = None;
let mut disputes = None;
let mut admin1 = None;
@@ -83,6 +113,9 @@ impl Arguments {
"--cache" => cache = Some(value.into()),
"--cities" => cities = Some(value.into()),
"--country-info" => country_info = Some(value.into()),
+ "--alternate-names" => alternate_names = Some(value.into()),
+ "--urban-centres" => urban_centres = Some(value.into()),
+ "--populated-places" => populated_places = Some(value.into()),
"--countries" => countries = Some(value.into()),
"--disputes" => disputes = Some(value.into()),
"--admin1" => admin1 = Some(value.into()),
@@ -94,6 +127,9 @@ impl Arguments {
cache: cache.unwrap_or_else(default_cache),
cities,
country_info,
+ alternate_names,
+ urban_centres,
+ populated_places,
countries,
disputes,
admin1,
diff --git a/rust/crates/location-dataset/src/sources.rs b/rust/crates/location-dataset/src/sources.rs
index 828f1ace49..6ca280850d 100644
--- a/rust/crates/location-dataset/src/sources.rs
+++ b/rust/crates/location-dataset/src/sources.rs
@@ -7,6 +7,17 @@ use crate::download::{download, extract};
const CITIES_URL: &str = "https://download.geonames.org/export/dump/cities5000.zip";
const COUNTRY_INFO_URL: &str = "https://download.geonames.org/export/dump/countryInfo.txt";
+const ALTERNATE_NAMES_URL: &str = "https://download.geonames.org/export/dump/alternateNamesV2.zip";
+const URBAN_CENTRES_FILE: &str = "GHS_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A.gpkg";
+const URBAN_CENTRES_URL: &str = concat!(
+ "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/",
+ "GHS_UCDB_GLOBE_R2024A/GHS_UCDB_THEME_GLOBE_R2024A/",
+ "GHS_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A/V1-2/",
+ "GHS_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A_V1_2.zip"
+);
+const POPULATED_PLACES: &str = "ne_10m_populated_places";
+const POPULATED_PLACES_URL: &str =
+ "https://naciscdn.org/naturalearth/10m/cultural/ne_10m_populated_places.zip";
const COUNTRIES: &str = "ne_10m_admin_0_countries";
const COUNTRIES_URL: &str =
"https://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_countries.zip";
@@ -21,6 +32,9 @@ const ADMIN1_URL: &str =
pub struct SourcePaths {
pub cities: PathBuf,
pub country_info: PathBuf,
+ pub alternate_names: PathBuf,
+ pub urban_centres: PathBuf,
+ pub populated_places: PathBuf,
pub countries: PathBuf,
pub disputes: PathBuf,
pub admin1: PathBuf,
@@ -46,21 +60,40 @@ impl RemoteSources {
let cities_archive = self.cache.join("cities5000.zip");
let cities = self.cache.join("cities5000.txt");
let country_info = self.cache.join("countryInfo.txt");
+ let alternate_names = self.cache.join("alternateNamesV2.zip");
download(&client, CITIES_URL, &cities_archive)?;
download(&client, COUNTRY_INFO_URL, &country_info)?;
+ download(&client, ALTERNATE_NAMES_URL, &alternate_names)?;
extract(&cities_archive, &cities, "cities5000.txt")?;
+ let urban_centres_archive = self.cache.join("ghsl-urban-centres-v1.2.zip");
+ let urban_centres = self.cache.join(URBAN_CENTRES_FILE);
+ download(&client, URBAN_CENTRES_URL, &urban_centres_archive)?;
+ extract(&urban_centres_archive, &urban_centres, URBAN_CENTRES_FILE)?;
+ let populated_places = self.fetch_dbf(&client, POPULATED_PLACES, POPULATED_PLACES_URL)?;
let countries = self.fetch_shapefile(&client, COUNTRIES, COUNTRIES_URL)?;
let disputes = self.fetch_shapefile(&client, DISPUTES, DISPUTES_URL)?;
let admin1 = self.fetch_shapefile(&client, ADMIN1, ADMIN1_URL)?;
Ok(SourcePaths {
cities,
country_info,
+ alternate_names,
+ urban_centres,
+ populated_places,
countries,
disputes,
admin1,
})
}
+ fn fetch_dbf(&self, client: &Client, name: &str, url: &str) -> Result<PathBuf> {
+ let archive = self.cache.join(format!("{name}.zip"));
+ download(client, url, &archive)?;
+ let file = format!("{name}.dbf");
+ let path = self.cache.join(&file);
+ extract(&archive, &path, &file)?;
+ Ok(path)
+ }
+
fn fetch_shapefile(&self, client: &Client, name: &str, url: &str) -> Result<PathBuf> {
let archive = self.cache.join(format!("{name}.zip"));
download(client, url, &archive)?;
diff --git a/rust/crates/location-dataset/src/urban.rs b/rust/crates/location-dataset/src/urban.rs
new file mode 100644
index 0000000000..1d0922bd43
--- /dev/null
+++ b/rust/crates/location-dataset/src/urban.rs
@@ -0,0 +1,694 @@
+use std::collections::BTreeMap;
+use std::f64::consts::{PI, SQRT_2};
+use std::path::Path;
+
+use ente_location::{City, CityIndex, Coordinate, UrbanCenterIndex, normalize_search_text};
+use geo::{BoundingRect, Coord, Distance, Haversine, LineString, MultiPolygon, Point, Polygon};
+use rusqlite::{Connection, OpenFlags};
+
+use crate::city::{read_countries, string_table_u32};
+use crate::{Result, invalid};
+
+const QUERY: &str = concat!(
+ "SELECT u.ID_UC_G0, u.GC_UCN_MAI_2025, u.GC_CNT_UNN_2025, ",
+ "c.GC_UCC_LON_2025, c.GC_UCC_LAT_2025, u.geom ",
+ "FROM GHSL_UCDB_THEME_GENERAL_CHARACTERISTICS_GLOBE_R2024A u ",
+ "JOIN UC_centroids c USING (ID_UC_G0) ",
+ "WHERE u.GC_PLS_SCR_2025 IN ('High', 'Medium') ",
+ "AND trim(u.GC_UCN_MAI_2025) != '' ORDER BY u.ID_UC_G0"
+);
+const COLUMNS: u16 = 360;
+const ROWS: u16 = 180;
+const HEADER_LEN: usize = 72;
+const RECORD_LEN: usize = 41;
+const BLOCK_SIZE: usize = 64;
+const BLOCK_LEN: usize = 12;
+const BOUNDS_PADDING: f64 = 0.0001;
+const MOLLWEIDE_RADIUS: f64 = 6_378_137.0;
+const GHSL_SOURCE_BIT: u32 = 1 << 31;
+const MAX_NAME_MATCH_DISTANCE_METERS: f64 = 30_000.0;
+
+struct SourceCenter {
+ source_id: u32,
+ name: String,
+ country_code: [u8; 2],
+ centroid: Coordinate,
+ geometry: MultiPolygon<f64>,
+}
+
+#[derive(Clone, Copy)]
+struct Bounds {
+ minimum_longitude: f64,
+ minimum_latitude: f64,
+ maximum_longitude: f64,
+ maximum_latitude: f64,
+}
+
+pub(crate) fn build(
+ path: &Path,
+ country_info_path: &Path,
+ city_index: &CityIndex,
+) -> Result<Vec<u8>> {
+ let countries = read_countries(country_info_path)?;
+ let mut cities_by_name = BTreeMap::<([u8; 2], String), Vec<City>>::new();
+ for city in city_index.search("", usize::MAX) {
+ cities_by_name
+ .entry((
+ city.country_code.as_bytes(),
+ normalize_search_text(&city.name),
+ ))
+ .or_default()
+ .push(city);
+ }
+ let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
+ let mut statement = connection.prepare(QUERY)?;
+ let rows = statement.query_map([], |row| {
+ Ok((
+ row.get::<_, u32>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, String>(2)?,
+ row.get::<_, f64>(3)?,
+ row.get::<_, f64>(4)?,
+ row.get::<_, Vec<u8>>(5)?,
+ ))
+ })?;
+ let mut centers = Vec::new();
+ for row in rows {
+ let (source_id, name, country, centroid_x, centroid_y, geometry) = row?;
+ if source_id & GHSL_SOURCE_BIT != 0 {
+ return Err(invalid("GHSL urban center ID exceeds 31 bits"));
+ }
+ let country_code = resolve_country(&country, &countries)
+ .ok_or_else(|| invalid(format!("unknown GHSL country {country}")))?;
+ let centroid = mollweide_to_wgs84(centroid_x, centroid_y)?;
+ let geometry = decode_geopackage_geometry(&geometry)?;
+ let key = (country_code, normalize_search_text(&name));
+ let named = canonical_name_city(
+ cities_by_name
+ .get(&key)
+ .map(Vec::as_slice)
+ .unwrap_or_default(),
+ centroid,
+ );
+ centers.push(match named {
+ Some(city) => SourceCenter {
+ source_id: city.source_id,
+ name: city.name.clone(),
+ country_code,
+ centroid: Coordinate::new(city.latitude, city.longitude),
+ geometry,
+ },
+ None => SourceCenter {
+ source_id: source_id | GHSL_SOURCE_BIT,
+ name,
+ country_code,
+ centroid,
+ geometry,
+ },
+ });
+ }
+ encode(&centers, &countries)
+}
+
+fn canonical_name_city(cities: &[City], centroid: Coordinate) -> Option<&City> {
+ let centroid = Point::new(centroid.longitude, centroid.latitude);
+ cities
+ .iter()
+ .map(|city| {
+ let distance = Haversine.distance(centroid, Point::new(city.longitude, city.latitude));
+ (city, distance)
+ })
+ .filter(|(_, distance)| *distance <= MAX_NAME_MATCH_DISTANCE_METERS)
+ .min_by(|left, right| left.1.total_cmp(&right.1))
+ .map(|(city, _)| city)
+}
+
+fn encode(
+ centers: &[SourceCenter],
+ source_countries: &BTreeMap<[u8; 2], String>,
+) -> Result<Vec<u8>> {
+ if centers.is_empty() || centers.len() > usize::from(u16::MAX) {
+ return Err(invalid("urban center count does not fit the index"));
+ }
+
+ let countries: BTreeMap<[u8; 2], &str> = centers
+ .iter()
+ .map(|center| {
+ (
+ center.country_code,
+ source_countries[&center.country_code].as_str(),
+ )
+ })
+ .collect();
+ if countries.len() > 256 {
+ return Err(invalid("urban center country count exceeds 256"));
+ }
+ let country_indices: BTreeMap<[u8; 2], u8> = countries
+ .keys()
+ .copied()
+ .enumerate()
+ .map(|(index, code)| (code, index as u8))
+ .collect();
+ let bounds: Vec<Bounds> = centers
+ .iter()
+ .map(|center| geometry_bounds(&center.geometry))
+ .collect::<Result<_>>()?;
+
+ let mut names = Vec::new();
+ let mut name_offsets = Vec::with_capacity(centers.len());
+ for center in centers {
+ if center.name.contains('\0') {
+ return Err(invalid("urban center name contains NUL"));
+ }
+ name_offsets.push(names.len());
+ names.extend_from_slice(center.name.as_bytes());
+ names.push(0);
+ }
+
+ let mut geometry = Vec::new();
+ let mut geometry_offsets = Vec::with_capacity(centers.len() + 1);
+ geometry_offsets.push(0);
+ for (center, &bounds) in centers.iter().zip(&bounds) {
+ encode_geometry(&mut geometry, &center.geometry, bounds)?;
+ geometry_offsets.push(geometry.len());
+ }
+
+ let (country_offsets, country_names) = string_table_u32(countries.values().copied())?;
+ let country_codes: Vec<u8> = countries.keys().flatten().copied().collect();
+ let cells = grid(&bounds)?;
+ let block_count = cells.len().div_ceil(BLOCK_SIZE);
+ let nonempty_count = cells.iter().filter(|cell| !cell.is_empty()).count();
+ let reference_count: usize = cells.iter().map(Vec::len).sum();
+
+ let records_offset = HEADER_LEN;
+ let names_offset = records_offset + centers.len() * RECORD_LEN;
+ let geometry_offset = names_offset + names.len();
+ let country_offsets_offset = geometry_offset + geometry.len();
+ let countries_offset = country_offsets_offset + country_offsets.len();
+ let country_codes_offset = countries_offset + country_names.len();
+ let blocks_offset = country_codes_offset + country_codes.len();
+ let cell_offsets_offset = blocks_offset + block_count * BLOCK_LEN;
+ let references_offset = cell_offsets_offset + (nonempty_count + 1) * 4;
+ let file_length = references_offset + reference_count * 2;
+
+ let mut output = Vec::with_capacity(file_length);
+ output.extend_from_slice(b"URBN");
+ push_u16(&mut output, 1);
+ push_u16(&mut output, HEADER_LEN as u16);
+ push_len(&mut output, centers.len())?;
+ push_u16(&mut output, countries.len() as u16);
+ push_u16(&mut output, COLUMNS);
+ push_u16(&mut output, ROWS);
+ push_u16(&mut output, BLOCK_SIZE as u16);
+ for value in [
+ block_count,
+ nonempty_count,
+ records_offset,
+ names_offset,
+ geometry_offset,
+ country_offsets_offset,
+ countries_offset,
+ country_codes_offset,
+ blocks_offset,
+ cell_offsets_offset,
+ references_offset,
+ file_length,
+ reference_count,
+ ] {
+ push_len(&mut output, value)?;
+ }
+ debug_assert_eq!(output.len(), HEADER_LEN);
+
+ for (index, center) in centers.iter().enumerate() {
+ let bounds = bounds[index];
+ for value in [
+ bounds.minimum_longitude,
+ bounds.minimum_latitude,
+ bounds.maximum_longitude,
+ bounds.maximum_latitude,
+ center.centroid.latitude,
+ center.centroid.longitude,
+ ] {
+ output.extend_from_slice(&(value as f32).to_le_bytes());
+ }
+ push_len(&mut output, names_offset + name_offsets[index])?;
+ push_len(&mut output, geometry_offset + geometry_offsets[index])?;
+ push_len(&mut output, geometry_offset + geometry_offsets[index + 1])?;
+ output.extend_from_slice(&center.source_id.to_le_bytes());
+ output.push(country_indices[&center.country_code]);
+ }
+ output.extend_from_slice(&names);
+ output.extend_from_slice(&geometry);
+ output.extend_from_slice(&country_offsets);
+ output.extend_from_slice(&country_names);
+ output.extend_from_slice(&country_codes);
+
+ let mut dense_start = 0;
+ for block in cells.chunks(BLOCK_SIZE) {
+ let mask = block.iter().enumerate().fold(0_u64, |mask, (bit, cell)| {
+ mask | ((!cell.is_empty()) as u64) << bit
+ });
+ output.extend_from_slice(&mask.to_le_bytes());
+ push_len(&mut output, dense_start)?;
+ dense_start += mask.count_ones() as usize;
+ }
+ let mut reference_offset = 0;
+ push_len(&mut output, reference_offset)?;
+ for cell in cells.iter().filter(|cell| !cell.is_empty()) {
+ reference_offset += cell.len();
+ push_len(&mut output, reference_offset)?;
+ }
+ for cell in cells.iter().filter(|cell| !cell.is_empty()) {
+ for &reference in cell {
+ push_u16(&mut output, reference);
+ }
+ }
+ debug_assert_eq!(output.len(), file_length);
+ UrbanCenterIndex::from_bytes(output.as_slice())?;
+ Ok(output)
+}
+
+fn geometry_bounds(geometry: &MultiPolygon<f64>) -> Result<Bounds> {
+ let bounds = geometry
+ .bounding_rect()
+ .ok_or_else(|| invalid("urban center has empty geometry"))?;
+ Ok(Bounds {
+ minimum_longitude: (bounds.min().x - BOUNDS_PADDING).max(-180.0),
+ minimum_latitude: (bounds.min().y - BOUNDS_PADDING).max(-90.0),
+ maximum_longitude: (bounds.max().x + BOUNDS_PADDING).min(180.0),
+ maximum_latitude: (bounds.max().y + BOUNDS_PADDING).min(90.0),
+ })
+}
+
+fn encode_geometry(
+ output: &mut Vec<u8>,
+ geometry: &MultiPolygon<f64>,
+ bounds: Bounds,
+) -> Result<()> {
+ push_count(output, geometry.0.len(), "too many urban center polygons")?;
+ for polygon in geometry {
+ push_count(
+ output,
+ polygon.interiors().len() + 1,
+ "too many urban center rings",
+ )?;
+ for ring in std::iter::once(polygon.exterior()).chain(polygon.interiors()) {
+ push_count(output, ring.0.len(), "urban center ring is too long")?;
+ for coordinate in &ring.0 {
+ push_u16(
+ output,
+ quantize(
+ coordinate.x,
+ bounds.minimum_longitude,
+ bounds.maximum_longitude,
+ ),
+ );
+ push_u16(
+ output,
+ quantize(
+ coordinate.y,
+ bounds.minimum_latitude,
+ bounds.maximum_latitude,
+ ),
+ );
+ }
+ }
+ }
+ Ok(())
+}
+
+fn grid(bounds: &[Bounds]) -> Result<Vec<Vec<u16>>> {
+ let mut cells = vec![Vec::new(); usize::from(COLUMNS) * usize::from(ROWS)];
+ for (index, bounds) in bounds.iter().enumerate() {
+ let index = u16::try_from(index).map_err(|_| invalid("too many urban centers"))?;
+ for row in latitude_row(bounds.maximum_latitude)..=latitude_row(bounds.minimum_latitude) {
+ for column in longitude_column(bounds.minimum_longitude)
+ ..=longitude_column(bounds.maximum_longitude)
+ {
+ cells[row * usize::from(COLUMNS) + column].push(index);
+ }
+ }
+ }
+ Ok(cells)
+}
+
+fn longitude_column(longitude: f64) -> usize {
+ (((longitude + 180.0) / 360.0) * f64::from(COLUMNS))
+ .floor()
+ .clamp(0.0, f64::from(COLUMNS - 1)) as usize
+}
+
+fn latitude_row(latitude: f64) -> usize {
+ (((90.0 - latitude) / 180.0) * f64::from(ROWS))
+ .floor()
+ .clamp(0.0, f64::from(ROWS - 1)) as usize
+}
+
+fn quantize(value: f64, minimum: f64, maximum: f64) -> u16 {
+ (((value - minimum) * f64::from(u16::MAX) / (maximum - minimum))
+ .clamp(0.0, f64::from(u16::MAX))) as u16
+}
+
+fn resolve_country(name: &str, countries: &BTreeMap<[u8; 2], String>) -> Option<[u8; 2]> {
+ countries
+ .iter()
+ .find_map(|(&code, country)| (country == name).then_some(code))
+ .or_else(|| country_alias(name).filter(|code| countries.contains_key(code)))
+}
+
+fn country_alias(name: &str) -> Option<[u8; 2]> {
+ Some(*match name {
+ "Bolivia (Plurinational State of)" => b"BO",
+ "Brunei Darussalam" => b"BN",
+ "China, Taiwan Province of China" => b"TW",
+ "Congo" => b"CG",
+ "Curaçao" => b"CW",
+ "Côte d'Ivoire" => b"CI",
+ "Dem. People's Republic of Korea" => b"KP",
+ "Iran (Islamic Republic of)" => b"IR",
+ "Kosovo (under UNSC res. 1244)" => b"XK",
+ "Lao People's Democratic Republic" => b"LA",
+ "Netherlands" => b"NL",
+ "Republic of Korea" => b"KR",
+ "Republic of Moldova" => b"MD",
+ "Russian Federation" => b"RU",
+ "Réunion" => b"RE",
+ "State of Palestine" => b"PS",
+ "Syrian Arab Republic" => b"SY",
+ "Timor-Leste" => b"TL",
+ "United Republic of Tanzania" => b"TZ",
+ "United States of America" => b"US",
+ "Venezuela (Bolivarian Republic of)" => b"VE",
+ "Viet Nam" => b"VN",
+ _ => return None,
+ })
+}
+
+fn decode_geopackage_geometry(bytes: &[u8]) -> Result<MultiPolygon<f64>> {
+ if bytes.len() < 8 || &bytes[..2] != b"GP" || bytes[2] != 0 {
+ return Err(invalid("invalid GHSL GeoPackage geometry header"));
+ }
+ let flags = bytes[3];
+ if flags & 1 == 0 || flags & 0x10 != 0 {
+ return Err(invalid("unsupported GHSL GeoPackage geometry flags"));
+ }
+ let srs = i32::from_le_bytes(bytes[4..8].try_into().expect("eight-byte header"));
+ if srs != 54_009 {
+ return Err(invalid(format!("unexpected GHSL spatial reference {srs}")));
+ }
+ let envelope_length = match (flags >> 1) & 7 {
+ 0 => 0,
+ 1 => 32,
+ 2 | 3 => 48,
+ 4 => 64,
+ _ => return Err(invalid("invalid GHSL GeoPackage envelope")),
+ };
+ let mut reader = WkbReader::new(
+ bytes
+ .get(8 + envelope_length..)
+ .ok_or_else(|| invalid("truncated GHSL GeoPackage geometry"))?,
+ );
+ let geometry = reader.multi_polygon()?;
+ if reader.position != reader.bytes.len() {
+ return Err(invalid("GHSL geometry has trailing bytes"));
+ }
+ Ok(geometry)
+}
+
+struct WkbReader<'a> {
+ bytes: &'a [u8],
+ position: usize,
+}
+
+impl<'a> WkbReader<'a> {
+ const fn new(bytes: &'a [u8]) -> Self {
+ Self { bytes, position: 0 }
+ }
+
+ fn multi_polygon(&mut self) -> Result<MultiPolygon<f64>> {
+ let endian = self.header(6)?;
+ let count = self.u32(endian)? as usize;
+ let polygons = (0..count)
+ .map(|_| self.polygon())
+ .collect::<Result<Vec<_>>>()?;
+ Ok(MultiPolygon::new(polygons))
+ }
+
+ fn polygon(&mut self) -> Result<Polygon<f64>> {
+ let endian = self.header(3)?;
+ let count = self.u32(endian)? as usize;
+ if count == 0 {
+ return Err(invalid("GHSL polygon has no rings"));
+ }
+ let mut rings = (0..count)
+ .map(|_| self.ring(endian))
+ .collect::<Result<Vec<_>>>()?;
+ Ok(Polygon::new(rings.remove(0), rings))
+ }
+
+ fn ring(&mut self, endian: Endian) -> Result<LineString<f64>> {
+ let count = self.u32(endian)? as usize;
+ if count < 4 {
+ return Err(invalid("GHSL polygon ring is too short"));
+ }
+ let coordinates = (0..count)
+ .map(|_| {
+ let x = self.f64(endian)?;
+ let y = self.f64(endian)?;
+ let coordinate = mollweide_to_wgs84(x, y)?;
+ Ok(Coord {
+ x: coordinate.longitude,
+ y: coordinate.latitude,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ Ok(LineString::new(coordinates))
+ }
+
+ fn header(&mut self, expected_type: u32) -> Result<Endian> {
+ let endian = match self.byte()? {
+ 0 => Endian::Big,
+ 1 => Endian::Little,
+ _ => return Err(invalid("invalid GHSL WKB byte order")),
+ };
+ let geometry_type = self.u32(endian)?;
+ if geometry_type != expected_type {
+ return Err(invalid(format!(
+ "unexpected GHSL WKB geometry type {geometry_type}"
+ )));
+ }
+ Ok(endian)
+ }
+
+ fn byte(&mut self) -> Result<u8> {
+ let byte = *self
+ .bytes
+ .get(self.position)
+ .ok_or_else(|| invalid("truncated GHSL WKB geometry"))?;
+ self.position += 1;
+ Ok(byte)
+ }
+
+ fn u32(&mut self, endian: Endian) -> Result<u32> {
+ let bytes = self.take::<4>()?;
+ Ok(match endian {
+ Endian::Little => u32::from_le_bytes(bytes),
+ Endian::Big => u32::from_be_bytes(bytes),
+ })
+ }
+
+ fn f64(&mut self, endian: Endian) -> Result<f64> {
+ let bytes = self.take::<8>()?;
+ Ok(match endian {
+ Endian::Little => f64::from_le_bytes(bytes),
+ Endian::Big => f64::from_be_bytes(bytes),
+ })
+ }
+
+ fn take<const N: usize>(&mut self) -> Result<[u8; N]> {
+ let end = self
+ .position
+ .checked_add(N)
+ .ok_or_else(|| invalid("truncated GHSL WKB geometry"))?;
+ let bytes = self
+ .bytes
+ .get(self.position..end)
+ .ok_or_else(|| invalid("truncated GHSL WKB geometry"))?
+ .try_into()
+ .expect("fixed-length slice");
+ self.position = end;
+ Ok(bytes)
+ }
+}
+
+#[derive(Clone, Copy)]
+enum Endian {
+ Little,
+ Big,
+}
+
+fn mollweide_to_wgs84(x: f64, y: f64) -> Result<Coordinate> {
+ let theta = (y / (MOLLWEIDE_RADIUS * SQRT_2)).asin();
+ let latitude = ((2.0 * theta + (2.0 * theta).sin()) / PI).asin();
+ let longitude = PI * x / (2.0 * MOLLWEIDE_RADIUS * SQRT_2 * theta.cos());
+ let coordinate = Coordinate::new(latitude.to_degrees(), longitude.to_degrees());
+ coordinate
+ .is_valid()
+ .then_some(coordinate)
+ .ok_or_else(|| invalid("GHSL geometry has an invalid coordinate"))
+}
+
+fn push_count(output: &mut Vec<u8>, value: usize, message: &str) -> Result<()> {
+ push_u16(output, u16::try_from(value).map_err(|_| invalid(message))?);
+ Ok(())
+}
+
+fn push_u16(output: &mut Vec<u8>, value: u16) {
+ output.extend_from_slice(&value.to_le_bytes());
+}
+
+fn push_len(output: &mut Vec<u8>, value: usize) -> Result<()> {
+ output.extend_from_slice(
+ &u32::try_from(value)
+ .map_err(|_| invalid("urban center index exceeds 4 GiB"))?
+ .to_le_bytes(),
+ );
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use ente_location::CountryCode;
+
+ use super::*;
+
+ #[test]
+ fn encoded_geometry_preserves_polygons_holes_and_metadata() {
+ let centers = vec![
+ center(
+ 7,
+ "Donut",
+ vec![
+ Polygon::new(square(0.0, 0.0, 2.0, 2.0), vec![square(0.5, 0.5, 1.5, 1.5)]),
+ Polygon::new(square(5.0, 5.0, 6.0, 6.0), Vec::new()),
+ ],
+ ),
+ center(
+ 8,
+ "Neighbour",
+ vec![Polygon::new(square(3.0, 3.0, 4.0, 4.0), Vec::new())],
+ ),
+ center(
+ 7,
+ "Donut",
+ vec![Polygon::new(square(7.0, 7.0, 8.0, 8.0), Vec::new())],
+ ),
+ ];
+ let countries = BTreeMap::from([(*b"AA", "Testland".to_owned())]);
+ let index = UrbanCenterIndex::from_bytes(encode(&centers, &countries).unwrap()).unwrap();
+ let coordinates = [
+ Coordinate::new(0.25, 0.25),
+ Coordinate::new(1.0, 1.0),
+ Coordinate::new(3.5, 3.5),
+ Coordinate::new(-0.01, -0.01),
+ Coordinate::new(5.5, 5.5),
+ Coordinate::new(91.0, 0.0),
+ Coordinate::new(7.5, 7.5),
+ ];
+
+ let matches = index.match_coordinates(&coordinates, "");
+
+ assert_eq!(matches.len(), 2);
+ assert_eq!(matches[0].city.name, "Donut");
+ assert_eq!(matches[0].city.source_id, 7);
+ assert_eq!(
+ matches[0].city.country_code,
+ CountryCode::from_bytes(*b"AA").unwrap()
+ );
+ assert_eq!(matches[0].coordinate_indices, [0, 4, 6]);
+ assert_eq!(matches[1].city.name, "Neighbour");
+ assert_eq!(matches[1].coordinate_indices, [2]);
+
+ let filtered = index.match_coordinates(&coordinates, "NEIGH");
+ assert_eq!(filtered.len(), 1);
+ assert_eq!(filtered[0].city.name, "Neighbour");
+ assert_eq!(filtered[0].coordinate_indices, [2]);
+ }
+
+ #[test]
+ fn inverse_mollweide_matches_published_apia_centroid() {
+ let coordinate = mollweide_to_wgs84(-16_907_128.97, -1_704_473.775).unwrap();
+ assert!((coordinate.latitude - -13.836858).abs() < 0.000001);
+ assert!((coordinate.longitude - -171.790524).abs() < 0.000001);
+ }
+
+ #[test]
+ fn canonical_name_city_uses_a_nearby_matching_place() {
+ let cities = [city(1, 47.5, 8.5), city(2, 47.3667, 8.55)];
+
+ assert_eq!(
+ canonical_name_city(&cities, Coordinate::new(47.37, 8.54))
+ .unwrap()
+ .source_id,
+ 2
+ );
+ assert!(canonical_name_city(&cities, Coordinate::new(0.0, 0.0)).is_none());
+ }
+
+ #[test]
+ fn decodes_geopackage_multipolygon_wkb() {
+ let mut bytes = b"GP\0\x01".to_vec();
+ bytes.extend_from_slice(&54_009_i32.to_le_bytes());
+ bytes.push(1);
+ bytes.extend_from_slice(&6_u32.to_le_bytes());
+ bytes.extend_from_slice(&1_u32.to_le_bytes());
+ bytes.push(1);
+ bytes.extend_from_slice(&3_u32.to_le_bytes());
+ bytes.extend_from_slice(&1_u32.to_le_bytes());
+ bytes.extend_from_slice(&4_u32.to_le_bytes());
+ for (x, y) in [(0.0_f64, 0.0_f64), (1000.0, 0.0), (0.0, 1000.0), (0.0, 0.0)] {
+ bytes.extend_from_slice(&x.to_le_bytes());
+ bytes.extend_from_slice(&y.to_le_bytes());
+ }
+
+ let geometry = decode_geopackage_geometry(&bytes).unwrap();
+
+ assert_eq!(geometry.0.len(), 1);
+ assert_eq!(geometry.0[0].exterior().0.len(), 4);
+ assert!(geometry.0[0].exterior().0[1].x > 0.0);
+ assert!(geometry.0[0].exterior().0[2].y > 0.0);
+ }
+
+ fn center(source_id: u32, name: &str, polygons: Vec<Polygon<f64>>) -> SourceCenter {
+ SourceCenter {
+ source_id,
+ name: name.to_owned(),
+ country_code: *b"AA",
+ centroid: Coordinate::new(0.25, 0.25),
+ geometry: MultiPolygon::new(polygons),
+ }
+ }
+
+ fn city(source_id: u32, latitude: f64, longitude: f64) -> City {
+ City {
+ point_index: source_id,
+ source_id,
+ name: "Example".to_owned(),
+ country_name: "Testland".to_owned(),
+ country_code: CountryCode::from_bytes(*b"AA").unwrap(),
+ latitude,
+ longitude,
+ rank: 0,
+ }
+ }
+
+ fn square(minimum_x: f64, minimum_y: f64, maximum_x: f64, maximum_y: f64) -> LineString<f64> {
+ LineString::from(vec![
+ (minimum_x, minimum_y),
+ (maximum_x, minimum_y),
+ (maximum_x, maximum_y),
+ (minimum_x, maximum_y),
+ (minimum_x, minimum_y),
+ ])
+ }
+}
diff --git a/rust/crates/location/src/city.rs b/rust/crates/location/src/city.rs
index c472671dc1..b7b4d77a2a 100644
--- a/rust/crates/location/src/city.rs
+++ b/rust/crates/location/src/city.rs
@@ -10,13 +10,14 @@ use crate::binary::{f32_at, range, u16_at, u24_at, u32_at};
use crate::{Coordinate, CountryCode, Error};
const MAGIC: &[u8; 4] = b"CITY";
-const VERSION: u16 = 1;
-const HEADER_LEN: usize = 72;
+const VERSION: u16 = 2;
+const HEADER_LEN: usize = 76;
const NODE_LEN: usize = 3;
const POINT_LEN: usize = 15;
const NAME_OFFSET_LEN: usize = 3;
const MAX_U24: usize = 0x00ff_ffff;
const MAX_CATCHMENT_KM: f64 = 30.0;
+const MAX_OVERRIDE_DISTANCE_KM: f64 = 3.0;
const SECTION: &str = "city index";
#[derive(Clone, Debug, PartialEq)]
@@ -40,6 +41,7 @@ pub struct CityMatch {
#[derive(Clone, Copy, Debug)]
struct Layout {
point_count: usize,
+ override_count: usize,
name_count: usize,
country_count: usize,
nodes: usize,
@@ -69,6 +71,12 @@ struct SearchBounds {
maximum_longitude: f64,
}
+#[derive(Clone, Copy)]
+struct Tree {
+ nodes: usize,
+ node_count: usize,
+}
+
impl SearchBounds {
const fn new(
minimum_latitude: f64,
@@ -85,13 +93,13 @@ impl SearchBounds {
}
}
-struct SearchPattern {
+pub(crate) struct SearchPattern {
characters: Vec<char>,
fallback: Vec<usize>,
}
impl SearchPattern {
- fn new(query: &str) -> Option<Self> {
+ pub(crate) fn new(query: &str) -> Option<Self> {
let characters: Vec<char> = search_characters(query).collect();
if characters.is_empty() {
return None;
@@ -113,7 +121,7 @@ impl SearchPattern {
})
}
- fn matches(&self, value: &str) -> bool {
+ pub(crate) fn matches(&self, value: &str) -> bool {
let mut matched = 0;
for character in search_characters(value) {
while matched > 0 && character != self.characters[matched] {
@@ -137,6 +145,10 @@ fn search_characters(value: &str) -> impl Iterator<Item = char> + '_ {
.flat_map(char::to_lowercase)
}
+pub fn normalize_search_text(value: &str) -> String {
+ search_characters(value).collect()
+}
+
pub struct CityIndex {
bytes: Box<[u8]>,
layout: Layout,
@@ -186,6 +198,29 @@ impl CityIndex {
}
pub fn match_coordinates(&self, coordinates: &[Coordinate], query: &str) -> Vec<CityMatch> {
+ self.match_coordinates_in(coordinates, query, self.tree(), MAX_CATCHMENT_KM)
+ }
+
+ pub fn match_override_coordinates(
+ &self,
+ coordinates: &[Coordinate],
+ query: &str,
+ ) -> Vec<CityMatch> {
+ self.match_coordinates_in(
+ coordinates,
+ query,
+ self.override_tree(),
+ MAX_OVERRIDE_DISTANCE_KM,
+ )
+ }
+
+ fn match_coordinates_in(
+ &self,
+ coordinates: &[Coordinate],
+ query: &str,
+ tree: Tree,
+ maximum_distance_km: f64,
+ ) -> Vec<CityMatch> {
if coordinates.is_empty() {
return Vec::new();
}
@@ -202,7 +237,13 @@ impl CityIndex {
if !coordinate.is_valid() {
continue;
}
- self.range_around(coordinate, MAX_CATCHMENT_KM, &mut candidates, &mut stack);
+ self.range_around(
+ tree,
+ coordinate,
+ maximum_distance_km,
+ &mut candidates,
+ &mut stack,
+ );
eligible.clear();
for &point_index in &candidates {
let point = self.point(point_index);
@@ -219,7 +260,7 @@ impl CityIndex {
f64::from(point.longitude),
);
let rank = self.rank(point_index);
- if distance_km <= catchment_km(rank) {
+ if distance_km <= maximum_distance_km.min(catchment_km(rank)) {
eligible.push(Candidate {
point: point_index,
distance_km,
@@ -271,6 +312,7 @@ impl CityIndex {
fn range_around(
&self,
+ tree: Tree,
coordinate: Coordinate,
maximum_distance_km: f64,
result: &mut Vec<usize>,
@@ -284,6 +326,7 @@ impl CityIndex {
let maximum_latitude = (coordinate.latitude + latitude_delta).min(90.0);
if longitude_delta >= 180.0 {
self.range_into(
+ tree,
SearchBounds::new(minimum_latitude, maximum_latitude, -180.0, 180.0),
result,
stack,
@@ -295,6 +338,7 @@ impl CityIndex {
let maximum_longitude = coordinate.longitude + longitude_delta;
if minimum_longitude < -180.0 {
self.range_into(
+ tree,
SearchBounds::new(
minimum_latitude,
maximum_latitude,
@@ -305,6 +349,7 @@ impl CityIndex {
stack,
);
self.range_into(
+ tree,
SearchBounds::new(
minimum_latitude,
maximum_latitude,
@@ -316,11 +361,13 @@ impl CityIndex {
);
} else if maximum_longitude > 180.0 {
self.range_into(
+ tree,
SearchBounds::new(minimum_latitude, maximum_latitude, minimum_longitude, 180.0),
result,
stack,
);
self.range_into(
+ tree,
SearchBounds::new(
minimum_latitude,
maximum_latitude,
@@ -332,6 +379,7 @@ impl CityIndex {
);
} else {
self.range_into(
+ tree,
SearchBounds::new(
minimum_latitude,
maximum_latitude,
@@ -346,17 +394,18 @@ impl CityIndex {
fn range_into(
&self,
+ tree: Tree,
bounds: SearchBounds,
result: &mut Vec<usize>,
stack: &mut Vec<(usize, usize, u8)>,
) {
- if self.layout.point_count == 0 {
+ if tree.node_count == 0 {
return;
}
stack.clear();
- stack.push((0, self.layout.point_count, 0));
+ stack.push((0, tree.node_count, 0));
while let Some((node_index, subtree_size, axis)) = stack.pop() {
- let point_index = self.node(node_index);
+ let point_index = self.node(tree, node_index);
let point = self.point(point_index);
let latitude = f64::from(point.latitude);
let longitude = f64::from(point.longitude);
@@ -401,8 +450,22 @@ impl CityIndex {
}
}
- fn node(&self, index: usize) -> usize {
- u24_at(&self.bytes, self.layout.nodes + index * NODE_LEN).expect("validated node") as usize
+ fn tree(&self) -> Tree {
+ Tree {
+ nodes: self.layout.nodes,
+ node_count: self.layout.point_count,
+ }
+ }
+
+ fn override_tree(&self) -> Tree {
+ Tree {
+ nodes: self.layout.nodes + self.layout.point_count * NODE_LEN,
+ node_count: self.layout.override_count,
+ }
+ }
+
+ fn node(&self, tree: Tree, index: usize) -> usize {
+ u24_at(&self.bytes, tree.nodes + index * NODE_LEN).expect("validated node") as usize
}
fn point(&self, index: usize) -> Point {
@@ -495,6 +558,7 @@ fn validate(bytes: &[u8]) -> crate::Result<Layout> {
let layout = Layout {
point_count: read_usize(bytes, 8)?,
+ override_count: read_usize(bytes, 72)?,
name_count: read_usize(bytes, 12)?,
country_count: usize::from(read_u16(bytes, 16)?),
nodes: read_usize(bytes, 24)?,
@@ -515,13 +579,29 @@ fn validate(bytes: &[u8]) -> crate::Result<Layout> {
validate_sections(bytes, layout, declared_length)?;
validate_strings(bytes, layout)?;
validate_points(bytes, layout)?;
- validate_tree(bytes, layout)?;
+ validate_tree(
+ bytes,
+ layout,
+ Tree {
+ nodes: layout.nodes,
+ node_count: layout.point_count,
+ },
+ )?;
+ validate_tree(
+ bytes,
+ layout,
+ Tree {
+ nodes: layout.nodes + layout.point_count * NODE_LEN,
+ node_count: layout.override_count,
+ },
+ )?;
Ok(layout)
}
fn validate_sections(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {
if layout.point_count > MAX_U24
|| layout.name_count > MAX_U24
+ || layout.override_count > layout.point_count
|| layout.country_count > 256
|| (layout.point_count > 0 && (layout.name_count == 0 || layout.country_count == 0))
|| layout.rank_ends[3] > layout.point_count
@@ -531,6 +611,8 @@ fn validate_sections(bytes: &[u8], layout: Layout, declared_length: usize) -> cr
}
let nodes = range(layout.nodes, layout.point_count, NODE_LEN)
.ok_or(invalid("node section overflow"))?;
+ let override_nodes = range(nodes.end, layout.override_count, NODE_LEN)
+ .ok_or(invalid("override-node section overflow"))?;
let points = range(layout.points, layout.point_count, POINT_LEN)
.ok_or(invalid("point section overflow"))?;
let name_offsets = offset_table_range(layout.name_offsets, layout.name_count, NAME_OFFSET_LEN)?;
@@ -554,7 +636,7 @@ fn validate_sections(bytes: &[u8], layout: Layout, declared_length: usize) -> cr
.ok_or(invalid("country section overflow"))?;
if layout.nodes != HEADER_LEN
- || layout.points != nodes.end
+ || layout.points != override_nodes.end
|| layout.name_offsets != points.end
|| layout.names != name_offsets.end
|| layout.country_offsets != names_end
@@ -587,9 +669,7 @@ fn validate_strings(bytes: &[u8], layout: Layout) -> crate::Result<()> {
)?;
for index in 0..layout.name_count {
let value = table_entry_u24(bytes, layout.name_offsets, layout.names, index);
- if let Some((name, alias)) = value.split_once('\0')
- && (name.is_empty() || alias.is_empty() || alias.contains('\0'))
- {
+ if value.split('\0').any(str::is_empty) {
return Err(invalid("invalid city name"));
}
}
@@ -630,23 +710,23 @@ fn validate_points(bytes: &[u8], layout: Layout) -> crate::Result<()> {
Ok(())
}
-fn validate_tree(bytes: &[u8], layout: Layout) -> crate::Result<()> {
- if layout.point_count == 0 {
+fn validate_tree(bytes: &[u8], layout: Layout, tree: Tree) -> crate::Result<()> {
+ if tree.node_count == 0 {
return Ok(());
}
let mut visited_points = vec![false; layout.point_count];
- let mut stack = vec![(0usize, layout.point_count, 0u8, -90.0, 90.0, -180.0, 180.0)];
+ let mut stack = vec![(0usize, tree.node_count, 0u8, -90.0, 90.0, -180.0, 180.0)];
while let Some((node_index, size, axis, min_lat, max_lat, min_lng, max_lng)) = stack.pop() {
if size == 0
|| node_index
.checked_add(size)
- .is_none_or(|end| end > layout.point_count)
+ .is_none_or(|end| end > tree.node_count)
{
return Err(invalid("invalid tree layout"));
}
- let point = u24_at(bytes, layout.nodes + node_index * NODE_LEN)
- .ok_or(invalid("truncated"))? as usize;
+ let point =
+ u24_at(bytes, tree.nodes + node_index * NODE_LEN).ok_or(invalid("truncated"))? as usize;
if point >= layout.point_count || visited_points[point] {
return Err(invalid("invalid tree node"));
}
@@ -709,7 +789,7 @@ fn validate_tree(bytes: &[u8], layout: Layout) -> crate::Result<()> {
}
}
}
- if visited_points.iter().any(|&visited| !visited) {
+ if tree.node_count == layout.point_count && visited_points.iter().any(|&visited| !visited) {
return Err(invalid("tree does not cover every point"));
}
Ok(())
@@ -815,7 +895,7 @@ fn catchment_km(rank: u8) -> f64 {
[8.0, 10.0, 14.0, 20.0, 30.0][usize::from(rank)]
}
-fn distance_km(lat1: f64, lng1: f64, lat2: f64, lng2: f64) -> f64 {
+pub(crate) fn distance_km(lat1: f64, lng1: f64, lat2: f64, lng2: f64) -> f64 {
let lat1 = lat1.to_radians();
let lat2 = lat2.to_radians();
let delta_latitude = lat2 - lat1;
diff --git a/rust/crates/location/src/city_resolver.rs b/rust/crates/location/src/city_resolver.rs
new file mode 100644
index 0000000000..666c924981
--- /dev/null
+++ b/rust/crates/location/src/city_resolver.rs
@@ -0,0 +1,242 @@
+use std::collections::BTreeMap;
+
+use crate::city::distance_km;
+use crate::{City, CityIndex, CityMatch, Coordinate, UrbanCenterIndex};
+
+const OVERRIDE_DISTANCE_MARGIN_KM: f64 = 1.0;
+
+impl UrbanCenterIndex {
+ pub fn match_coordinates_with_cities(
+ &self,
+ cities: &CityIndex,
+ coordinates: &[Coordinate],
+ query: &str,
+ ) -> Vec<CityMatch> {
+ let mut matches = self.match_coordinates(coordinates, query);
+ if matches.is_empty() {
+ return cities.match_coordinates(coordinates, query);
+ }
+ let mut urban_groups = vec![usize::MAX; coordinates.len()];
+ for (group_index, group) in matches.iter().enumerate() {
+ for &index in &group.coordinate_indices {
+ urban_groups[index as usize] = group_index;
+ }
+ }
+ let covered_indices: Vec<u32> = urban_groups
+ .iter()
+ .enumerate()
+ .filter_map(|(index, &group)| (group != usize::MAX).then_some(index as u32))
+ .collect();
+ let mut overrides = cities
+ .match_override_coordinates(&coordinates_at(coordinates, &covered_indices), query);
+ retain_closer_overrides(
+ &mut overrides,
+ &covered_indices,
+ coordinates,
+ &matches,
+ &urban_groups,
+ );
+ replace_matches(&mut matches, &covered_indices, overrides, coordinates.len());
+ let missing_indices: Vec<u32> = urban_groups
+ .iter()
+ .enumerate()
+ .filter_map(|(index, &group)| (group == usize::MAX).then_some(index as u32))
+ .collect();
+ if missing_indices.is_empty() {
+ return matches;
+ }
+ let nearby =
+ cities.match_coordinates(&coordinates_at(coordinates, &missing_indices), query);
+ append_fallback(&mut matches, &missing_indices, nearby);
+ matches
+ }
+}
+
+fn coordinates_at(coordinates: &[Coordinate], indices: &[u32]) -> Vec<Coordinate> {
+ indices
+ .iter()
+ .map(|&index| coordinates[index as usize])
+ .collect()
+}
+
+fn retain_closer_overrides(
+ overrides: &mut Vec<CityMatch>,
+ original_indices: &[u32],
+ coordinates: &[Coordinate],
+ urban: &[CityMatch],
+ urban_groups: &[usize],
+) {
+ for group in overrides.iter_mut() {
+ let city = &group.city;
+ group.coordinate_indices.retain(|&relative_index| {
+ let index = original_indices[relative_index as usize] as usize;
+ distance_to_city(coordinates[index], city) + OVERRIDE_DISTANCE_MARGIN_KM
+ < distance_to_city(coordinates[index], &urban[urban_groups[index]].city)
+ });
+ }
+ overrides.retain(|group| !group.coordinate_indices.is_empty());
+}
+
+fn distance_to_city(coordinate: Coordinate, city: &City) -> f64 {
+ distance_km(
+ coordinate.latitude,
+ coordinate.longitude,
+ city.latitude,
+ city.longitude,
+ )
+}
+
+fn append_fallback(
+ matches: &mut Vec<CityMatch>,
+ original_indices: &[u32],
+ mut fallback: Vec<CityMatch>,
+) {
+ remap_indices(&mut fallback, original_indices);
+ append_matches(matches, fallback);
+}
+
+fn replace_matches(
+ matches: &mut Vec<CityMatch>,
+ original_indices: &[u32],
+ mut replacements: Vec<CityMatch>,
+ coordinate_count: usize,
+) {
+ remap_indices(&mut replacements, original_indices);
+ let mut replaced = vec![false; coordinate_count];
+ for group in &replacements {
+ for &index in &group.coordinate_indices {
+ replaced[index as usize] = true;
+ }
+ }
+ for group in matches.iter_mut() {
+ group
+ .coordinate_indices
+ .retain(|&index| !replaced[index as usize]);
+ }
+ matches.retain(|group| !group.coordinate_indices.is_empty());
+ append_matches(matches, replacements);
+}
+
+fn remap_indices(matches: &mut [CityMatch], original_indices: &[u32]) {
+ for group in matches {
+ for index in &mut group.coordinate_indices {
+ *index = original_indices[*index as usize];
+ }
+ }
+}
+
+fn append_matches(matches: &mut Vec<CityMatch>, additions: Vec<CityMatch>) {
+ let mut positions: BTreeMap<u32, usize> = matches
+ .iter()
+ .enumerate()
+ .map(|(index, group)| (group.city.source_id, index))
+ .collect();
+ for group in additions {
+ if let Some(&position) = positions.get(&group.city.source_id) {
+ matches[position]
+ .coordinate_indices
+ .extend(group.coordinate_indices);
+ matches[position].coordinate_indices.sort_unstable();
+ } else {
+ positions.insert(group.city.source_id, matches.len());
+ matches.push(group);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::CountryCode;
+
+ use super::*;
+
+ #[test]
+ fn fallback_indices_are_mapped_back_to_the_original_coordinates() {
+ let mut matches = vec![city_match(1, "Metropolis", vec![0, 2])];
+ let nearby = vec![
+ city_match(2, "Small Town", vec![0]),
+ city_match(3, "Rural Place", vec![1]),
+ ];
+
+ append_fallback(&mut matches, &[1, 3], nearby);
+
+ assert_eq!(label(&matches, 0), "Metropolis");
+ assert_eq!(label(&matches, 1), "Small Town");
+ assert_eq!(label(&matches, 2), "Metropolis");
+ assert_eq!(label(&matches, 3), "Rural Place");
+ }
+
+ #[test]
+ fn fallback_with_the_same_canonical_city_is_merged() {
+ let mut matches = vec![city_match(1, "Zürich", vec![0])];
+ let nearby = vec![city_match(1, "Zürich", vec![0])];
+
+ append_fallback(&mut matches, &[1], nearby);
+
+ assert_eq!(matches.len(), 1);
+ assert_eq!(matches[0].coordinate_indices, [0, 1]);
+ }
+
+ #[test]
+ fn curated_city_replaces_the_urban_center_for_its_coordinates() {
+ let mut matches = vec![city_match(1, "Metropolis", vec![0, 1, 2])];
+ let prominent = vec![city_match(2, "Edge City", vec![1])];
+
+ replace_matches(&mut matches, &[0, 2], prominent, 3);
+
+ assert_eq!(label(&matches, 0), "Metropolis");
+ assert_eq!(label(&matches, 1), "Metropolis");
+ assert_eq!(label(&matches, 2), "Edge City");
+ }
+
+ #[test]
+ fn override_must_be_materially_closer_than_the_urban_center() {
+ let urban = vec![city_match_at(1, "Metropolis", 0.0, 0.0, vec![0, 1])];
+ let mut overrides = vec![city_match_at(2, "Edge City", 0.0, 0.1, vec![0, 1])];
+
+ retain_closer_overrides(
+ &mut overrides,
+ &[0, 1],
+ &[Coordinate::new(0.0, 0.01), Coordinate::new(0.0, 0.09)],
+ &urban,
+ &[0, 0],
+ );
+
+ assert_eq!(overrides[0].coordinate_indices, [1]);
+ }
+
+ fn label(matches: &[CityMatch], coordinate_index: u32) -> &str {
+ &matches
+ .iter()
+ .find(|group| group.coordinate_indices.contains(&coordinate_index))
+ .unwrap()
+ .city
+ .name
+ }
+
+ fn city_match(source_id: u32, name: &str, coordinate_indices: Vec<u32>) -> CityMatch {
+ city_match_at(source_id, name, 0.0, 0.0, coordinate_indices)
+ }
+
+ fn city_match_at(
+ source_id: u32,
+ name: &str,
+ latitude: f64,
+ longitude: f64,
+ coordinate_indices: Vec<u32>,
+ ) -> CityMatch {
+ CityMatch {
+ city: City {
+ point_index: 0,
+ source_id,
+ name: name.to_owned(),
+ country_name: String::new(),
+ country_code: CountryCode::from_bytes(*b"AA").unwrap(),
+ latitude,
+ longitude,
+ rank: 0,
+ },
+ coordinate_indices,
+ }
+ }
+}
diff --git a/rust/crates/location/src/country.rs b/rust/crates/location/src/country.rs
index 9f2492f1b1..e8cab43f1c 100644
--- a/rust/crates/location/src/country.rs
+++ b/rust/crates/location/src/country.rs
@@ -1,4 +1,5 @@
use crate::binary::{array, range, u16_at, u32_at};
+use crate::geometry::{Point, Reader};
use crate::{CountryCode, Error};
const MAGIC: &[u8; 4] = b"CTRY";
@@ -135,7 +136,7 @@ impl CountryGeometry {
}
fn lookup_cell(&self, bytes: &[u8], point: Point) -> crate::Result<Vec<CountryCode>> {
- let mut reader = Reader::new(bytes);
+ let mut reader = Reader::new(bytes, SECTION);
let mut matches = Vec::new();
for _ in 0..reader.byte()? {
push_unique(&mut matches, self.country_code(reader.byte()?));
@@ -157,7 +158,7 @@ impl CountryGeometry {
push_unique(&mut matches, country);
}
}
- debug_assert_eq!(reader.position, bytes.len());
+ debug_assert!(reader.is_done());
Ok(matches)
}
}
@@ -171,66 +172,6 @@ pub(crate) struct PreparedCell {
index: usize,
}
-#[derive(Clone, Copy)]
-struct Point {
- x: u16,
- y: u16,
-}
-
-struct Reader<'a> {
- bytes: &'a [u8],
- position: usize,
-}
-
-impl<'a> Reader<'a> {
- const fn new(bytes: &'a [u8]) -> Self {
- Self { bytes, position: 0 }
- }
-
- fn byte(&mut self) -> crate::Result<u8> {
- let value = *self.bytes.get(self.position).ok_or(invalid("truncated"))?;
- self.position += 1;
- Ok(value)
- }
-
- fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {
- let end = self
- .position
- .checked_add(count)
- .ok_or(invalid("truncated"))?;
- let bytes = self
- .bytes
- .get(self.position..end)
- .ok_or(invalid("truncated"))?;
- self.position = end;
- Ok(bytes)
- }
-
- fn ring_contains(&mut self, point: Point) -> crate::Result<bool> {
- let count = usize::from(u16::from_le_bytes(
- self.take(2)?.try_into().expect("two-byte slice"),
- ));
- if count == 0 {
- return Ok(false);
- }
- let ring = self.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;
- let mut winding = 0_i32;
- let mut previous = point_at(ring, count - 1);
- for index in 0..count {
- let current = point_at(ring, index);
- if previous.y <= point.y {
- if current.y > point.y && is_left(previous, current, point) > 0 {
- winding += 1;
- }
- } else if current.y <= point.y && is_left(previous, current, point) < 0 {
- winding -= 1;
- }
- previous = current;
- }
- Ok(winding != 0)
- }
-}
-
fn validate(bytes: &[u8]) -> crate::Result<Layout> {
if bytes.len() < HEADER_LEN {
return Err(invalid("truncated"));
@@ -366,7 +307,7 @@ fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crat
}
fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {
- let mut reader = Reader::new(bytes);
+ let mut reader = Reader::new(bytes, SECTION);
for _ in 0..reader.byte()? {
validate_country(reader.byte()?, country_count)?;
}
@@ -375,7 +316,7 @@ fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {
skip_rings(&mut reader)?;
skip_rings(&mut reader)?;
}
- if reader.position != bytes.len() {
+ if !reader.is_done() {
return Err(invalid("cell has trailing bytes"));
}
Ok(())
@@ -383,9 +324,7 @@ fn validate_cell(bytes: &[u8], country_count: usize) -> crate::Result<()> {
fn skip_rings(reader: &mut Reader<'_>) -> crate::Result<()> {
for _ in 0..reader.byte()? {
- let count = usize::from(u16::from_le_bytes(
- reader.take(2)?.try_into().expect("two-byte slice"),
- ));
+ let count = usize::from(reader.u16()?);
reader.take(count.checked_mul(4).ok_or(invalid("ring size overflow"))?)?;
}
Ok(())
@@ -399,19 +338,6 @@ fn validate_country(index: u8, country_count: usize) -> crate::Result<()> {
}
}
-fn point_at(ring: &[u8], index: usize) -> Point {
- let offset = index * 4;
- Point {
- x: u16::from_le_bytes([ring[offset], ring[offset + 1]]),
- y: u16::from_le_bytes([ring[offset + 2], ring[offset + 3]]),
- }
-}
-
-fn is_left(start: Point, end: Point, point: Point) -> i64 {
- (i64::from(end.x) - i64::from(start.x)) * (i64::from(point.y) - i64::from(start.y))
- - (i64::from(point.x) - i64::from(start.x)) * (i64::from(end.y) - i64::from(start.y))
-}
-
fn normalize(value: f64, minimum: f64, width: f64) -> f64 {
(value - minimum).rem_euclid(width) + minimum
}
diff --git a/rust/crates/location/src/geometry.rs b/rust/crates/location/src/geometry.rs
new file mode 100644
index 0000000000..ff717bdb5d
--- /dev/null
+++ b/rust/crates/location/src/geometry.rs
@@ -0,0 +1,98 @@
+use crate::Error;
+
+#[derive(Clone, Copy)]
+pub(crate) struct Point {
+ pub x: u16,
+ pub y: u16,
+}
+
+pub(crate) struct Reader<'a> {
+ bytes: &'a [u8],
+ position: usize,
+ section: &'static str,
+}
+
+impl<'a> Reader<'a> {
+ pub(crate) const fn new(bytes: &'a [u8], section: &'static str) -> Self {
+ Self {
+ bytes,
+ position: 0,
+ section,
+ }
+ }
+
+ pub(crate) const fn is_done(&self) -> bool {
+ self.position == self.bytes.len()
+ }
+
+ pub(crate) fn byte(&mut self) -> crate::Result<u8> {
+ let value = *self
+ .bytes
+ .get(self.position)
+ .ok_or(self.invalid("truncated"))?;
+ self.position += 1;
+ Ok(value)
+ }
+
+ pub(crate) fn u16(&mut self) -> crate::Result<u16> {
+ Ok(u16::from_le_bytes(
+ self.take(2)?.try_into().expect("two-byte geometry slice"),
+ ))
+ }
+
+ pub(crate) fn take(&mut self, count: usize) -> crate::Result<&'a [u8]> {
+ let end = self
+ .position
+ .checked_add(count)
+ .ok_or(self.invalid("truncated"))?;
+ let bytes = self
+ .bytes
+ .get(self.position..end)
+ .ok_or(self.invalid("truncated"))?;
+ self.position = end;
+ Ok(bytes)
+ }
+
+ pub(crate) fn ring_contains(&mut self, point: Point) -> crate::Result<bool> {
+ let count = usize::from(self.u16()?);
+ if count == 0 {
+ return Ok(false);
+ }
+ let ring = self.take(
+ count
+ .checked_mul(4)
+ .ok_or(self.invalid("ring size overflow"))?,
+ )?;
+ let mut winding = 0_i32;
+ let mut previous = point_at(ring, count - 1);
+ for index in 0..count {
+ let current = point_at(ring, index);
+ if previous.y <= point.y {
+ if current.y > point.y && is_left(previous, current, point) > 0 {
+ winding += 1;
+ }
+ } else if current.y <= point.y && is_left(previous, current, point) < 0 {
+ winding -= 1;
+ }
+ previous = current;
+ }
+ Ok(winding != 0)
+ }
+
+ fn invalid(&self, reason: &'static str) -> Error {
+ Error::invalid(self.section, reason)
+ }
+}
+
+fn point_at(ring: &[u8], index: usize) -> Point {
+ let offset = index * 4;
+ Point {
+ x: u16::from_le_bytes([ring[offset], ring[offset + 1]]),
+ y: u16::from_le_bytes([ring[offset + 2], ring[offset + 3]]),
+ }
+}
+
+fn is_left(start: Point, end: Point, point: Point) -> i64 {
+ (i64::from(end.x) - i64::from(start.x)) * (i64::from(point.y) - i64::from(start.y))
+ - (i64::from(point.x) - i64::from(start.x)) * (i64::from(end.y) - i64::from(start.y))
+}
diff --git a/rust/crates/location/src/lib.rs b/rust/crates/location/src/lib.rs
index 8b4f69de7b..3451bbb5eb 100644
--- a/rust/crates/location/src/lib.rs
+++ b/rust/crates/location/src/lib.rs
@@ -6,14 +6,18 @@ use std::str::FromStr;
mod binary;
mod city;
+mod city_resolver;
pub mod cluster;
mod country;
mod dispute;
mod error;
+mod geometry;
+mod urban;
-pub use city::{City, CityIndex, CityMatch};
+pub use city::{City, CityIndex, CityMatch, normalize_search_text};
pub use dispute::{CountryView, DisputeMatch, TerritoryId};
pub use error::{Error, Result};
+pub use urban::UrbanCenterIndex;
use country::CountryGeometry;
use dispute::DisputeIndex;
diff --git a/rust/crates/location/src/urban.rs b/rust/crates/location/src/urban.rs
new file mode 100644
index 0000000000..4939b95b95
--- /dev/null
+++ b/rust/crates/location/src/urban.rs
@@ -0,0 +1,513 @@
+use std::collections::BTreeMap;
+use std::fs;
+use std::path::Path;
+
+use crate::binary::{array, f32_at, range, u16_at, u32_at};
+use crate::city::SearchPattern;
+use crate::geometry::{Point, Reader as GeometryReader};
+use crate::{City, CityMatch, Coordinate, CountryCode, Error};
+
+const MAGIC: &[u8; 4] = b"URBN";
+const VERSION: u16 = 1;
+const HEADER_LEN: usize = 72;
+const RECORD_LEN: usize = 41;
+const BLOCK_SIZE: usize = 64;
+const BLOCK_LEN: usize = 12;
+const SECTION: &str = "urban center index";
+
+#[derive(Clone, Copy)]
+struct Layout {
+ feature_count: usize,
+ country_count: usize,
+ columns: usize,
+ rows: usize,
+ block_count: usize,
+ nonempty_count: usize,
+ reference_count: usize,
+ records: usize,
+ names: usize,
+ geometry: usize,
+ country_offsets: usize,
+ countries: usize,
+ country_codes: usize,
+ blocks: usize,
+ cell_offsets: usize,
+ references: usize,
+}
+
+#[derive(Clone, Copy)]
+struct Feature {
+ minimum_longitude: f32,
+ minimum_latitude: f32,
+ maximum_longitude: f32,
+ maximum_latitude: f32,
+ latitude: f32,
+ longitude: f32,
+ name: usize,
+ geometry_start: usize,
+ geometry_end: usize,
+ source_id: u32,
+ country: usize,
+}
+
+pub struct UrbanCenterIndex {
+ bytes: Box<[u8]>,
+ layout: Layout,
+}
+
+impl UrbanCenterIndex {
+ pub fn from_path(path: impl AsRef<Path>) -> crate::Result<Self> {
+ Self::from_bytes(fs::read(path)?)
+ }
+
+ pub fn from_bytes(bytes: impl Into<Box<[u8]>>) -> crate::Result<Self> {
+ let bytes = bytes.into();
+ let layout = validate(&bytes)?;
+ Ok(Self { bytes, layout })
+ }
+
+ pub const fn len(&self) -> usize {
+ self.layout.feature_count
+ }
+
+ pub const fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ pub fn match_coordinates(&self, coordinates: &[Coordinate], query: &str) -> Vec<CityMatch> {
+ let matching: Option<Vec<bool>> = SearchPattern::new(query).map(|pattern| {
+ (0..self.layout.feature_count)
+ .map(|feature| pattern.matches(self.name(self.feature(feature).name)))
+ .collect()
+ });
+ if matching
+ .as_ref()
+ .is_some_and(|matching| !matching.iter().any(|&matched| matched))
+ {
+ return Vec::new();
+ }
+ let mut matches = BTreeMap::<usize, Vec<u32>>::new();
+ for (coordinate_index, &coordinate) in coordinates.iter().enumerate() {
+ if let Some(feature) = self.lookup(coordinate, matching.as_deref()) {
+ matches
+ .entry(feature)
+ .or_default()
+ .push(coordinate_index as u32);
+ }
+ }
+ let mut groups = BTreeMap::<u32, CityMatch>::new();
+ for (feature, coordinate_indices) in matches {
+ let city = self.city(feature);
+ groups
+ .entry(city.source_id)
+ .and_modify(|group| {
+ group
+ .coordinate_indices
+ .extend_from_slice(&coordinate_indices);
+ group.coordinate_indices.sort_unstable();
+ })
+ .or_insert(CityMatch {
+ city,
+ coordinate_indices,
+ });
+ }
+ groups.into_values().collect()
+ }
+
+ fn lookup(&self, coordinate: Coordinate, matching: Option<&[bool]>) -> Option<usize> {
+ if !coordinate.is_valid() {
+ return None;
+ }
+ let column = (((coordinate.longitude + 180.0) * self.layout.columns as f64 / 360.0).floor()
+ as usize)
+ .min(self.layout.columns - 1);
+ let row = ((self.layout.rows as f64 * (90.0 - coordinate.latitude) / 180.0).ceil()
+ as usize)
+ .saturating_sub(1);
+ let dense = self.dense_cell_index(row * self.layout.columns + column)?;
+ let start = self.cell_offset(dense);
+ let end = self.cell_offset(dense + 1);
+ for reference in start..end {
+ let feature = usize::from(
+ u16_at(&self.bytes, self.layout.references + reference * 2)
+ .expect("validated reference"),
+ );
+ if matching.is_some_and(|matching| !matching[feature]) {
+ continue;
+ }
+ if self.contains(feature, coordinate) {
+ return Some(feature);
+ }
+ }
+ None
+ }
+
+ fn contains(&self, index: usize, coordinate: Coordinate) -> bool {
+ let feature = self.feature(index);
+ let longitude = coordinate.longitude as f32;
+ let latitude = coordinate.latitude as f32;
+ if longitude < feature.minimum_longitude
+ || longitude > feature.maximum_longitude
+ || latitude < feature.minimum_latitude
+ || latitude > feature.maximum_latitude
+ {
+ return false;
+ }
+ let point = Point {
+ x: quantize(
+ coordinate.longitude,
+ f64::from(feature.minimum_longitude),
+ f64::from(feature.maximum_longitude),
+ ),
+ y: quantize(
+ coordinate.latitude,
+ f64::from(feature.minimum_latitude),
+ f64::from(feature.maximum_latitude),
+ ),
+ };
+ let mut reader = GeometryReader::new(
+ &self.bytes[feature.geometry_start..feature.geometry_end],
+ SECTION,
+ );
+ for _ in 0..reader.u16().expect("validated polygon count") {
+ let ring_count = reader.u16().expect("validated ring count");
+ let mut inside = reader.ring_contains(point).expect("validated ring");
+ for _ in 1..ring_count {
+ if reader.ring_contains(point).expect("validated ring") {
+ inside = false;
+ }
+ }
+ if inside {
+ return true;
+ }
+ }
+ false
+ }
+
+ fn city(&self, index: usize) -> City {
+ let feature = self.feature(index);
+ City {
+ point_index: index as u32,
+ source_id: feature.source_id,
+ name: self.name(feature.name).to_owned(),
+ country_name: self.country(feature.country).to_owned(),
+ country_code: self.country_code(feature.country),
+ latitude: f64::from(feature.latitude),
+ longitude: f64::from(feature.longitude),
+ rank: 4,
+ }
+ }
+
+ fn feature(&self, index: usize) -> Feature {
+ let offset = self.layout.records + index * RECORD_LEN;
+ Feature {
+ minimum_longitude: f32_at(&self.bytes, offset).expect("validated feature"),
+ minimum_latitude: f32_at(&self.bytes, offset + 4).expect("validated feature"),
+ maximum_longitude: f32_at(&self.bytes, offset + 8).expect("validated feature"),
+ maximum_latitude: f32_at(&self.bytes, offset + 12).expect("validated feature"),
+ latitude: f32_at(&self.bytes, offset + 16).expect("validated feature"),
+ longitude: f32_at(&self.bytes, offset + 20).expect("validated feature"),
+ name: u32_at(&self.bytes, offset + 24).expect("validated feature") as usize,
+ geometry_start: u32_at(&self.bytes, offset + 28).expect("validated feature") as usize,
+ geometry_end: u32_at(&self.bytes, offset + 32).expect("validated feature") as usize,
+ source_id: u32_at(&self.bytes, offset + 36).expect("validated feature"),
+ country: usize::from(self.bytes[offset + 40]),
+ }
+ }
+
+ fn name(&self, offset: usize) -> &str {
+ let end = self.bytes[offset..self.layout.geometry]
+ .iter()
+ .position(|&byte| byte == 0)
+ .map(|length| offset + length)
+ .expect("validated name terminator");
+ std::str::from_utf8(&self.bytes[offset..end]).expect("validated name")
+ }
+
+ fn country(&self, index: usize) -> &str {
+ let start = u32_at(&self.bytes, self.layout.country_offsets + index * 4)
+ .expect("validated country offset") as usize;
+ let end = u32_at(&self.bytes, self.layout.country_offsets + (index + 1) * 4)
+ .expect("validated country offset") as usize;
+ std::str::from_utf8(&self.bytes[self.layout.countries + start..self.layout.countries + end])
+ .expect("validated country")
+ }
+
+ fn country_code(&self, index: usize) -> CountryCode {
+ let offset = self.layout.country_codes + index * 2;
+ CountryCode::from_validated([self.bytes[offset], self.bytes[offset + 1]])
+ }
+
+ fn dense_cell_index(&self, cell_index: usize) -> Option<usize> {
+ let block_index = cell_index / BLOCK_SIZE;
+ let bit_index = cell_index % BLOCK_SIZE;
+ let offset = self.layout.blocks + block_index * BLOCK_LEN;
+ let mask = u64::from_le_bytes(array(&self.bytes, offset).expect("validated block"));
+ let bit = 1_u64 << bit_index;
+ (mask & bit != 0).then(|| {
+ let first = u32_at(&self.bytes, offset + 8).expect("validated block") as usize;
+ first + (mask & bit.wrapping_sub(1)).count_ones() as usize
+ })
+ }
+
+ fn cell_offset(&self, dense_index: usize) -> usize {
+ u32_at(&self.bytes, self.layout.cell_offsets + dense_index * 4)
+ .expect("validated cell offset") as usize
+ }
+}
+
+fn validate(bytes: &[u8]) -> crate::Result<Layout> {
+ if bytes.len() < HEADER_LEN || &bytes[..4] != MAGIC {
+ return Err(invalid("truncated or invalid magic"));
+ }
+ let version = u16_at(bytes, 4).ok_or(invalid("truncated"))?;
+ if version != VERSION {
+ return Err(Error::invalid(
+ SECTION,
+ format!("unsupported format version {version}"),
+ ));
+ }
+ if usize::from(u16_at(bytes, 6).ok_or(invalid("truncated"))?) != HEADER_LEN
+ || usize::from(u16_at(bytes, 18).ok_or(invalid("truncated"))?) != BLOCK_SIZE
+ {
+ return Err(invalid("unexpected header or block length"));
+ }
+ let layout = Layout {
+ feature_count: read_usize(bytes, 8)?,
+ country_count: usize::from(u16_at(bytes, 12).ok_or(invalid("truncated"))?),
+ columns: usize::from(u16_at(bytes, 14).ok_or(invalid("truncated"))?),
+ rows: usize::from(u16_at(bytes, 16).ok_or(invalid("truncated"))?),
+ block_count: read_usize(bytes, 20)?,
+ nonempty_count: read_usize(bytes, 24)?,
+ records: read_usize(bytes, 28)?,
+ names: read_usize(bytes, 32)?,
+ geometry: read_usize(bytes, 36)?,
+ country_offsets: read_usize(bytes, 40)?,
+ countries: read_usize(bytes, 44)?,
+ country_codes: read_usize(bytes, 48)?,
+ blocks: read_usize(bytes, 52)?,
+ cell_offsets: read_usize(bytes, 56)?,
+ references: read_usize(bytes, 60)?,
+ reference_count: read_usize(bytes, 68)?,
+ };
+ let declared_length = read_usize(bytes, 64)?;
+ validate_layout(bytes, layout, declared_length)?;
+ Ok(layout)
+}
+
+fn validate_layout(bytes: &[u8], layout: Layout, declared_length: usize) -> crate::Result<()> {
+ let cell_count = layout
+ .columns
+ .checked_mul(layout.rows)
+ .ok_or(invalid("cell count overflow"))?;
+ if layout.feature_count == 0
+ || layout.feature_count > usize::from(u16::MAX)
+ || layout.country_count == 0
+ || layout.country_count > 256
+ || layout.columns == 0
+ || layout.rows == 0
+ || layout.block_count != cell_count.div_ceil(BLOCK_SIZE)
+ || layout.nonempty_count > cell_count
+ {
+ return Err(invalid("invalid index counts"));
+ }
+ let records = range(layout.records, layout.feature_count, RECORD_LEN)
+ .ok_or(invalid("record section overflow"))?;
+ let country_offsets = range(layout.country_offsets, layout.country_count + 1, 4)
+ .ok_or(invalid("country offset section overflow"))?;
+ let country_codes = range(layout.country_codes, layout.country_count, 2)
+ .ok_or(invalid("country code section overflow"))?;
+ let blocks = range(layout.blocks, layout.block_count, BLOCK_LEN)
+ .ok_or(invalid("block section overflow"))?;
+ let cell_offsets = range(layout.cell_offsets, layout.nonempty_count + 1, 4)
+ .ok_or(invalid("cell offset section overflow"))?;
+ let references = range(layout.references, layout.reference_count, 2)
+ .ok_or(invalid("reference section overflow"))?;
+ if layout.records != HEADER_LEN
+ || layout.names != records.end
+ || layout.names > layout.geometry
+ || layout.geometry > layout.country_offsets
+ || layout.countries != country_offsets.end
+ || layout.countries > layout.country_codes
+ || layout.blocks != country_codes.end
+ || layout.cell_offsets != blocks.end
+ || layout.references != cell_offsets.end
+ || references.end != bytes.len()
+ || declared_length != bytes.len()
+ {
+ return Err(invalid("sections are not contiguous"));
+ }
+ validate_countries(bytes, layout)?;
+ validate_features(bytes, layout)?;
+ validate_grid(bytes, layout, cell_count)?;
+ Ok(())
+}
+
+fn validate_countries(bytes: &[u8], layout: Layout) -> crate::Result<()> {
+ let country_bytes = layout.country_codes - layout.countries;
+ let mut previous = 0;
+ let mut seen_codes = [false; 26 * 26];
+ for index in 0..layout.country_count {
+ let offset = u32_at(bytes, layout.country_offsets + index * 4)
+ .ok_or(invalid("truncated country offset"))? as usize;
+ let end = u32_at(bytes, layout.country_offsets + (index + 1) * 4)
+ .ok_or(invalid("truncated country offset"))? as usize;
+ if offset != previous || end < offset || end > country_bytes {
+ return Err(invalid("invalid country offsets"));
+ }
+ std::str::from_utf8(&bytes[layout.countries + offset..layout.countries + end])
+ .map_err(|_| invalid("country name is not UTF-8"))?;
+ previous = end;
+
+ let code_offset = layout.country_codes + index * 2;
+ let code = [bytes[code_offset], bytes[code_offset + 1]];
+ if !code.iter().all(u8::is_ascii_uppercase) {
+ return Err(invalid("invalid country code"));
+ }
+ let slot = usize::from(code[0] - b'A') * 26 + usize::from(code[1] - b'A');
+ if seen_codes[slot] {
+ return Err(invalid("duplicate country code"));
+ }
+ seen_codes[slot] = true;
+ }
+ if previous != country_bytes {
+ return Err(invalid("last country offset is not terminal"));
+ }
+ Ok(())
+}
+
+fn validate_features(bytes: &[u8], layout: Layout) -> crate::Result<()> {
+ for index in 0..layout.feature_count {
+ let offset = layout.records + index * RECORD_LEN;
+ let values = [
+ f32_at(bytes, offset).ok_or(invalid("truncated feature"))?,
+ f32_at(bytes, offset + 4).ok_or(invalid("truncated feature"))?,
+ f32_at(bytes, offset + 8).ok_or(invalid("truncated feature"))?,
+ f32_at(bytes, offset + 12).ok_or(invalid("truncated feature"))?,
+ f32_at(bytes, offset + 16).ok_or(invalid("truncated feature"))?,
+ f32_at(bytes, offset + 20).ok_or(invalid("truncated feature"))?,
+ ];
+ if values.iter().any(|value| !value.is_finite())
+ || !(-180.0..=180.0).contains(&values[0])
+ || !(-90.0..=90.0).contains(&values[1])
+ || !(-180.0..=180.0).contains(&values[2])
+ || !(-90.0..=90.0).contains(&values[3])
+ || values[0] >= values[2]
+ || values[1] >= values[3]
+ || !(-90.0..=90.0).contains(&values[4])
+ || !(-180.0..=180.0).contains(&values[5])
+ {
+ return Err(invalid("feature has invalid coordinates"));
+ }
+ let name = u32_at(bytes, offset + 24).ok_or(invalid("truncated feature"))? as usize;
+ if !(layout.names..layout.geometry).contains(&name) {
+ return Err(invalid("feature name is outside the name section"));
+ }
+ let end = bytes[name..layout.geometry]
+ .iter()
+ .position(|&byte| byte == 0)
+ .ok_or(invalid("unterminated feature name"))?;
+ std::str::from_utf8(&bytes[name..name + end])
+ .map_err(|_| invalid("feature name is not UTF-8"))?;
+ let geometry_start =
+ u32_at(bytes, offset + 28).ok_or(invalid("truncated feature"))? as usize;
+ let geometry_end = u32_at(bytes, offset + 32).ok_or(invalid("truncated feature"))? as usize;
+ if geometry_start < layout.geometry
+ || geometry_end <= geometry_start
+ || geometry_end > layout.country_offsets
+ {
+ return Err(invalid("feature geometry is outside the geometry section"));
+ }
+ validate_geometry(&bytes[geometry_start..geometry_end])?;
+ let country = usize::from(bytes[offset + 40]);
+ if country >= layout.country_count {
+ return Err(invalid("feature references unknown country"));
+ }
+ }
+ Ok(())
+}
+
+fn validate_geometry(bytes: &[u8]) -> crate::Result<()> {
+ let mut reader = GeometryReader::new(bytes, SECTION);
+ let polygon_count = reader.u16()?;
+ if polygon_count == 0 {
+ return Err(invalid("feature geometry has no polygons"));
+ }
+ for _ in 0..polygon_count {
+ let ring_count = reader.u16()?;
+ if ring_count == 0 {
+ return Err(invalid("feature polygon has no rings"));
+ }
+ for _ in 0..ring_count {
+ let point_count = usize::from(reader.u16()?);
+ if point_count < 4 {
+ return Err(invalid("feature ring is too short"));
+ }
+ reader.take(
+ point_count
+ .checked_mul(4)
+ .ok_or(invalid("ring size overflow"))?,
+ )?;
+ }
+ }
+ if !reader.is_done() {
+ return Err(invalid("feature geometry has trailing bytes"));
+ }
+ Ok(())
+}
+
+fn validate_grid(bytes: &[u8], layout: Layout, cell_count: usize) -> crate::Result<()> {
+ let mut expected_dense = 0;
+ for block in 0..layout.block_count {
+ let offset = layout.blocks + block * BLOCK_LEN;
+ let mask = u64::from_le_bytes(array(bytes, offset).expect("validated block range"));
+ let first = u32_at(bytes, offset + 8).expect("validated block range") as usize;
+ if first != expected_dense {
+ return Err(invalid("invalid block dense offset"));
+ }
+ if block + 1 == layout.block_count {
+ let used_bits = cell_count - block * BLOCK_SIZE;
+ if used_bits < BLOCK_SIZE && mask >> used_bits != 0 {
+ return Err(invalid("last block references nonexistent cells"));
+ }
+ }
+ expected_dense += mask.count_ones() as usize;
+ }
+ if expected_dense != layout.nonempty_count {
+ return Err(invalid("wrong nonempty cell count"));
+ }
+ let mut previous = 0;
+ for index in 0..=layout.nonempty_count {
+ let offset = u32_at(bytes, layout.cell_offsets + index * 4)
+ .expect("validated cell offset range") as usize;
+ if offset < previous || offset > layout.reference_count {
+ return Err(invalid("invalid cell offset"));
+ }
+ previous = offset;
+ }
+ if previous != layout.reference_count {
+ return Err(invalid("last cell offset is not terminal"));
+ }
+ for index in 0..layout.reference_count {
+ let feature = usize::from(
+ u16_at(bytes, layout.references + index * 2).expect("validated reference range"),
+ );
+ if feature >= layout.feature_count {
+ return Err(invalid("cell references unknown feature"));
+ }
+ }
+ Ok(())
+}
+
+fn quantize(value: f64, minimum: f64, maximum: f64) -> u16 {
+ (((value - minimum) * f64::from(u16::MAX) / (maximum - minimum))
+ .clamp(0.0, f64::from(u16::MAX))) as u16
+}
+
+fn read_usize(bytes: &[u8], offset: usize) -> crate::Result<usize> {
+ Ok(u32_at(bytes, offset).ok_or(invalid("truncated"))? as usize)
+}
+
+fn invalid(reason: &'static str) -> Error {
+ Error::invalid(SECTION, reason)
+}
diff --git a/rust/crates/photos/src/location.rs b/rust/crates/photos/src/location.rs
index 3994da781d..11444df519 100644
--- a/rust/crates/photos/src/location.rs
+++ b/rust/crates/photos/src/location.rs
@@ -2,10 +2,13 @@ use std::path::Path;
use ente_assets::download::CancellationToken;
use ente_assets::{Asset, AssetFile, AssetStore};
-use ente_location::{CityIndex, CityMatch, Coordinate, CountryIndex, CountryView};
+use ente_location::{
+ CityIndex, CityMatch, Coordinate, CountryIndex, CountryView, UrbanCenterIndex,
+};
-const ASSET_URL: &str = "https://assets.ente.com";
+const ASSET_URL: &str = "https://assets.ente.com/location/v2";
const CITY_FILE: &str = "cities.bin";
+const URBAN_CENTER_FILE: &str = "urban-centres.bin";
const COUNTRY_FILE: &str = "countries.bin";
const DISPUTE_FILE: &str = "disputes.bin";
@@ -21,6 +24,7 @@ pub enum Error {
pub struct LocationIndex {
cities: CityIndex,
+ urban_centers: UrbanCenterIndex,
countries: CountryIndex,
}
@@ -34,6 +38,7 @@ impl LocationIndex {
.await?;
Ok(Self {
cities: CityIndex::from_path(directory.join(CITY_FILE))?,
+ urban_centers: UrbanCenterIndex::from_path(directory.join(URBAN_CENTER_FILE))?,
countries: CountryIndex::from_paths(
directory.join(COUNTRY_FILE),
directory.join(DISPUTE_FILE),
@@ -46,7 +51,8 @@ impl LocationIndex {
}
pub fn group_cities(&self, coordinates: &[Coordinate], query: &str) -> Vec<CityMatch> {
- self.cities.match_coordinates(coordinates, query)
+ self.urban_centers
+ .match_coordinates_with_cities(&self.cities, coordinates, query)
}
pub fn group_countries(
@@ -64,11 +70,15 @@ impl LocationIndex {
fn asset() -> Result<Asset, ente_assets::download::Error> {
Asset::files(
- vec!["location".into(), "v1".into()],
+ vec!["location".into(), "v2".into()],
vec![
file(
CITY_FILE,
- "929a488aeac782d1cfc9e90b3eeb9536423977aab9f21cb1f86b9324ac53efbc",
+ "898dca892a71fd601ae8e75e5c55fd6d4591e4c98de335d9b33e7f076aa668f5",
+ ),
+ file(
+ URBAN_CENTER_FILE,
+ "e981b0383e6502ede56b52d5be96cda66f53d7352492e677d8978467c106dab0",
),
file(
COUNTRY_FILE,