Skip to content

High performance self-hosted photo and video management solution.

fix(server): use GeoNames for reverse-geocoded country names (#30199)

Rufus Uttleyaa023378477f
diff --git a/e2e/src/specs/server/api/map.e2e-spec.ts b/e2e/src/specs/server/api/map.e2e-spec.ts
index cf6a2abdb0..d70b047eaa 100644
--- a/e2e/src/specs/server/api/map.e2e-spec.ts
+++ b/e2e/src/specs/server/api/map.e2e-spec.ts
@@ -65,7 +65,7 @@ describe('/map', () => {
expect(body).toEqual([
{
city: 'Palisade',
- country: 'United States of America',
+ country: 'United States',
id: expect.any(String),
lat: expect.closeTo(39.115),
lon: expect.closeTo(-108.400968),
@@ -73,7 +73,7 @@ describe('/map', () => {
},
{
city: 'Ralston',
- country: 'United States of America',
+ country: 'United States',
id: expect.any(String),
lat: expect.closeTo(41.2203),
lon: expect.closeTo(-96.071625),
@@ -113,7 +113,7 @@ describe('/map', () => {
expect(body).toEqual([
{
city: 'Palisade',
- country: 'United States of America',
+ country: 'United States',
id: expect.any(String),
lat: expect.closeTo(39.115),
lon: expect.closeTo(-108.400968),
@@ -121,7 +121,7 @@ describe('/map', () => {
},
{
city: 'Ralston',
- country: 'United States of America',
+ country: 'United States',
id: expect.any(String),
lat: expect.closeTo(41.2203),
lon: expect.closeTo(-96.071625),
diff --git a/e2e/src/specs/server/api/search.e2e-spec.ts b/e2e/src/specs/server/api/search.e2e-spec.ts
index 82ff06138e..e4616bd032 100644
--- a/e2e/src/specs/server/api/search.e2e-spec.ts
+++ b/e2e/src/specs/server/api/search.e2e-spec.ts
@@ -491,6 +491,7 @@ describe('/search', () => {
.get('/search/suggestions?type=country&includeNull=true')
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(body).toEqual([
+ 'China',
'Cuba',
'France',
'Georgia',
@@ -498,12 +499,11 @@ describe('/search', () => {
'Ghana',
'Japan',
'Morocco',
- "People's Republic of China",
- 'Russian Federation',
+ 'Russia',
'Singapore',
'Spain',
'Switzerland',
- 'United States of America',
+ 'United States',
null,
]);
expect(status).toBe(200);
@@ -514,6 +514,7 @@ describe('/search', () => {
.get('/search/suggestions?type=country')
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(body).toEqual([
+ 'China',
'Cuba',
'France',
'Georgia',
@@ -521,12 +522,11 @@ describe('/search', () => {
'Ghana',
'Japan',
'Morocco',
- "People's Republic of China",
- 'Russian Federation',
+ 'Russia',
'Singapore',
'Spain',
'Switzerland',
- 'United States of America',
+ 'United States',
]);
expect(status).toBe(200);
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 640c52e748..a4cf143913 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -529,9 +529,6 @@ importers:
helmet:
specifier: ^8.1.0
version: 8.3.0
- i18n-iso-countries:
- specifier: ^7.6.0
- version: 7.14.0
ioredis:
specifier: ^5.8.2
version: 5.11.1([email protected])
@@ -7436,9 +7433,6 @@ packages:
resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==}
- resolution: {integrity: sha512-wlwEkqcsaxvPJML+rDh/2iS824jbREk6DUMUKkEaSlxdYHeS43cClJtsWglvw2RfeXGm6ohKDqsXteJ5sP5enA==}
-
resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==}
@@ -8651,10 +8645,6 @@ packages:
resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==}
engines: {node: '>=10.18'}
- resolution: {integrity: sha512-nXHJZYtNrfsi1UQbyRqm3Gou431elgLjKl//CYlnBGt5aTWdRPH1PiS2T/p/n8Q8LnqYqzQJik3Q7mkwvLokeg==}
- engines: {node: '>= 12'}
-
resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
engines: {node: '>=0.10.0'}
@@ -20614,8 +20604,6 @@ snapshots:
asap: 2.0.6
wrappy: 1.0.2
-
@@ -22341,10 +22329,6 @@ snapshots:
- dependencies:
- diacritics: 1.3.0
-
dependencies:
safer-buffer: 2.1.2
diff --git a/server/package.json b/server/package.json
index 3c23340376..f857ea6678 100644
--- a/server/package.json
+++ b/server/package.json
@@ -82,7 +82,6 @@
"geo-tz": "^8.0.0",
"handlebars": "^4.7.8",
"helmet": "^8.1.0",
- "i18n-iso-countries": "^7.6.0",
"ioredis": "^5.8.2",
"jose": "^6.0.0",
"js-yaml": "^5.0.0",
diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts
index 7c8c7b70ed..85621179b3 100644
--- a/server/src/repositories/config.repository.ts
+++ b/server/src/repositories/config.repository.ts
@@ -90,6 +90,7 @@ export interface EnvData {
admin1: string;
admin2: string;
cities500: string;
+ countryInfo: string;
naturalEarthCountriesPath: string;
};
web: {
@@ -338,6 +339,7 @@ const getEnv = (): EnvData => {
admin1: join(folders.geodata, 'admin1CodesASCII.txt'),
admin2: join(folders.geodata, 'admin2Codes.txt'),
cities500: join(folders.geodata, citiesFile),
+ countryInfo: join(folders.geodata, 'countryInfo.txt'),
naturalEarthCountriesPath: join(folders.geodata, 'ne_10m_admin_0_countries.geojson'),
},
web: {
diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts
index d7234faf05..d797c8c167 100644
--- a/server/src/repositories/map.repository.ts
+++ b/server/src/repositories/map.repository.ts
@@ -1,6 +1,5 @@
import { Injectable } from '@nestjs/common';
-import isoCountries from 'i18n-iso-countries';
-import { type Expression, type Insertable, type Kysely, type NotNull, type SqlBool, sql } from 'kysely';
+import { Expression, Insertable, Kysely, NotNull, SqlBool, sql } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { createReadStream, existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
@@ -38,8 +37,15 @@ interface MapDB extends DB {
naturalearth_countries_tmp: NaturalEarthCountriesTable;
}
+interface CountryNames {
+ byAlpha2: Map<string, string>;
+ byAlpha3: Map<string, string>;
+}
+
@Injectable()
export class MapRepository {
+ private countryNames?: Promise<CountryNames>;
+
constructor(
private configRepository: ConfigRepository,
private metadataRepository: SystemMetadataRepository,
@@ -148,6 +154,8 @@ export class MapRepository {
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
+ const countryNames = await this.loadCountryNames();
+
const response = await this.db
.selectFrom('geodata_places')
.selectAll()
@@ -166,8 +174,7 @@ export class MapRepository {
this.logger.verboseFn(() => `Raw: ${JSON.stringify(response, null, 2)}`);
const { countryCode, name: city, admin1Name } = response;
- // eslint-disable-next-line import-x/no-named-as-default-member
- const country = isoCountries.getName(countryCode, 'en') ?? null;
+ const country = countryNames.byAlpha2.get(countryCode) ?? null;
const state = admin1Name;
return { country, state, city };
@@ -195,14 +202,59 @@ export class MapRepository {
this.logger.verboseFn(() => `Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`);
const { admin_a3 } = ne_response;
- // eslint-disable-next-line import-x/no-named-as-default-member
- const country = isoCountries.getName(admin_a3, 'en') ?? null;
+ const country = countryNames.byAlpha3.get(admin_a3) ?? null;
const state = null;
const city = null;
return { country, state, city };
}
+ // init() skips the geodata import when it is up to date, so load the names on demand
+ private async loadCountryNames(): Promise<CountryNames> {
+ this.countryNames ??= this.readCountryNames();
+ try {
+ return await this.countryNames;
+ } catch (error) {
+ this.countryNames = undefined;
+ throw error;
+ }
+ }
+
+ private async readCountryNames(): Promise<CountryNames> {
+ const byAlpha2 = new Map<string, string>();
+ const byAlpha3 = new Map<string, string>();
+
+ const { resourcePaths } = this.configRepository.getEnv();
+ const filePath = resourcePaths.geodata.countryInfo;
+ if (!existsSync(filePath)) {
+ throw new Error(`Geodata file ${filePath} not found`);
+ }
+
+ const lineReader = readLine.createInterface({ input: createReadStream(filePath) });
+ for await (const line of lineReader) {
+ if (line.startsWith('#')) {
+ continue;
+ }
+ // Columns: ISO alpha-2, ISO alpha-3, ISO numeric, fips, Country, ...
+ const fields = line.split('\t', 5);
+ const alpha2 = fields[0];
+ const alpha3 = fields[1];
+ const name = fields[4]?.trim();
+ if (!name) {
+ continue;
+ }
+ if (alpha2) {
+ byAlpha2.set(alpha2, name);
+ }
+ if (alpha3) {
+ byAlpha3.set(alpha3, name);
+ }
+ }
+
+ this.logger.log(`Loaded ${byAlpha2.size} country names from ${filePath}`);
+ return { byAlpha2, byAlpha3 };
+ }
+
private async importNaturalEarthCountries() {
const { resourcePaths } = this.configRepository.getEnv();
const geoJSONData = JSON.parse(await readFile(resourcePaths.geodata.naturalEarthCountriesPath, 'utf8'));
diff --git a/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts b/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts
new file mode 100644
index 0000000000..b1e016fb6d
--- /dev/null
+++ b/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts
@@ -0,0 +1,56 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+ // (#30199) Reverse geocoding now uses the GeoNames names, so rewrite the stored ones
+ await sql`
+ WITH renames ("old", "new") AS (
+ VALUES
+ ('Åland Islands', 'Aland Islands'),
+ ('Bonaire, Sint Eustatius and Saba', 'Bonaire, Saint Eustatius and Saba'),
+ ('Brunei Darussalam', 'Brunei'),
+ ('Cape Verde', 'Cabo Verde'),
+ ('Cocos (Keeling) Islands', 'Cocos Islands'),
+ ('Cote d''Ivoire', 'Ivory Coast'),
+ ('Cote D''Ivoire', 'Ivory Coast'),
+ ('Curaçao', 'Curacao'),
+ ('Czech Republic', 'Czechia'),
+ ('Falkland Islands (Malvinas)', 'Falkland Islands'),
+ ('Holy See (Vatican City State)', 'Vatican'),
+ ('Islamic Republic of Iran', 'Iran'),
+ ('Lao People''s Democratic Republic', 'Laos'),
+ ('Micronesia, Federated States of', 'Micronesia'),
+ ('Moldova, Republic of', 'Moldova'),
+ ('Netherlands', 'The Netherlands'),
+ ('People''s Republic of China', 'China'),
+ ('Republic of The Gambia', 'Gambia'),
+ ('Russian Federation', 'Russia'),
+ ('Saint Barthélemy', 'Saint Barthelemy'),
+ ('Saint Martin (French part)', 'Saint Martin'),
+ ('Sint Maarten (Dutch part)', 'Sint Maarten'),
+ ('State of Palestine', 'Palestinian Territory'),
+ ('Syrian Arab Republic', 'Syria'),
+ ('Taiwan, Province of China', 'Taiwan'),
+ ('The Republic of North Macedonia', 'North Macedonia'),
+ ('Timor-Leste', 'Timor Leste'),
+ ('Türkiye', 'Turkey'),
+ ('United Republic of Tanzania', 'Tanzania'),
+ ('United States of America', 'United States'),
+ ('Virgin Islands, British', 'British Virgin Islands'),
+ ('Virgin Islands, U.S.', 'U.S. Virgin Islands')
+ ),
+ exif AS (
+ UPDATE "asset_exif"
+ SET "country" = renames."new"
+ FROM renames
+ WHERE "country" = renames."old"
+ )
+ UPDATE "workflow_step"
+ SET "config" = jsonb_set("config", '{region,country}', to_jsonb(renames."new"))
+ FROM renames
+ WHERE "config"->'region'->>'country' = renames."old"
+ `.execute(db);
+}
+
+export async function down(): Promise<void> {
+ // Not implemented: the old names came from a removed dependency
+}
diff --git a/server/src/schema/migrations/ORDER b/server/src/schema/migrations/ORDER
index f4bd7df467..2db4ae0ca6 100644
--- a/server/src/schema/migrations/ORDER
+++ b/server/src/schema/migrations/ORDER
@@ -95,3 +95,4 @@
1787148183729-ClusterGroups
1787148183730-DeleteMismatchedMemoryAssets
1789419229196-ConvertUserOAuthIdEmptyStringToNull
+1790587508209-RenameGeoNamesCountries
diff --git a/server/test/factories/asset-exif.factory.ts b/server/test/factories/asset-exif.factory.ts
index f6b0e7d48f..c015151569 100644
--- a/server/test/factories/asset-exif.factory.ts
+++ b/server/test/factories/asset-exif.factory.ts
@@ -19,7 +19,7 @@ export class AssetExifFactory {
bitsPerSample: null,
city: 'Austin',
colorspace: null,
- country: 'United States of America',
+ country: 'United States',
dateTimeOriginal: factory.date(),
description: '',
exifImageHeight: 420,
diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts
index 4ccf89e171..2b4aa2a192 100644
--- a/server/test/repositories/config.repository.mock.ts
+++ b/server/test/repositories/config.repository.mock.ts
@@ -70,6 +70,7 @@ export const envData: EnvData = {
admin1: '/build/geodata/admin1CodesASCII.txt',
admin2: '/build/geodata/admin2Codes.txt',
cities500: '/build/geodata/cities500.txt',
+ countryInfo: '/build/geodata/countryInfo.txt',
naturalEarthCountriesPath: 'build/ne_10m_admin_0_countries.geojson',
},
web: {