High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(server): use GeoNames for reverse-geocoded country names (#30199)
Rufus Uttleyaa023378477f
diff --git a/e2e/src/specs/server/api/map.e2e-spec.ts b/e2e/src/specs/server/api/map.e2e-spec.tsindex cf6a2abdb0..d70b047eaa 100644--- a/e2e/src/specs/server/api/map.e2e-spec.ts+++ b/e2e/src/specs/server/api/map.e2e-spec.ts@@ -65,7 +65,7 @@ describe('/map', () => {expect(body).toEqual([{city: 'Palisade',- country: 'United States of America',+ country: 'United States',id: expect.any(String),lat: expect.closeTo(39.115),lon: expect.closeTo(-108.400968),@@ -73,7 +73,7 @@ describe('/map', () => {},{city: 'Ralston',- country: 'United States of America',+ country: 'United States',id: expect.any(String),lat: expect.closeTo(41.2203),lon: expect.closeTo(-96.071625),@@ -113,7 +113,7 @@ describe('/map', () => {expect(body).toEqual([{city: 'Palisade',- country: 'United States of America',+ country: 'United States',id: expect.any(String),lat: expect.closeTo(39.115),lon: expect.closeTo(-108.400968),@@ -121,7 +121,7 @@ describe('/map', () => {},{city: 'Ralston',- country: 'United States of America',+ country: 'United States',id: expect.any(String),lat: expect.closeTo(41.2203),lon: expect.closeTo(-96.071625),diff --git a/e2e/src/specs/server/api/search.e2e-spec.ts b/e2e/src/specs/server/api/search.e2e-spec.tsindex 82ff06138e..e4616bd032 100644--- a/e2e/src/specs/server/api/search.e2e-spec.ts+++ b/e2e/src/specs/server/api/search.e2e-spec.ts@@ -491,6 +491,7 @@ describe('/search', () => {.get('/search/suggestions?type=country&includeNull=true').set('Authorization', `Bearer ${admin.accessToken}`);expect(body).toEqual([+ 'China','Cuba','France','Georgia',@@ -498,12 +499,11 @@ describe('/search', () => {'Ghana','Japan','Morocco',- "People's Republic of China",- 'Russian Federation',+ 'Russia','Singapore','Spain','Switzerland',- 'United States of America',+ 'United States',null,]);expect(status).toBe(200);@@ -514,6 +514,7 @@ describe('/search', () => {.get('/search/suggestions?type=country').set('Authorization', `Bearer ${admin.accessToken}`);expect(body).toEqual([+ 'China','Cuba','France','Georgia',@@ -521,12 +522,11 @@ describe('/search', () => {'Ghana','Japan','Morocco',- "People's Republic of China",- 'Russian Federation',+ 'Russia','Singapore','Spain','Switzerland',- 'United States of America',+ 'United States',]);expect(status).toBe(200);});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 640c52e748..a4cf143913 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -529,9 +529,6 @@ importers:helmet:specifier: ^8.1.0version: 8.3.0- i18n-iso-countries:- specifier: ^7.6.0- version: 7.14.0ioredis:specifier: ^5.8.2version: 5.11.1([email protected])@@ -7436,9 +7433,6 @@ packages:resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==}- resolution: {integrity: sha512-wlwEkqcsaxvPJML+rDh/2iS824jbREk6DUMUKkEaSlxdYHeS43cClJtsWglvw2RfeXGm6ohKDqsXteJ5sP5enA==}-resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==}@@ -8651,10 +8645,6 @@ packages:resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==}engines: {node: '>=10.18'}- resolution: {integrity: sha512-nXHJZYtNrfsi1UQbyRqm3Gou431elgLjKl//CYlnBGt5aTWdRPH1PiS2T/p/n8Q8LnqYqzQJik3Q7mkwvLokeg==}- engines: {node: '>= 12'}-resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}engines: {node: '>=0.10.0'}@@ -20614,8 +20604,6 @@ snapshots:asap: 2.0.6wrappy: 1.0.2- [email protected]: {}-@@ -22341,10 +22329,6 @@ snapshots:- dependencies:- diacritics: 1.3.0-dependencies:safer-buffer: 2.1.2diff --git a/server/package.json b/server/package.jsonindex 3c23340376..f857ea6678 100644--- a/server/package.json+++ b/server/package.json@@ -82,7 +82,6 @@"geo-tz": "^8.0.0","handlebars": "^4.7.8","helmet": "^8.1.0",- "i18n-iso-countries": "^7.6.0","ioredis": "^5.8.2","jose": "^6.0.0","js-yaml": "^5.0.0",diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.tsindex 7c8c7b70ed..85621179b3 100644--- a/server/src/repositories/config.repository.ts+++ b/server/src/repositories/config.repository.ts@@ -90,6 +90,7 @@ export interface EnvData {admin1: string;admin2: string;cities500: string;+ countryInfo: string;naturalEarthCountriesPath: string;};web: {@@ -338,6 +339,7 @@ const getEnv = (): EnvData => {admin1: join(folders.geodata, 'admin1CodesASCII.txt'),admin2: join(folders.geodata, 'admin2Codes.txt'),cities500: join(folders.geodata, citiesFile),+ countryInfo: join(folders.geodata, 'countryInfo.txt'),naturalEarthCountriesPath: join(folders.geodata, 'ne_10m_admin_0_countries.geojson'),},web: {diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.tsindex d7234faf05..d797c8c167 100644--- a/server/src/repositories/map.repository.ts+++ b/server/src/repositories/map.repository.ts@@ -1,6 +1,5 @@import { Injectable } from '@nestjs/common';-import isoCountries from 'i18n-iso-countries';-import { type Expression, type Insertable, type Kysely, type NotNull, type SqlBool, sql } from 'kysely';+import { Expression, Insertable, Kysely, NotNull, SqlBool, sql } from 'kysely';import { InjectKysely } from 'nestjs-kysely';import { createReadStream, existsSync } from 'node:fs';import { readFile } from 'node:fs/promises';@@ -38,8 +37,15 @@ interface MapDB extends DB {naturalearth_countries_tmp: NaturalEarthCountriesTable;}+interface CountryNames {+ byAlpha2: Map<string, string>;+ byAlpha3: Map<string, string>;+}+@Injectable()export class MapRepository {+ private countryNames?: Promise<CountryNames>;+constructor(private configRepository: ConfigRepository,private metadataRepository: SystemMetadataRepository,@@ -148,6 +154,8 @@ export class MapRepository {async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {this.logger.debug(`Request: ${point.latitude},${point.longitude}`);+ const countryNames = await this.loadCountryNames();+const response = await this.db.selectFrom('geodata_places').selectAll()@@ -166,8 +174,7 @@ export class MapRepository {this.logger.verboseFn(() => `Raw: ${JSON.stringify(response, null, 2)}`);const { countryCode, name: city, admin1Name } = response;- // eslint-disable-next-line import-x/no-named-as-default-member- const country = isoCountries.getName(countryCode, 'en') ?? null;+ const country = countryNames.byAlpha2.get(countryCode) ?? null;const state = admin1Name;return { country, state, city };@@ -195,14 +202,59 @@ export class MapRepository {this.logger.verboseFn(() => `Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`);const { admin_a3 } = ne_response;- // eslint-disable-next-line import-x/no-named-as-default-member- const country = isoCountries.getName(admin_a3, 'en') ?? null;+ const country = countryNames.byAlpha3.get(admin_a3) ?? null;const state = null;const city = null;return { country, state, city };}+ // init() skips the geodata import when it is up to date, so load the names on demand+ private async loadCountryNames(): Promise<CountryNames> {+ this.countryNames ??= this.readCountryNames();+ try {+ return await this.countryNames;+ } catch (error) {+ this.countryNames = undefined;+ throw error;+ }+ }++ private async readCountryNames(): Promise<CountryNames> {+ const byAlpha2 = new Map<string, string>();+ const byAlpha3 = new Map<string, string>();++ const { resourcePaths } = this.configRepository.getEnv();+ const filePath = resourcePaths.geodata.countryInfo;+ if (!existsSync(filePath)) {+ throw new Error(`Geodata file ${filePath} not found`);+ }++ const lineReader = readLine.createInterface({ input: createReadStream(filePath) });+ for await (const line of lineReader) {+ if (line.startsWith('#')) {+ continue;+ }+ // Columns: ISO alpha-2, ISO alpha-3, ISO numeric, fips, Country, ...+ const fields = line.split('\t', 5);+ const alpha2 = fields[0];+ const alpha3 = fields[1];+ const name = fields[4]?.trim();+ if (!name) {+ continue;+ }+ if (alpha2) {+ byAlpha2.set(alpha2, name);+ }+ if (alpha3) {+ byAlpha3.set(alpha3, name);+ }+ }++ this.logger.log(`Loaded ${byAlpha2.size} country names from ${filePath}`);+ return { byAlpha2, byAlpha3 };+ }+private async importNaturalEarthCountries() {const { resourcePaths } = this.configRepository.getEnv();const geoJSONData = JSON.parse(await readFile(resourcePaths.geodata.naturalEarthCountriesPath, 'utf8'));diff --git a/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts b/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.tsnew file mode 100644index 0000000000..b1e016fb6d--- /dev/null+++ b/server/src/schema/migrations/1790587508209-RenameGeoNamesCountries.ts@@ -0,0 +1,56 @@+import { Kysely, sql } from 'kysely';++export async function up(db: Kysely<any>): Promise<void> {+ // (#30199) Reverse geocoding now uses the GeoNames names, so rewrite the stored ones+ await sql`+ WITH renames ("old", "new") AS (+ VALUES+ ('Åland Islands', 'Aland Islands'),+ ('Bonaire, Sint Eustatius and Saba', 'Bonaire, Saint Eustatius and Saba'),+ ('Brunei Darussalam', 'Brunei'),+ ('Cape Verde', 'Cabo Verde'),+ ('Cocos (Keeling) Islands', 'Cocos Islands'),+ ('Cote d''Ivoire', 'Ivory Coast'),+ ('Cote D''Ivoire', 'Ivory Coast'),+ ('Curaçao', 'Curacao'),+ ('Czech Republic', 'Czechia'),+ ('Falkland Islands (Malvinas)', 'Falkland Islands'),+ ('Holy See (Vatican City State)', 'Vatican'),+ ('Islamic Republic of Iran', 'Iran'),+ ('Lao People''s Democratic Republic', 'Laos'),+ ('Micronesia, Federated States of', 'Micronesia'),+ ('Moldova, Republic of', 'Moldova'),+ ('Netherlands', 'The Netherlands'),+ ('People''s Republic of China', 'China'),+ ('Republic of The Gambia', 'Gambia'),+ ('Russian Federation', 'Russia'),+ ('Saint Barthélemy', 'Saint Barthelemy'),+ ('Saint Martin (French part)', 'Saint Martin'),+ ('Sint Maarten (Dutch part)', 'Sint Maarten'),+ ('State of Palestine', 'Palestinian Territory'),+ ('Syrian Arab Republic', 'Syria'),+ ('Taiwan, Province of China', 'Taiwan'),+ ('The Republic of North Macedonia', 'North Macedonia'),+ ('Timor-Leste', 'Timor Leste'),+ ('Türkiye', 'Turkey'),+ ('United Republic of Tanzania', 'Tanzania'),+ ('United States of America', 'United States'),+ ('Virgin Islands, British', 'British Virgin Islands'),+ ('Virgin Islands, U.S.', 'U.S. Virgin Islands')+ ),+ exif AS (+ UPDATE "asset_exif"+ SET "country" = renames."new"+ FROM renames+ WHERE "country" = renames."old"+ )+ UPDATE "workflow_step"+ SET "config" = jsonb_set("config", '{region,country}', to_jsonb(renames."new"))+ FROM renames+ WHERE "config"->'region'->>'country' = renames."old"+ `.execute(db);+}++export async function down(): Promise<void> {+ // Not implemented: the old names came from a removed dependency+}diff --git a/server/src/schema/migrations/ORDER b/server/src/schema/migrations/ORDERindex f4bd7df467..2db4ae0ca6 100644--- a/server/src/schema/migrations/ORDER+++ b/server/src/schema/migrations/ORDER@@ -95,3 +95,4 @@1787148183729-ClusterGroups1787148183730-DeleteMismatchedMemoryAssets1789419229196-ConvertUserOAuthIdEmptyStringToNull+1790587508209-RenameGeoNamesCountriesdiff --git a/server/test/factories/asset-exif.factory.ts b/server/test/factories/asset-exif.factory.tsindex f6b0e7d48f..c015151569 100644--- a/server/test/factories/asset-exif.factory.ts+++ b/server/test/factories/asset-exif.factory.ts@@ -19,7 +19,7 @@ export class AssetExifFactory {bitsPerSample: null,city: 'Austin',colorspace: null,- country: 'United States of America',+ country: 'United States',dateTimeOriginal: factory.date(),description: '',exifImageHeight: 420,diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.tsindex 4ccf89e171..2b4aa2a192 100644--- a/server/test/repositories/config.repository.mock.ts+++ b/server/test/repositories/config.repository.mock.ts@@ -70,6 +70,7 @@ export const envData: EnvData = {admin1: '/build/geodata/admin1CodesASCII.txt',admin2: '/build/geodata/admin2Codes.txt',cities500: '/build/geodata/cities500.txt',+ countryInfo: '/build/geodata/countryInfo.txt',naturalEarthCountriesPath: 'build/ne_10m_admin_0_countries.geojson',},web: {