A continuation of HomeBox the inventory and organization system built for the Home User
feat: replace the deprecated countries API with v5 (#1622)
ydkmlt842eed76a76338
diff --git a/.github/scripts/test_update_currencies.py b/.github/scripts/test_update_currencies.pynew file mode 100644index 00000000..40ac343b--- /dev/null+++ b/.github/scripts/test_update_currencies.py@@ -0,0 +1,212 @@+#!/usr/bin/env python3+import unittest+from unittest import mock++import update_currencies+++class ParseCountryPageTests(unittest.TestCase):+ def test_parses_v5_response(self):+ countries, meta = update_currencies.parse_country_page({+ 'data': {+ 'objects': [{'names': {'common': 'Canada'}, 'currencies': []}],+ 'meta': {'more': False, 'total': 1},+ }+ })++ self.assertEqual('Canada', countries[0]['names']['common'])+ self.assertFalse(meta['more'])++ def test_reports_api_error_message(self):+ with self.assertRaisesRegex(ValueError, 'API key is invalid'):+ update_currencies.parse_country_page({+ 'errors': [{'message': 'API key is invalid'}]+ })++ def test_rejects_legacy_response(self):+ with self.assertRaisesRegex(ValueError, 'missing the data object'):+ update_currencies.parse_country_page({+ 'success': False,+ 'errors': [],+ })++ def test_requires_complete_pagination_metadata(self):+ with self.assertRaisesRegex(ValueError, 'total count'):+ update_currencies.parse_country_page({+ 'data': {+ 'objects': [],+ 'meta': {'more': False},+ }+ })+++class CountriesToCurrenciesTests(unittest.TestCase):+ def test_converts_v5_currency_list(self):+ result = update_currencies.countries_to_currencies([+ {+ 'names': {'common': 'Canada'},+ 'currencies': [{+ 'code': 'CAD',+ 'name': 'Canadian dollar',+ 'symbol': '$',+ }],+ }+ ], {'CAD': 2})++ self.assertEqual([{+ 'code': 'CAD',+ 'local': 'Canada',+ 'symbol': '$',+ 'name': 'Canadian dollar',+ 'decimals': 2,+ }], result)++ def test_rejects_old_currency_object_shape(self):+ with self.assertRaisesRegex(ValueError, 'currencies must be a JSON list'):+ update_currencies.countries_to_currencies([+ {+ 'names': {'common': 'Canada'},+ 'currencies': {'CAD': {'name': 'Canadian dollar'}},+ }+ ], {})+++class FetchCurrenciesTests(unittest.TestCase):+ class FakeResponse:+ def __init__(self, payload):+ self.payload = payload++ def raise_for_status(self):+ return None++ def json(self):+ return self.payload++ class FakeSession:+ def __init__(self, payloads):+ self.payloads = iter(payloads)+ self.calls = []++ def get(self, url, **kwargs):+ self.calls.append((url, kwargs))+ return FetchCurrenciesTests.FakeResponse(next(self.payloads))++ def fetch_with_pages(self, pages, iso_data=None):+ session = self.FakeSession(pages)+ with mock.patch.dict(update_currencies.os.environ, {+ update_currencies.API_KEY_ENV: 'test-key',+ }), mock.patch.object(update_currencies, 'create_session', return_value=session), \+ mock.patch.object(+ update_currencies,+ 'fetch_iso_4217_data',+ return_value=iso_data or {},+ ):+ return update_currencies.fetch_currencies(), session++ def test_fetches_every_page_with_authentication(self):+ pages = [+ {+ 'data': {+ 'objects': [{+ 'names': {'common': 'Canada'},+ 'currencies': [{+ 'code': 'CAD',+ 'name': 'Canadian dollar',+ 'symbol': '$',+ }],+ }],+ 'meta': {'more': True, 'total': 2},+ }+ },+ {+ 'data': {+ 'objects': [{+ 'names': {'common': 'Japan'},+ 'currencies': [{+ 'code': 'JPY',+ 'name': 'Japanese yen',+ 'symbol': '¥',+ }],+ }],+ 'meta': {'more': False, 'total': 2},+ }+ },+ ]+ result, session = self.fetch_with_pages(pages, {'CAD': 2})++ self.assertEqual(['CAD', 'JPY'], [currency['code'] for currency in result])+ self.assertEqual(2, len(session.calls))+ self.assertEqual('Bearer test-key', session.calls[0][1]['headers']['Authorization'])+ self.assertEqual(0, session.calls[0][1]['params']['offset'])+ self.assertEqual(1, session.calls[1][1]['params']['offset'])++ def test_missing_api_key_is_fatal_before_network_access(self):+ with mock.patch.dict(update_currencies.os.environ, {}, clear=True), \+ mock.patch.object(update_currencies, 'create_session') as create_session:+ result = update_currencies.fetch_currencies()++ self.assertIsNone(result)+ create_session.assert_not_called()++ def test_stops_when_more_is_true_after_reaching_total(self):+ result, session = self.fetch_with_pages([{+ 'data': {+ 'objects': [{+ 'names': {'common': 'Canada'},+ 'currencies': [],+ }],+ 'meta': {'more': True, 'total': 1},+ }+ }])++ self.assertIsNone(result)+ self.assertEqual(1, len(session.calls))++ def test_rejects_total_changing_between_pages(self):+ country = {'names': {'common': 'Canada'}, 'currencies': []}+ result, session = self.fetch_with_pages([+ {+ 'data': {+ 'objects': [country],+ 'meta': {'more': True, 'total': 3},+ }+ },+ {+ 'data': {+ 'objects': [country],+ 'meta': {'more': False, 'total': 2},+ }+ },+ ])++ self.assertIsNone(result)+ self.assertEqual(2, len(session.calls))++ def test_rejects_final_count_mismatch(self):+ result, session = self.fetch_with_pages([{+ 'data': {+ 'objects': [{+ 'names': {'common': 'Canada'},+ 'currencies': [],+ }],+ 'meta': {'more': False, 'total': 2},+ }+ }])++ self.assertIsNone(result)+ self.assertEqual(1, len(session.calls))++ def test_rejects_empty_page_when_more_is_true(self):+ result, session = self.fetch_with_pages([{+ 'data': {+ 'objects': [],+ 'meta': {'more': True, 'total': 1},+ }+ }])++ self.assertIsNone(result)+ self.assertEqual(1, len(session.calls))+++if __name__ == '__main__':+ unittest.main()diff --git a/.github/scripts/update_currencies.py b/.github/scripts/update_currencies.pyindex 8f4965ac..08d5fa8f 100644--- a/.github/scripts/update_currencies.py+++ b/.github/scripts/update_currencies.py@@ -11,7 +11,9 @@ import requestsfrom requests.adapters import HTTPAdapterfrom urllib3.util.retry import Retry-API_URL = 'https://restcountries.com/v3.1/all?fields=name,currencies'+API_URL = 'https://api.restcountries.com/countries/v5'+API_KEY_ENV = 'RESTCOUNTRIES_API_KEY'+API_PAGE_LIMIT = 100# Default to a pinned commit for supply-chain securityDEFAULT_ISO_4217_URL = 'https://raw.githubusercontent.com/datasets/currency-codes/refs/heads/main/data/codes-all.csv'ISO_4217_URL = os.environ.get('ISO_4217_URL', DEFAULT_ISO_4217_URL)@@ -36,6 +38,18 @@ def setup_logging():)+def create_session():+ session = requests.Session()+ retries = Retry(+ total=3,+ backoff_factor=1,+ status_forcelist=[429, 500, 502, 503, 504],+ allowed_methods=frozenset(['GET'])+ )+ session.mount('https://', HTTPAdapter(max_retries=retries))+ return session++def get_currency_decimals(code, iso_data):"""Get the decimal places for a currency code.@@ -75,14 +89,7 @@ def fetch_iso_4217_data():logging.error("Refusing non-HTTPS ISO_4217_URL: %s", ISO_4217_URL)return {}- session = requests.Session()- retries = Retry(- total=3,- backoff_factor=1,- status_forcelist=[429, 500, 502, 503, 504],- allowed_methods=frozenset(['GET'])- )- session.mount('https://', HTTPAdapter(max_retries=retries))+ session = create_session()try:# Add Accept header for CSV content@@ -119,52 +126,80 @@ def fetch_iso_4217_data():return {}-def fetch_currencies():- # First, fetch ISO 4217 data for decimal places- iso_data = fetch_iso_4217_data()-- session = requests.Session()- retries = Retry(- total=3,- backoff_factor=1,- status_forcelist=[429, 500, 502, 503, 504],- allowed_methods=frozenset(['GET'])- )- session.mount('https://', HTTPAdapter(max_retries=retries))+def parse_country_page(payload):+ if not isinstance(payload, dict):+ raise ValueError("response must be a JSON object")- try:- resp = session.get(API_URL, timeout=TIMEOUT)- resp.raise_for_status()- except requests.exceptions.RequestException as e:- logging.error("API request failed: %s", e)- return None # signal fatal+ errors = payload.get('errors')+ if errors:+ messages = [+ error.get('message', str(error)) if isinstance(error, dict) else str(error)+ for error in errors+ ]+ raise ValueError("API returned errors: %s" % "; ".join(messages))- try:- countries = resp.json()- except ValueError as e:- logging.error("Failed to parse JSON response: %s", e)- return None # signal fatal+ data = payload.get('data')+ if not isinstance(data, dict):+ raise ValueError("response is missing the data object")++ countries = data.get('objects')+ meta = data.get('meta')+ if not isinstance(countries, list):+ raise ValueError("response data is missing the objects list")+ if not isinstance(meta, dict):+ raise ValueError("response data is missing pagination metadata")+ if not isinstance(meta.get('more'), bool):+ raise ValueError("pagination metadata is missing the more flag")+ if isinstance(meta.get('total'), bool) or not isinstance(meta.get('total'), int):+ raise ValueError("pagination metadata is missing the total count")+ return countries, meta+++def countries_to_currencies(countries, iso_data):results = []for country in countries:- name_field = country.get('name', {})- if isinstance(name_field, dict):- country_name = name_field.get('common') or "Unknown"- else:- country_name = name_field or "Unknown"- for code, info in country.get('currencies', {}).items():+ if not isinstance(country, dict):+ raise ValueError("country entry must be a JSON object")++ names = country.get('names') or {}+ currencies = country.get('currencies') or []+ if not isinstance(names, dict):+ raise ValueError("country names must be a JSON object")+ if not isinstance(currencies, list):+ raise ValueError("country currencies must be a JSON list")+ country_name = names.get('common') or "Unknown"+ if not isinstance(country_name, str):+ raise ValueError("country common name must be a string")++ for info in currencies:+ if not isinstance(info, dict):+ raise ValueError("currency entry must be a JSON object")+ code = info.get('code') or ''+ if not isinstance(code, str):+ raise ValueError("currency code must be a string")+ if not code:+ logging.warning("Skipping currency without a code for %s", country_name)+ continue+# Get decimal places using the helper functiondecimals = get_currency_decimals(code, iso_data)# Capitalize the first letter of the currency name- currency_name = info.get('name', '')+ currency_name = info.get('name') or ''+ if not isinstance(currency_name, str):+ raise ValueError("currency name must be a string")if currency_name:currency_name = currency_name[0].upper() + currency_name[1:]+ symbol = info.get('symbol') or ''+ if not isinstance(symbol, str):+ raise ValueError("currency symbol must be a string")+results.append({'code': code,'local': country_name,- 'symbol': info.get('symbol', ''),+ 'symbol': symbol,'name': currency_name,'decimals': decimals})@@ -173,6 +208,74 @@ def fetch_currencies():return sorted(results, key=lambda x: x['local'].lower())+def fetch_currencies():+ api_key = os.environ.get(API_KEY_ENV, '').strip()+ if not api_key:+ logging.error("Missing required %s environment variable", API_KEY_ENV)+ return None++ # First, fetch ISO 4217 data for decimal places+ iso_data = fetch_iso_4217_data()+ session = create_session()+ countries = []+ offset = 0+ expected_total = None++ while True:+ try:+ resp = session.get(+ API_URL,+ timeout=TIMEOUT,+ headers={'Authorization': 'Bearer %s' % api_key},+ params={+ 'response_fields': 'names.common,currencies',+ 'limit': API_PAGE_LIMIT,+ 'offset': offset,+ }+ )+ resp.raise_for_status()+ except requests.exceptions.RequestException as e:+ logging.error("Countries API request failed: %s", e)+ return None # signal fatal++ try:+ payload = resp.json()+ page, meta = parse_country_page(payload)+ except (ValueError, TypeError) as e:+ logging.error("Failed to parse Countries API response: %s", e)+ return None # signal fatal++ countries.extend(page)+ page_total = meta['total']+ if expected_total is None:+ expected_total = page_total+ elif page_total != expected_total:+ logging.error("Countries API total changed while fetching pages")+ return None+ if not meta.get('more'):+ break+ if not page:+ logging.error("Countries API returned an empty page while more data was expected")+ return None+ if len(countries) >= expected_total:+ logging.error("Countries API reported more pages than the declared total")+ return None+ offset += len(page)++ if len(countries) != expected_total:+ logging.error(+ "Countries API returned %d of %d expected countries",+ len(countries), expected_total+ )+ return None++ try:+ return countries_to_currencies(countries, iso_data)+ except (ValueError, TypeError) as e:+ logging.error("Failed to parse country data: %s", e)+ return None++def load_existing(path: Path):if not path.exists():return []@@ -203,10 +306,10 @@ def main():logging.error("Aborting: failed to fetch or parse API data.")sys.exit(1)- # Empty array → log & exit zero (no file change)+ # Empty array means an incomplete or malformed all-countries response.if not new:- logging.warning("API returned empty list; skipping write.")- sys.exit(0)+ logging.error("Aborting: Countries API returned no currencies.")+ sys.exit(1)# Identical to existing → nothing to doif new == existing:diff --git a/.github/workflows/update-currencies.yml b/.github/workflows/update-currencies.ymlindex 0c56c48e..aa201b20 100644--- a/.github/workflows/update-currencies.yml+++ b/.github/workflows/update-currencies.yml@@ -31,8 +31,13 @@ jobs:python -m pip install --upgrade pippip install -r .github/workflows/update-currencies/requirements.txt+ - name: Test currency update script+ run: python -m unittest discover -s .github/scripts -p "test_*.py"+- name: Run currency update scriptrun: python .github/scripts/update_currencies.py+ env:+ RESTCOUNTRIES_API_KEY: ${{ secrets.RESTCOUNTRIES_API_KEY }}- name: Check for currencies.json changesrun: |