Skip to content

A continuation of HomeBox the inventory and organization system built for the Home User

feat: replace the deprecated countries API with v5 (#1622)

ydkmlt842eed76a76338
diff --git a/.github/scripts/test_update_currencies.py b/.github/scripts/test_update_currencies.py
new file mode 100644
index 00000000..40ac343b
--- /dev/null
+++ b/.github/scripts/test_update_currencies.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+import unittest
+from unittest import mock
+
+import update_currencies
+
+
+class ParseCountryPageTests(unittest.TestCase):
+ def test_parses_v5_response(self):
+ countries, meta = update_currencies.parse_country_page({
+ 'data': {
+ 'objects': [{'names': {'common': 'Canada'}, 'currencies': []}],
+ 'meta': {'more': False, 'total': 1},
+ }
+ })
+
+ self.assertEqual('Canada', countries[0]['names']['common'])
+ self.assertFalse(meta['more'])
+
+ def test_reports_api_error_message(self):
+ with self.assertRaisesRegex(ValueError, 'API key is invalid'):
+ update_currencies.parse_country_page({
+ 'errors': [{'message': 'API key is invalid'}]
+ })
+
+ def test_rejects_legacy_response(self):
+ with self.assertRaisesRegex(ValueError, 'missing the data object'):
+ update_currencies.parse_country_page({
+ 'success': False,
+ 'errors': [],
+ })
+
+ def test_requires_complete_pagination_metadata(self):
+ with self.assertRaisesRegex(ValueError, 'total count'):
+ update_currencies.parse_country_page({
+ 'data': {
+ 'objects': [],
+ 'meta': {'more': False},
+ }
+ })
+
+
+class CountriesToCurrenciesTests(unittest.TestCase):
+ def test_converts_v5_currency_list(self):
+ result = update_currencies.countries_to_currencies([
+ {
+ 'names': {'common': 'Canada'},
+ 'currencies': [{
+ 'code': 'CAD',
+ 'name': 'Canadian dollar',
+ 'symbol': '$',
+ }],
+ }
+ ], {'CAD': 2})
+
+ self.assertEqual([{
+ 'code': 'CAD',
+ 'local': 'Canada',
+ 'symbol': '$',
+ 'name': 'Canadian dollar',
+ 'decimals': 2,
+ }], result)
+
+ def test_rejects_old_currency_object_shape(self):
+ with self.assertRaisesRegex(ValueError, 'currencies must be a JSON list'):
+ update_currencies.countries_to_currencies([
+ {
+ 'names': {'common': 'Canada'},
+ 'currencies': {'CAD': {'name': 'Canadian dollar'}},
+ }
+ ], {})
+
+
+class FetchCurrenciesTests(unittest.TestCase):
+ class FakeResponse:
+ def __init__(self, payload):
+ self.payload = payload
+
+ def raise_for_status(self):
+ return None
+
+ def json(self):
+ return self.payload
+
+ class FakeSession:
+ def __init__(self, payloads):
+ self.payloads = iter(payloads)
+ self.calls = []
+
+ def get(self, url, **kwargs):
+ self.calls.append((url, kwargs))
+ return FetchCurrenciesTests.FakeResponse(next(self.payloads))
+
+ def fetch_with_pages(self, pages, iso_data=None):
+ session = self.FakeSession(pages)
+ with mock.patch.dict(update_currencies.os.environ, {
+ update_currencies.API_KEY_ENV: 'test-key',
+ }), mock.patch.object(update_currencies, 'create_session', return_value=session), \
+ mock.patch.object(
+ update_currencies,
+ 'fetch_iso_4217_data',
+ return_value=iso_data or {},
+ ):
+ return update_currencies.fetch_currencies(), session
+
+ def test_fetches_every_page_with_authentication(self):
+ pages = [
+ {
+ 'data': {
+ 'objects': [{
+ 'names': {'common': 'Canada'},
+ 'currencies': [{
+ 'code': 'CAD',
+ 'name': 'Canadian dollar',
+ 'symbol': '$',
+ }],
+ }],
+ 'meta': {'more': True, 'total': 2},
+ }
+ },
+ {
+ 'data': {
+ 'objects': [{
+ 'names': {'common': 'Japan'},
+ 'currencies': [{
+ 'code': 'JPY',
+ 'name': 'Japanese yen',
+ 'symbol': '¥',
+ }],
+ }],
+ 'meta': {'more': False, 'total': 2},
+ }
+ },
+ ]
+ result, session = self.fetch_with_pages(pages, {'CAD': 2})
+
+ self.assertEqual(['CAD', 'JPY'], [currency['code'] for currency in result])
+ self.assertEqual(2, len(session.calls))
+ self.assertEqual('Bearer test-key', session.calls[0][1]['headers']['Authorization'])
+ self.assertEqual(0, session.calls[0][1]['params']['offset'])
+ self.assertEqual(1, session.calls[1][1]['params']['offset'])
+
+ def test_missing_api_key_is_fatal_before_network_access(self):
+ with mock.patch.dict(update_currencies.os.environ, {}, clear=True), \
+ mock.patch.object(update_currencies, 'create_session') as create_session:
+ result = update_currencies.fetch_currencies()
+
+ self.assertIsNone(result)
+ create_session.assert_not_called()
+
+ def test_stops_when_more_is_true_after_reaching_total(self):
+ result, session = self.fetch_with_pages([{
+ 'data': {
+ 'objects': [{
+ 'names': {'common': 'Canada'},
+ 'currencies': [],
+ }],
+ 'meta': {'more': True, 'total': 1},
+ }
+ }])
+
+ self.assertIsNone(result)
+ self.assertEqual(1, len(session.calls))
+
+ def test_rejects_total_changing_between_pages(self):
+ country = {'names': {'common': 'Canada'}, 'currencies': []}
+ result, session = self.fetch_with_pages([
+ {
+ 'data': {
+ 'objects': [country],
+ 'meta': {'more': True, 'total': 3},
+ }
+ },
+ {
+ 'data': {
+ 'objects': [country],
+ 'meta': {'more': False, 'total': 2},
+ }
+ },
+ ])
+
+ self.assertIsNone(result)
+ self.assertEqual(2, len(session.calls))
+
+ def test_rejects_final_count_mismatch(self):
+ result, session = self.fetch_with_pages([{
+ 'data': {
+ 'objects': [{
+ 'names': {'common': 'Canada'},
+ 'currencies': [],
+ }],
+ 'meta': {'more': False, 'total': 2},
+ }
+ }])
+
+ self.assertIsNone(result)
+ self.assertEqual(1, len(session.calls))
+
+ def test_rejects_empty_page_when_more_is_true(self):
+ result, session = self.fetch_with_pages([{
+ 'data': {
+ 'objects': [],
+ 'meta': {'more': True, 'total': 1},
+ }
+ }])
+
+ self.assertIsNone(result)
+ self.assertEqual(1, len(session.calls))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/.github/scripts/update_currencies.py b/.github/scripts/update_currencies.py
index 8f4965ac..08d5fa8f 100644
--- a/.github/scripts/update_currencies.py
+++ b/.github/scripts/update_currencies.py
@@ -11,7 +11,9 @@ import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
-API_URL = 'https://restcountries.com/v3.1/all?fields=name,currencies'
+API_URL = 'https://api.restcountries.com/countries/v5'
+API_KEY_ENV = 'RESTCOUNTRIES_API_KEY'
+API_PAGE_LIMIT = 100
# Default to a pinned commit for supply-chain security
DEFAULT_ISO_4217_URL = 'https://raw.githubusercontent.com/datasets/currency-codes/refs/heads/main/data/codes-all.csv'
ISO_4217_URL = os.environ.get('ISO_4217_URL', DEFAULT_ISO_4217_URL)
@@ -36,6 +38,18 @@ def setup_logging():
)
+def create_session():
+ session = requests.Session()
+ retries = Retry(
+ total=3,
+ backoff_factor=1,
+ status_forcelist=[429, 500, 502, 503, 504],
+ allowed_methods=frozenset(['GET'])
+ )
+ session.mount('https://', HTTPAdapter(max_retries=retries))
+ return session
+
+
def get_currency_decimals(code, iso_data):
"""
Get the decimal places for a currency code.
@@ -75,14 +89,7 @@ def fetch_iso_4217_data():
logging.error("Refusing non-HTTPS ISO_4217_URL: %s", ISO_4217_URL)
return {}
- session = requests.Session()
- retries = Retry(
- total=3,
- backoff_factor=1,
- status_forcelist=[429, 500, 502, 503, 504],
- allowed_methods=frozenset(['GET'])
- )
- session.mount('https://', HTTPAdapter(max_retries=retries))
+ session = create_session()
try:
# Add Accept header for CSV content
@@ -119,52 +126,80 @@ def fetch_iso_4217_data():
return {}
-def fetch_currencies():
- # First, fetch ISO 4217 data for decimal places
- iso_data = fetch_iso_4217_data()
-
- session = requests.Session()
- retries = Retry(
- total=3,
- backoff_factor=1,
- status_forcelist=[429, 500, 502, 503, 504],
- allowed_methods=frozenset(['GET'])
- )
- session.mount('https://', HTTPAdapter(max_retries=retries))
+def parse_country_page(payload):
+ if not isinstance(payload, dict):
+ raise ValueError("response must be a JSON object")
- try:
- resp = session.get(API_URL, timeout=TIMEOUT)
- resp.raise_for_status()
- except requests.exceptions.RequestException as e:
- logging.error("API request failed: %s", e)
- return None # signal fatal
+ errors = payload.get('errors')
+ if errors:
+ messages = [
+ error.get('message', str(error)) if isinstance(error, dict) else str(error)
+ for error in errors
+ ]
+ raise ValueError("API returned errors: %s" % "; ".join(messages))
- try:
- countries = resp.json()
- except ValueError as e:
- logging.error("Failed to parse JSON response: %s", e)
- return None # signal fatal
+ data = payload.get('data')
+ if not isinstance(data, dict):
+ raise ValueError("response is missing the data object")
+
+ countries = data.get('objects')
+ meta = data.get('meta')
+ if not isinstance(countries, list):
+ raise ValueError("response data is missing the objects list")
+ if not isinstance(meta, dict):
+ raise ValueError("response data is missing pagination metadata")
+ if not isinstance(meta.get('more'), bool):
+ raise ValueError("pagination metadata is missing the more flag")
+ if isinstance(meta.get('total'), bool) or not isinstance(meta.get('total'), int):
+ raise ValueError("pagination metadata is missing the total count")
+ return countries, meta
+
+
+def countries_to_currencies(countries, iso_data):
results = []
for country in countries:
- name_field = country.get('name', {})
- if isinstance(name_field, dict):
- country_name = name_field.get('common') or "Unknown"
- else:
- country_name = name_field or "Unknown"
- for code, info in country.get('currencies', {}).items():
+ if not isinstance(country, dict):
+ raise ValueError("country entry must be a JSON object")
+
+ names = country.get('names') or {}
+ currencies = country.get('currencies') or []
+ if not isinstance(names, dict):
+ raise ValueError("country names must be a JSON object")
+ if not isinstance(currencies, list):
+ raise ValueError("country currencies must be a JSON list")
+ country_name = names.get('common') or "Unknown"
+ if not isinstance(country_name, str):
+ raise ValueError("country common name must be a string")
+
+ for info in currencies:
+ if not isinstance(info, dict):
+ raise ValueError("currency entry must be a JSON object")
+ code = info.get('code') or ''
+ if not isinstance(code, str):
+ raise ValueError("currency code must be a string")
+ if not code:
+ logging.warning("Skipping currency without a code for %s", country_name)
+ continue
+
# Get decimal places using the helper function
decimals = get_currency_decimals(code, iso_data)
# Capitalize the first letter of the currency name
- currency_name = info.get('name', '')
+ currency_name = info.get('name') or ''
+ if not isinstance(currency_name, str):
+ raise ValueError("currency name must be a string")
if currency_name:
currency_name = currency_name[0].upper() + currency_name[1:]
+ symbol = info.get('symbol') or ''
+ if not isinstance(symbol, str):
+ raise ValueError("currency symbol must be a string")
+
results.append({
'code': code,
'local': country_name,
- 'symbol': info.get('symbol', ''),
+ 'symbol': symbol,
'name': currency_name,
'decimals': decimals
})
@@ -173,6 +208,74 @@ def fetch_currencies():
return sorted(results, key=lambda x: x['local'].lower())
+def fetch_currencies():
+ api_key = os.environ.get(API_KEY_ENV, '').strip()
+ if not api_key:
+ logging.error("Missing required %s environment variable", API_KEY_ENV)
+ return None
+
+ # First, fetch ISO 4217 data for decimal places
+ iso_data = fetch_iso_4217_data()
+ session = create_session()
+ countries = []
+ offset = 0
+ expected_total = None
+
+ while True:
+ try:
+ resp = session.get(
+ API_URL,
+ timeout=TIMEOUT,
+ headers={'Authorization': 'Bearer %s' % api_key},
+ params={
+ 'response_fields': 'names.common,currencies',
+ 'limit': API_PAGE_LIMIT,
+ 'offset': offset,
+ }
+ )
+ resp.raise_for_status()
+ except requests.exceptions.RequestException as e:
+ logging.error("Countries API request failed: %s", e)
+ return None # signal fatal
+
+ try:
+ payload = resp.json()
+ page, meta = parse_country_page(payload)
+ except (ValueError, TypeError) as e:
+ logging.error("Failed to parse Countries API response: %s", e)
+ return None # signal fatal
+
+ countries.extend(page)
+ page_total = meta['total']
+ if expected_total is None:
+ expected_total = page_total
+ elif page_total != expected_total:
+ logging.error("Countries API total changed while fetching pages")
+ return None
+ if not meta.get('more'):
+ break
+ if not page:
+ logging.error("Countries API returned an empty page while more data was expected")
+ return None
+ if len(countries) >= expected_total:
+ logging.error("Countries API reported more pages than the declared total")
+ return None
+ offset += len(page)
+
+ if len(countries) != expected_total:
+ logging.error(
+ "Countries API returned %d of %d expected countries",
+ len(countries), expected_total
+ )
+ return None
+
+ try:
+ return countries_to_currencies(countries, iso_data)
+ except (ValueError, TypeError) as e:
+ logging.error("Failed to parse country data: %s", e)
+ return None
+
+
def load_existing(path: Path):
if not path.exists():
return []
@@ -203,10 +306,10 @@ def main():
logging.error("Aborting: failed to fetch or parse API data.")
sys.exit(1)
- # Empty array → log & exit zero (no file change)
+ # Empty array means an incomplete or malformed all-countries response.
if not new:
- logging.warning("API returned empty list; skipping write.")
- sys.exit(0)
+ logging.error("Aborting: Countries API returned no currencies.")
+ sys.exit(1)
# Identical to existing → nothing to do
if new == existing:
diff --git a/.github/workflows/update-currencies.yml b/.github/workflows/update-currencies.yml
index 0c56c48e..aa201b20 100644
--- a/.github/workflows/update-currencies.yml
+++ b/.github/workflows/update-currencies.yml
@@ -31,8 +31,13 @@ jobs:
python -m pip install --upgrade pip
pip install -r .github/workflows/update-currencies/requirements.txt
+ - name: Test currency update script
+ run: python -m unittest discover -s .github/scripts -p "test_*.py"
+
- name: Run currency update script
run: python .github/scripts/update_currencies.py
+ env:
+ RESTCOUNTRIES_API_KEY: ${{ secrets.RESTCOUNTRIES_API_KEY }}
- name: Check for currencies.json changes
run: |