Open-source personal finance manager. Self-hosted, privacy-first.
fix(market-price): normalize GBp/ZAc prices in batch refresh (#1169)
Canus Lector2ad96be9bfcb
diff --git a/backend/app/providers/market_price.py b/backend/app/providers/market_price.pyindex 1e1b7212..1d398404 100644--- a/backend/app/providers/market_price.py+++ b/backend/app/providers/market_price.py@@ -172,14 +172,12 @@ class YFinanceProvider(MarketPriceProvider):creation time; we just update the price and let ``current_value``recompute from ``units × last_price``.- Note on minor-unit currencies (GBp, ZAc on LSE/JSE): those are- normalized to GBP/ZAR ×0.01 at creation (via ``get_quote``), but- on refresh we only get the raw trading-currency close. The batch- preserves the same unit as the upstream quote, so internal- consistency is maintained as long as the asset's stored currency- matches the ticker's reporting unit. If Yahoo changes a listing's- reporting unit mid-stream the price will appear off by 100× — rare- enough to address reactively rather than pre-emptively.+ Note on minor-unit currencies (GBp, ZAc on LSE/JSE): ``download``+ returns the raw pence/cents close, so ``*.L`` / ``*.JO`` tickers get+ a follow-up ``fast_info.currency`` lookup and are scaled ×0.01 to+ match the GBP/ZAR normalization ``get_quote`` applies at creation.+ If that lookup fails for a ticker its price is returned as ``None``+ rather than raw, so the caller falls back to ``get_quote``."""if not symbols:return {}@@ -196,6 +194,39 @@ class YFinanceProvider(MarketPriceProvider):raise MarketPriceRateLimitedError("Yahoo Finance rate-limited the batch price download")++ # Normalize minor-unit prices (GBp → GBP, ZAc → ZAR, both ×0.01).+ # yf.download returns raw pence/cents for LSE/JSE tickers without+ # a currency tag; we fetch fast_info.currency only for the tickers+ # that actually need it (*.L and *.JO suffixes) to keep the extra+ # round-trips minimal.+ candidates = [+ s for s, p in chunk_prices.items()+ if p is not None and (s.endswith(".L") or s.endswith(".JO"))+ ]+ if candidates:+ try:+ currencies = await asyncio.to_thread(+ self._get_currencies_sync, candidates+ )+ except Exception as e: # pragma: no cover — upstream flakiness, not logic+ logger.warning(+ "minor-unit currency check failed for %s: %s", candidates, e+ )+ currencies = {}+ for sym in candidates:+ currency = currencies.get(sym)+ if not currency:+ # Unit unknown — a raw pence/cents close could be+ # 100× off, so drop it and let the caller fall back+ # to ``get_quote``, which normalizes on its own.+ chunk_prices[sym] = None+ continue+ conv = _MINOR_UNIT_CURRENCIES.get(currency)+ price = chunk_prices[sym]+ if conv is not None and price is not None:+ chunk_prices[sym] = price * Decimal(str(conv[1]))+out.update(chunk_prices)return out@@ -253,6 +284,26 @@ class YFinanceProvider(MarketPriceProvider):logger.warning("yfinance search failed for %r: %s", query, e)return []+ @staticmethod+ def _get_currencies_sync(symbols: list[str]) -> dict[str, Optional[str]]:+ """Return ``fast_info.currency`` for each symbol.++ Used by ``get_latest_prices`` to identify minor-unit tickers (GBp,+ ZAc) after a ``yf.download`` batch, which returns raw prices without+ currency metadata. Only called for the small subset of tickers whose+ exchange suffix suggests a minor-unit quote (``*.L``, ``*.JO``).+ """+ import yfinance as yf++ out: dict[str, Optional[str]] = {}+ for sym in symbols:+ try:+ currency = _fast_info_value(yf.Ticker(sym).fast_info, "currency")+ out[sym] = str(currency) if currency is not None else None+ except Exception:+ out[sym] = None+ return out+@staticmethoddef _download_prices_sync(symbols: list[str]) -> dict[str, Optional[Decimal]]:"""Run ``yf.download`` and extract last-close per symbol.diff --git a/backend/tests/test_market_price_assets.py b/backend/tests/test_market_price_assets.pyindex bc2cf82e..c3784597 100644--- a/backend/tests/test_market_price_assets.py+++ b/backend/tests/test_market_price_assets.py@@ -12,6 +12,7 @@ from __future__ import annotationsfrom datetime import datefrom decimal import Decimalfrom typing import Optional+from unittest.mock import patchimport pytestfrom sqlalchemy import select@@ -23,6 +24,7 @@ from app.models.user import Userfrom app.providers.market_price import (MarketPriceProvider,MarketPriceRateLimitedError,+ YFinanceProvider,)from app.schemas.asset import (AssetCreate,@@ -422,3 +424,137 @@ async def test_refresh_all_halts_on_rate_limit(# skipped bucket, no per-asset fallback was attempted.assert result["refreshed"] == 0assert result["skipped"] == 2+++# ---- YFinanceProvider batch refresh: minor-unit normalization ----+++async def test_yfinance_batch_normalizes_gbp_pence_to_pounds():+ # yf.download reports LSE tickers in pence; the batch path must scale+ # them to pounds the same way get_quote does at creation.+ provider = YFinanceProvider()+ with (+ patch.object(+ YFinanceProvider,+ "_download_prices_sync",+ return_value={"EQQQ.L": Decimal("38450")},+ ),+ patch.object(+ YFinanceProvider,+ "_get_currencies_sync",+ return_value={"EQQQ.L": "GBp"},+ ) as currencies,+ ):+ prices = await provider.get_latest_prices(["EQQQ.L"])++ currencies.assert_called_once_with(["EQQQ.L"])+ assert prices["EQQQ.L"] == Decimal("384.50")+++async def test_yfinance_batch_normalizes_zac_cents_to_rand():+ provider = YFinanceProvider()+ with (+ patch.object(+ YFinanceProvider,+ "_download_prices_sync",+ return_value={"NPN.JO": Decimal("612300")},+ ),+ patch.object(+ YFinanceProvider,+ "_get_currencies_sync",+ return_value={"NPN.JO": "ZAc"},+ ),+ ):+ prices = await provider.get_latest_prices(["NPN.JO"])++ assert prices["NPN.JO"] == Decimal("6123.00")+++async def test_yfinance_batch_leaves_usd_ticker_unchanged():+ # No *.L / *.JO suffix → no currency lookup at all, price passes through.+ provider = YFinanceProvider()+ with (+ patch.object(+ YFinanceProvider,+ "_download_prices_sync",+ return_value={"AAPL": Decimal("181.25")},+ ),+ patch.object(YFinanceProvider, "_get_currencies_sync") as currencies,+ ):+ prices = await provider.get_latest_prices(["AAPL"])++ currencies.assert_not_called()+ assert prices["AAPL"] == Decimal("181.25")+++async def test_yfinance_batch_keeps_decimal_precision_when_scaling():+ provider = YFinanceProvider()+ with (+ patch.object(+ YFinanceProvider,+ "_download_prices_sync",+ return_value={"EQQQ.L": Decimal("123.45678901234567")},+ ),+ patch.object(+ YFinanceProvider,+ "_get_currencies_sync",+ return_value={"EQQQ.L": "GBp"},+ ),+ ):+ prices = await provider.get_latest_prices(["EQQQ.L"])++ assert prices["EQQQ.L"] == Decimal("1.2345678901234567")+++async def test_yfinance_batch_leaves_major_unit_lse_ticker_unchanged():+ # Not every *.L listing trades in pence — some are quoted in GBP/USD.+ provider = YFinanceProvider()+ with (+ patch.object(+ YFinanceProvider,+ "_download_prices_sync",+ return_value={"CSPX.L": Decimal("612.40")},+ ),+ patch.object(+ YFinanceProvider,+ "_get_currencies_sync",+ return_value={"CSPX.L": "USD"},+ ),+ ):+ prices = await provider.get_latest_prices(["CSPX.L"])++ assert prices["CSPX.L"] == Decimal("612.40")+++async def test_yfinance_batch_drops_price_when_currency_unknown():+ # A failed currency lookup must not let a raw pence close through as+ # pounds — the symbol comes back None so the refresh falls back to+ # get_quote. Other symbols in the same chunk are unaffected.+ provider = YFinanceProvider()+ with (+ patch.object(+ YFinanceProvider,+ "_download_prices_sync",+ return_value={+ "EQQQ.L": Decimal("38450"),+ "SGLN.L": Decimal("5120"),+ "AAPL": Decimal("181.25"),+ },+ ),+ patch.object(+ YFinanceProvider,+ "_get_currencies_sync",+ return_value={"EQQQ.L": None, "SGLN.L": "GBp"},+ ),+ ):+ prices = await provider.get_latest_prices(["EQQQ.L", "SGLN.L", "AAPL"])++ assert prices["EQQQ.L"] is None+ assert prices["SGLN.L"] == Decimal("51.20")+ assert prices["AAPL"] == Decimal("181.25")