"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Deterministic, read-only aggregation across configured trading profiles."""
from __future__ import annotations
import csv
import io
import json
import re
import socket
import subprocess
import sys
import urllib.request
import uuid
from datetime import date, datetime, timezone
from decimal import Decimal, InvalidOperation
from typing import Any, Callable
from src.portfolio.config import (
PortfolioSettingsStore,
PortfolioSource,
source_catalog,
)
from src.portfolio.compatibility import (
adapt_and_validate_payloads,
ensure_supported_currencies,
profile_compatibility,
)
from src.portfolio.fx import FxQuote, Rates, build_rates, from_usd
from src.portfolio.normalization import (
STABLECOINS,
account_balance_details,
account_cash_usd,
account_total_usd,
auth_metadata,
normalize_position,
value_position,
)
from src.portfolio.store import PortfolioStore
from src.tools.redaction import redact_text
from src.trading.connections import requires_account_selection
from src.trading.profiles import profile_by_id
from src.trading.types import TradingProfile
# ``portfolio_risk_xray`` caps a basket at 50 symbols, and its loaders route on
# the market suffix: ``AAPL`` alone is read as an A-share code, ``AAPL.US`` is
# not (see ``src.market_data._SOURCE_PATTERNS``).
_RISK_XRAY_MAX_SYMBOLS = 50
# Version 3 makes all valuation use an explicit FX rates map. Existing v2
# snapshots/history are intentionally hidden after upgrade, including
# USD/HKD/CNY snapshots whose numeric values would otherwise remain valid.
# Version 4 rebuilds persisted holdings after market-prefix normalization;
# old snapshots may have split one instrument into separate broker holdings.
PORTFOLIO_VALUATION_VERSION = 4
_LOADER_MARKET_SUFFIXES = frozenset({"US", "HK", "SZ", "SH", "BJ", "KS", "KQ", "NS", "BO", "TO", "V"})
_NON_EQUITY_ASSET_TYPES = frozenset({"crypto", "stablecoin", "cash"})
# Futu reports codes market-first (``HK.00700``, ``US.AAPL``); the rest of the
# portfolio layer and the loaders use the suffix form (``00700.HK``).
_PREFIX_MARKETS = ("US", "HK", "SH", "SZ", "BJ")
def _suffix_form(symbol: str) -> str:
"""Rewrite a market-prefixed code (``HK.00700``) to suffix form (``00700.HK``).
Args:
symbol: Upper-cased connector symbol.
Returns:
The suffix-form symbol, or ``symbol`` unchanged when it carries no
market prefix.
"""
prefix, dot, code = symbol.partition(".")
if dot and prefix in _PREFIX_MARKETS and code:
return f"{code}.{prefix}"
return symbol
_AUTH_REQUIRED_MARKERS = (
"not_authorized",
"not authorized",
"authorization required",
"oauth authorization required",
"invalid_grant",
"token expired",
"oauth token missing",
"connector authorize",
)
_REDACT_URL_QUERY = re.compile(r"(?i)(https?://[^\s?#]+)\?[^\s#]+")
_REDACT_SECRET_PARAMETER = re.compile(
r"""(?i)(["']?\b(?:signature|api[_-]?key|secret)["']?\s*[:=]\s*)"""
r"""(?:\[redacted\]|"[^"]*"|'[^']*'|[^&\s,}\]]+)"""
)
def _portfolio_error_details(
error: BaseException | str,
*,
broker: str | None = None,
default_code: str = "ConnectorReadError",
) -> tuple[str, str]:
"""Return a user-safe connector error without signed request parameters."""
message = str(error)
if (
str(broker or "").lower() == "binance"
and re.search(r"\b451\b", message)
and any(marker in message.lower() for marker in ("restricted location", "eligibility"))
):
return (
"binance_restricted_location",
"Binance denied the read with HTTP 451 because the current network location is restricted. "
"Confirm Binance is available for your region and that your account uses its supported API endpoint.",
)
message = redact_text(message)
message = _REDACT_URL_QUERY.sub(r"\1?<redacted-query>", message)
message = _REDACT_SECRET_PARAMETER.sub(r"\1<redacted>", message)
return default_code, message[:300]
def _sanitize_snapshot_errors(snapshot: dict[str, Any]) -> dict[str, Any]:
"""Clean error strings from stored snapshots before returning them to APIs."""
for account in snapshot.get("accounts", []):
if account.get("status") != "error":
continue
error_code, error = _portfolio_error_details(
account.get("error") or "",
broker=account.get("broker"),
default_code=str(account.get("error_code") or "ConnectorReadError"),
)
account["error_code"] = error_code
account["error"] = error
return snapshot
def _authorization_required(payload: dict[str, Any]) -> bool:
"""Return whether a connector failure explicitly asks for OAuth renewal."""
status = str(payload.get("status") or "").strip().lower()
if status in {"not_authorized", "unauthorized"}:
return True
message = " ".join(
str(payload.get(key) or "")
for key in ("error", "error_code", "error_type")
).lower()
return any(marker in message for marker in _AUTH_REQUIRED_MARKERS)
def _decimal(value: Any, default: Decimal = Decimal("0")) -> Decimal:
try:
if value is None or value == "":
return default
result = Decimal(str(value))
return result if result.is_finite() else default
except (InvalidOperation, ValueError, TypeError):
return default
def _number(value: Decimal) -> float:
return float(value.quantize(Decimal("0.00000001")))
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _ensure_callback_port_available(port: int) -> None:
"""Fail cleanly before an embedded OAuth callback server tries to bind."""
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", port))
except OSError as exc:
raise RuntimeError(
f"OAuth callback port {port} is already in use; close the old authorization flow and retry"
) from exc
def _contains_system_exit(error: BaseException) -> bool:
"""Return whether an exception or nested exception group contains SystemExit."""
if isinstance(error, SystemExit):
return True
nested = getattr(error, "exceptions", ())
return any(_contains_system_exit(item) for item in nested if isinstance(item, BaseException))
def _quote_price(payload: dict[str, Any]) -> Decimal | None:
quote = payload.get("quote") or {}
for key in ("last", "close"):
price = _decimal(quote.get(key))
if price > 0:
return price
bid, ask = _decimal(quote.get("bid")), _decimal(quote.get("ask"))
if bid > 0 and ask > 0:
return (bid + ask) / 2
return bid if bid > 0 else ask if ask > 0 else None
class PortfolioService:
"""Aggregate every enabled read-only source into one immutable snapshot.
The service reads; it never places or cancels an order. A source that
cannot be read contributes nothing at all — see :meth:`refresh`.
"""
def __init__(
self,
store: PortfolioStore | None = None,
*,
settings_store: PortfolioSettingsStore | None = None,
get_account: Callable[..., dict[str, Any]] | None = None,
get_positions: Callable[..., dict[str, Any]] | None = None,
get_quote: Callable[..., dict[str, Any]] | None = None,
get_longbridge_quotes: Callable[..., dict[str, Any]] | None = None,
fx_fetcher: Callable[[], FxQuote | tuple[Decimal, Decimal, str]] | None = None,
progress_callback: Callable[[str, str, str | None], None] | None = None,
) -> None:
"""Wire the service to its stores and connector read functions.
Args:
store: Snapshot/FX database. Defaults to the runtime-root store.
settings_store: Editable portfolio settings. Defaults to the
runtime-root store.
get_account: Account reader. Defaults to
``src.trading.service.get_account``.
get_positions: Positions reader. Defaults to
``src.trading.service.get_positions``.
get_quote: Single-symbol quote reader. Defaults to
``src.trading.service.get_quote``.
get_longbridge_quotes: Optional batch quote reader used only for
Longbridge positions the connector reports without a price.
fx_fetcher: Returns a dated FxQuote (or the legacy CNY/HKD tuple
for injected readers). Defaults to the reference-rate HTTPS fetch.
progress_callback: Called as ``(source_id, status, error)`` while a
refresh walks its sources.
"""
self._use_connection_scoped_reads = get_account is None and get_positions is None
self._use_connection_scoped_quotes = get_quote is None
self._use_longbridge_worker = self._use_connection_scoped_reads
if get_account is None or get_positions is None or get_quote is None:
from src.trading import service as trading
get_account = get_account or trading.get_account
get_positions = get_positions or trading.get_positions
get_quote = get_quote or trading.get_quote
self.store = store or PortfolioStore()
self._get_account = get_account
self._get_positions = get_positions
self._get_quote = get_quote
self._get_longbridge_quotes = get_longbridge_quotes
self._fx_fetcher = fx_fetcher or self._fetch_fx
self._progress_callback = progress_callback
self.settings_store = settings_store or PortfolioSettingsStore()
def settings(self) -> dict[str, Any]:
"""Return credential-free editable settings.
Returns:
The persisted display currency and source selection.
"""
return self.settings_store.load().to_dict()
def save_settings(self, payload: dict[str, Any]) -> dict[str, Any]:
"""Validate and persist editable settings.
Args:
payload: Untrusted settings from the Web UI or API.
Returns:
The validated settings that were written.
Raises:
ValueError: If the payload fails validation; nothing is written.
"""
return self.settings_store.save(payload).to_dict()
def sources(self) -> list[dict[str, Any]]:
"""Return local read-only connections eligible for this portfolio.
Returns:
One catalog row per registered connection, flagged as selected or
not, and free of secret values.
"""
return source_catalog(
self.settings_store.load(),
self.settings_store.connection_store,
self.settings_store.import_store,
)
def _connection_profile(self, source: PortfolioSource):
"""Resolve one configured source to its connection and profile.
Args:
source: The configured portfolio source.
Returns:
A ``(connection, profile)`` pair.
"""
if source.source_kind != "connection":
raise ValueError("file imports have no broker connection")
connection = self.settings_store.connection_store.get(source.connection_id)
return connection, profile_by_id(connection.profile_id)
def refresh(self) -> dict[str, Any]:
"""Read every enabled source once and persist an immutable snapshot.
A source that fails contributes nothing: no positions, no combined
holdings, and no value in the totals. Its account row carries the
error and the timestamp of its last healthy read, and the snapshot is
marked incomplete. A partial read is an error to be surfaced, never a
quietly shorter portfolio.
Returns:
The snapshot envelope that was stored.
Raises:
RuntimeError: If no source is enabled, or if FX rates can neither
be fetched nor loaded from cache.
"""
settings = self.settings_store.load()
sources = [source for source in settings.sources if source.enabled]
if not sources:
raise RuntimeError("Add and enable at least one read-only account on the Portfolio page before refreshing")
quote, fx_stale = self._rates()
rates = quote.rates
usd_cny, usd_hkd, fx_at = rates["CNY"], rates["HKD"], quote.fetched_at
display_currency = settings.display_currency
if display_currency not in rates:
raise RuntimeError(f"FX rate for {display_currency} is unavailable; refresh when the reference feed recovers")
refreshed_at = _now()
results: dict[str, dict[str, Any]] = {}
# The Longbridge Python SDK wraps a native runtime whose contexts are
# not safe to create inside this refresh thread pool on macOS. Keep the
# manual refresh deterministic and sequential; a failed broker remains
# isolated and never prevents the other accounts from refreshing.
for source in sources:
if self._progress_callback is not None:
self._progress_callback(source.id, "refreshing", None)
try:
results[source.id] = self._collect_source(source, rates)
if self._progress_callback is not None:
self._progress_callback(source.id, "ok", None)
except Exception as exc: # read failures are isolated per connector
failed_profile = None
failure_broker = None
try:
_, failed_profile = self._connection_profile(source)
failure_broker = failed_profile.connector
auth_required = failed_profile.transport == "remote_mcp" and _authorization_required(
{"error": str(exc)}
)
except Exception:
auth_required = False
error_code, error = _portfolio_error_details(
exc,
broker=failure_broker,
default_code=type(exc).__name__,
)
results[source.id] = {
"status": "error",
"error_code": error_code,
"error": error,
"failure_kind": "authorization" if auth_required else "transient",
"reconnect_required": auth_required,
}
if self._progress_callback is not None:
self._progress_callback(source.id, "error", error[:160])
positions: list[dict[str, Any]] = []
accounts: list[dict[str, Any]] = []
for source in sources:
result = results[source.id]
if source.source_kind == "extraetf":
profile = None
profile_id = None
broker = "extraetf"
else:
connection, profile = self._connection_profile(source)
profile_id = connection.profile_id
broker = profile.connector
if result["status"] != "ok":
accounts.append(self._failed_account(source, profile_id, profile, result))
continue
try:
broker_positions = [
value_position(row, rates=rates) for row in result["positions"]
]
priced_total = sum(
(_decimal(row.get("market_value_usd")) for row in broker_positions),
Decimal("0"),
)
account_total = account_total_usd(
broker,
result["account"],
rates,
priced_total,
)
if broker == "binance":
account_total = priced_total
cash_total = min(
account_total,
account_cash_usd(broker, result["account"], rates),
)
if not source.include_cash:
account_total = max(Decimal("0"), account_total - cash_total)
cash_total = Decimal("0")
unpriced_or_other = max(
Decimal("0"), account_total - priced_total - cash_total
)
priced_count = sum(
1 for row in broker_positions if row.get("priced")
)
balance_details = account_balance_details(broker, result["account"])
except Exception as exc:
error_code, error = _portfolio_error_details(
exc,
broker=broker,
default_code=type(exc).__name__,
)
failure = {
"status": "error",
"error_code": error_code,
"error": error,
"failure_kind": "transient",
"reconnect_required": False,
}
accounts.append(
self._failed_account(
source, profile_id, profile, failure
)
)
if self._progress_callback is not None:
self._progress_callback(source.id, "error", error[:160])
continue
accounts.append(
{
"source_id": source.id,
"profile_id": profile_id,
"label": source.label,
"broker": broker,
"source_kind": source.source_kind,
"import_metadata": result.get("import_metadata"),
"status": "ok",
"last_success_at": refreshed_at,
"total_usd": _number(account_total),
"total_cny": _number(from_usd(account_total, "CNY", rates)),
"total_display": _number(
from_usd(account_total, display_currency, rates)
),
"priced_value_usd": _number(priced_total),
"cash_usd": _number(cash_total),
"balance_details": balance_details,
"unpriced_or_other_usd": _number(unpriced_or_other),
"position_count": len(broker_positions),
"priced_position_count": priced_count,
"unpriced_position_count": len(broker_positions) - priced_count,
"auth": auth_metadata(profile) if profile is not None else None,
"portfolio_compatibility": profile_compatibility(profile) if profile is not None else None,
}
)
positions.extend(broker_positions)
total_usd = sum((_decimal(row.get("total_usd")) for row in accounts), Decimal("0"))
# Binance derives its account total from priced positions. Other
# connectors report an independent broker account total, so an
# unpriced position there does not make the account total incomplete.
complete = all(
row["status"] == "ok"
and not (
row.get("broker") == "binance"
and int(row.get("unpriced_position_count") or 0) > 0
)
for row in accounts
)
priced_usd = sum((_decimal(row.get("priced_value_usd")) for row in accounts), Decimal("0"))
cash_usd = sum((_decimal(row.get("cash_usd")) for row in accounts), Decimal("0"))
unpriced_usd = sum(
(_decimal(row.get("unpriced_or_other_usd")) for row in accounts),
Decimal("0"),
)
identified_usd = priced_usd + cash_usd
payload = {
"snapshot_id": uuid.uuid4().hex,
"valuation_version": PORTFOLIO_VALUATION_VERSION,
"created_at": refreshed_at,
"complete": complete,
"display_currency": display_currency,
"totals": {
"usd": _number(total_usd),
"cny": _number(from_usd(total_usd, "CNY", rates)),
"display": _number(from_usd(total_usd, display_currency, rates)),
},
"valuation": {
"priced_usd": _number(priced_usd),
"cash_usd": _number(cash_usd),
"unpriced_or_other_usd": _number(unpriced_usd),
"identified_coverage": (_number(identified_usd / total_usd) if total_usd > 0 else 0.0),
},
"fx": {
"usd_cny": _number(usd_cny),
"usd_hkd": _number(usd_hkd),
"rates": {
code: _number(rate) for code, rate in sorted(rates.items())
},
"fetched_at": fx_at,
"as_of": quote.as_of,
"source": quote.source,
"stale": fx_stale,
},
"accounts": accounts,
"positions": sorted(
positions,
key=lambda row: _decimal(row.get("market_value_usd")),
reverse=True,
),
"combined_holdings": self._combine_holdings(positions),
"warnings": self._warnings(accounts, positions, fx_stale),
}
self.store.save_snapshot(payload)
return payload
def _failed_account(
self,
source: PortfolioSource,
profile_id: str | None,
profile: TradingProfile | None,
result: dict[str, Any],
) -> dict[str, Any]:
"""Build the account row for a source that could not be read.
The row is deliberately valueless: totals are ``None`` and no position
is contributed, so a failed source can never be mistaken for an account
that is genuinely worth less. ``last_success_at`` still reports when the
source was last healthy, when such a snapshot exists.
Args:
source: The configured source that failed.
profile_id: The connection's connector profile id.
profile: The resolved connector profile.
result: The captured failure carrying ``error`` and ``error_code``.
Returns:
An account row with status ``error`` and no monetary value.
"""
cached = self.store.latest_successful_source(source.id)
return {
"source_id": source.id,
"profile_id": profile_id,
"label": source.label,
"broker": profile.connector if profile is not None else "extraetf",
"source_kind": source.source_kind,
"status": "error",
"error": result.get("error"),
"error_code": result.get("error_code"),
"failure_kind": result.get("failure_kind", "transient"),
"reconnect_required": bool(result.get("reconnect_required")),
"last_success_at": cached["created_at"] if cached is not None else None,
"total_usd": None,
"total_cny": None,
"total_display": None,
"position_count": 0,
"auth": auth_metadata(profile) if profile is not None else None,
"portfolio_compatibility": profile_compatibility(profile) if profile is not None else None,
}
def latest(self) -> dict[str, Any] | None:
"""Return the newest snapshot, but only if it covers today's selection.
Returns:
The snapshot envelope, or ``None`` when no snapshot exists or the
stored one was taken over a different set of enabled sources.
"""
snapshot = self.store.latest()
if snapshot is None:
return None
if snapshot.get("valuation_version") != PORTFOLIO_VALUATION_VERSION:
return None
settings = self.settings_store.load()
enabled_sources = [source for source in settings.sources if source.enabled]
enabled = {source.id for source in enabled_sources}
observed = {str(account.get("source_id") or account.get("broker")) for account in snapshot.get("accounts", [])}
if not enabled or observed != enabled:
return None
for source in enabled_sources:
if source.source_kind == "extraetf" and any(row.get("source_id") == source.id and row.get("status") == "ok" for row in snapshot["accounts"]):
try:
self.settings_store.import_store.get(source.id, require_fresh=True)
except ValueError:
return None
compatibility_by_source = {}
for source in enabled_sources:
try:
_, profile = self._connection_profile(source)
except ValueError:
continue
compatibility_by_source[source.id] = profile_compatibility(profile)
snapshot["accounts"] = [
{
**account,
"portfolio_compatibility": account.get("portfolio_compatibility")
or compatibility_by_source.get(str(account.get("source_id") or account.get("broker"))),
}
for account in snapshot.get("accounts", [])
]
return _sanitize_snapshot_errors(snapshot)
def snapshot_by_id(self, snapshot_id: str) -> dict[str, Any] | None:
"""Return one stored snapshot exactly as persisted, never recomputed.
Args:
snapshot_id: Exact persisted snapshot identifier.
Returns:
The stored envelope, or ``None`` when the id is unknown, was valued
under another contract version, or includes a source that is no
longer enabled (a removed or revoked connector's data stays hidden).
"""
snapshot = self.store.get(snapshot_id)
if snapshot is None or snapshot.get("valuation_version") != PORTFOLIO_VALUATION_VERSION:
return None
enabled = {source.id for source in self.settings_store.load().sources if source.enabled}
observed = {str(account.get("source_id") or account.get("broker")) for account in snapshot.get("accounts", [])}
return _sanitize_snapshot_errors(snapshot) if observed <= enabled else None
def reconnect_source(self, source_id: str) -> dict[str, Any]:
"""Run a source's interactive OAuth flow on explicit user request.
This is the only interactive path in the portfolio feature; an ordinary
refresh always reads non-interactively so it can never open a browser.
Args:
source_id: The configured source to re-authorize.
Returns:
The authorization result and the read tools the server exposes.
Raises:
RuntimeError: If the source is unknown, does not use OAuth, or its
MCP server is not configured.
"""
from src.config.accessor import get_env_config
from src.tools.mcp import MCPServerAdapter
source, profile, server = self.reconnect_target(source_id)
authorize_timeout = float(get_env_config().agent_tuning.vibe_live_authorize_timeout_s or 300)
if hasattr(server, "model_copy"):
updates = {}
if float(server.init_timeout or 0) < authorize_timeout:
updates["init_timeout"] = authorize_timeout
if float(server.tool_timeout or 0) < authorize_timeout:
updates["tool_timeout"] = authorize_timeout
if updates:
server = server.model_copy(update=updates)
callback_port = server.auth.callback_port if server.auth is not None else None
if callback_port is not None:
_ensure_callback_port_available(callback_port)
try:
tools = MCPServerAdapter(
profile.connector,
server,
max_list_tools_attempts=1,
interactive_oauth=True,
).discover_tools()
except BaseException as exc:
if not _contains_system_exit(exc):
raise
raise RuntimeError("OAuth callback startup failed; authorization stopped safely") from exc
return {
"status": "ok",
"authorized": True,
"source_id": source.id,
"enabled_read_tools": [item.remote_name for item in tools],
}
def reconnect_target(self, source_id: str):
"""Validate and return the local-only configuration for an OAuth source."""
from src.config.loader import load_agent_config
source = next(
(item for item in self.settings_store.load().sources if item.id == source_id),
None,
)
if source is None:
raise RuntimeError(f"unknown portfolio source: {source_id}")
_, profile = self._connection_profile(source)
if profile.transport != "remote_mcp":
raise RuntimeError("this source does not use OAuth; no reconnect is needed")
server = (load_agent_config().mcp_servers or {}).get(profile.connector)
if server is None:
raise RuntimeError(f"the {profile.connector} MCP connection is not configured")
if server.auth is None:
raise RuntimeError(f"the {profile.connector} MCP connection does not configure OAuth")
return source, profile, server
def history(self, limit: int = 180) -> list[dict[str, Any]]:
"""Return the value series built only from complete snapshots.
Incomplete snapshots are excluded because a source that failed
contributes nothing to the total; charting them beside complete ones
would draw a drop in portfolio value that never happened.
Args:
limit: Maximum number of snapshots to return, oldest first.
Returns:
One row per complete snapshot with its id, timestamp and totals.
"""
return self.store.history(
limit,
complete_only=True,
valuation_version=PORTFOLIO_VALUATION_VERSION,
)
def export_csv(self) -> str:
"""Render the latest snapshot's positions as CSV.
Returns:
The CSV text, or an empty string when no usable snapshot exists.
"""
snapshot = self.latest()
if snapshot is None:
return ""
fields = [
"source_id",
"source_label",
"profile_id",
"broker",
"symbol",
"name",
"asset_type",
"market",
"currency",
"quantity",
"cost_price",
"market_price",
"market_value_usd",
"market_value_cny",
"unrealized_pnl_usd",
"priced",
"updated_at",
"portfolio_id", "portfolio_name", "instrument_id_kind", "source_mtime", "as_of_source",
]
output = io.StringIO()
writer = csv.DictWriter(output, fieldnames=fields, extrasaction="ignore")
writer.writeheader()
writer.writerows(snapshot["positions"])
return output.getvalue()
def analysis_context(self, snapshot_id: str | None = None) -> dict[str, Any] | None:
"""Return a credential/account-id-free snapshot suitable for an LLM.
With ``snapshot_id`` the read is pinned to that stored snapshot (no
fallback to the latest one when it is unavailable); without it the
latest semantics are unchanged.
The context also carries ``risk_xray_args`` — the ``symbols`` and
``weights`` arguments for the existing ``portfolio_risk_xray`` tool.
The portfolio supplies that tool's arguments; it does not reimplement
or alter the analysis.
Returns:
The sanitized context, or ``None`` when no usable snapshot exists.
"""
snapshot = self.latest() if snapshot_id is None else self.snapshot_by_id(snapshot_id)
if snapshot is None:
return None
total = _decimal(snapshot["totals"]["usd"])
holdings = []
for row in snapshot.get("combined_holdings", []):
market_value = _decimal(row.get("market_value_usd"))
holdings.append(
{
"canonical_id": row["canonical_id"],
"symbol": row["symbol"],
"asset_type": row["asset_type"],
"markets": row["markets"],
"brokers": row["brokers"],
"market_value_usd": row["market_value_usd"],
"weight": _number(market_value / total) if total > 0 else 0.0,
"unrealized_pnl_usd": row["unrealized_pnl_usd"],
}
)
return {
"snapshot_id": snapshot.get("snapshot_id"),
"as_of": snapshot["created_at"],
"read_identity": {
"mode": "latest" if snapshot_id is None else "pinned",
"snapshot_id": snapshot.get("snapshot_id"),
"as_of": snapshot["created_at"],
"valuation_version": snapshot.get("valuation_version"),
},
"complete": snapshot["complete"],
"totals": snapshot["totals"],
"account_allocation": [
{
"broker_alias": f"account_{index + 1}",
"asset_provider_type": row["broker"],
"status": row["status"],
"total_usd": row.get("total_usd"),
"balance_details": row.get("balance_details", []),
"source_kind": row.get("source_kind", "connection"),
"file_evidence": {
key: row["import_metadata"].get(key)
for key in ("source_mtime", "as_of_source", "expires_at", "valuation_basis", "cash_included")
} if row.get("import_metadata") else None,
}
for index, row in enumerate(snapshot["accounts"])
],
"holdings": holdings,
"risk_xray_args": self._risk_xray_args(snapshot.get("positions", [])),
"warnings": snapshot["warnings"],
"privacy": "No account numbers, credentials, order IDs, names, or local paths included.",
}
def rebalance(self, snapshot_id: str, target_weights: dict[str, Any]) -> dict[str, Any]:
"""Return value-only allocation changes against one explicitly pinned snapshot."""
from src.portfolio.rebalance import propose_rebalance
snapshot = self.snapshot_by_id(snapshot_id)
if snapshot is None:
raise ValueError("portfolio snapshot unavailable")
return propose_rebalance(snapshot, target_weights)
@staticmethod
def _risk_xray_symbol(position: dict[str, Any]) -> str | None:
"""Map one held position onto a symbol the market-data loaders accept.
The loader chain routes on the market suffix, so the connector-reported
ticker is used as-is when it already carries one (``700.HK``,
``600519.SH``) and is qualified from the position's currency/market
otherwise (IBKR reports a bare ``AAPL``, which would be read as an
A-share code). A symbol whose market cannot be established is not
guessed at.
Args:
position: A valued position row from a stored snapshot.
Returns:
A loader-routable symbol, or ``None`` when the market is unknown.
"""
if position.get("instrument_id_kind") or position.get("source_kind") == "extraetf":
return None # An ISIN or exported pair is not an observed listing/venue.
symbol = _suffix_form(str(position.get("symbol") or "").strip().upper())
if not symbol:
return None
head, _, suffix = symbol.rpartition(".")
if head and suffix in _LOADER_MARKET_SUFFIXES:
return symbol
currency = str(position.get("price_currency") or position.get("currency") or "").upper()
market = str(position.get("market") or "").upper()
if currency == "HKD" or market == "HK":
return f"{symbol}.HK" if symbol.isdigit() else None
if currency == "USD" and symbol.isalpha():
return f"{symbol}.US"
return None
@classmethod
def _risk_xray_args(cls, positions: list[dict[str, Any]]) -> dict[str, Any]:
"""Build the ``portfolio_risk_xray`` arguments for the held equities.
Crypto, stablecoin and cash rows are dropped because the risk x-ray
prices a long-only equity basket through the daily-bar loaders; so are
unpriced rows and non-positive values, which carry no weight. The
remaining positions are merged per symbol across sources and their
weights renormalized to sum to 1.
Args:
positions: The valued positions of a stored snapshot.
Returns:
``{"symbols": [...], "weights": {symbol: weight}}``, empty when the
snapshot holds no priced equity position.
"""
values: dict[str, Decimal] = {}
for position in positions:
if str(position.get("asset_type") or "").lower() in _NON_EQUITY_ASSET_TYPES:
continue
if not position.get("priced"):
continue
value = _decimal(position.get("market_value_usd"))
if value <= 0:
continue
symbol = cls._risk_xray_symbol(position)
if symbol is None:
continue
values[symbol] = values.get(symbol, Decimal("0")) + value
ranked = sorted(values.items(), key=lambda item: (-item[1], item[0]))
ranked = ranked[:_RISK_XRAY_MAX_SYMBOLS]
total = sum((value for _, value in ranked), Decimal("0"))
if total <= 0:
return {"symbols": [], "weights": {}}
return {
"symbols": [symbol for symbol, _ in ranked],
"weights": {symbol: _number(value / total) for symbol, value in ranked},
}
@staticmethod
def _combine_holdings(positions: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Merge the same instrument held at several sources into one row.
Args:
positions: Valued positions from every successfully read source.
Returns:
Holdings sorted by USD value, each flagged when it is held at more
than one broker.
"""
grouped: dict[str, dict[str, Any]] = {}
for row in positions:
symbol = _suffix_form(str(row.get("symbol") or "").strip().upper())
currency = str(row.get("currency") or "").upper()
market = str(row.get("market") or "").upper()
if symbol.endswith(".US"):
canonical = symbol[:-3]
elif symbol.endswith(".HK"):
canonical = symbol[:-3].lstrip("0") or "0"
else:
canonical = symbol
region = (
"HK"
if currency == "HKD" or market == "HK" or symbol.endswith(".HK")
else ("CRYPTO" if row.get("asset_type") in {"crypto", "stablecoin"} else "US")
)
key = (f"{row['instrument_id_kind'].upper()}:{row['instrument_id']}"
if row.get("instrument_id_kind") else f"{region}:{canonical}")
item = grouped.setdefault(
key,
{
"canonical_id": key,
"symbol": canonical,
"asset_type": row.get("asset_type"),
"brokers": [],
"sources": [],
"markets": [],
"market_value_usd": Decimal("0"),
"market_value_cny": Decimal("0"),
"unrealized_pnl_usd": Decimal("0"),
},
)
if row.get("broker") not in item["brokers"]:
item["brokers"].append(row.get("broker"))
source_label = row.get("source_label") or row.get("broker")
if source_label not in item["sources"]:
item["sources"].append(source_label)
if row.get("market") not in item["markets"]:
item["markets"].append(row.get("market"))
item["market_value_usd"] += _decimal(row.get("market_value_usd"))
item["market_value_cny"] += _decimal(row.get("market_value_cny"))
item["unrealized_pnl_usd"] += _decimal(row.get("unrealized_pnl_usd"))
result = []
for item in grouped.values():
item["brokers"].sort()
item["sources"].sort()
item["markets"] = sorted(str(value) for value in item["markets"] if value)
item["duplicate_across_brokers"] = len(item["brokers"]) > 1
for key in ("market_value_usd", "market_value_cny", "unrealized_pnl_usd"):
item[key] = _number(item[key])
result.append(item)
return sorted(result, key=lambda row: _decimal(row["market_value_usd"]), reverse=True)
def _collect_source(
self, source: PortfolioSource, rates: Rates
) -> dict[str, Any]:
"""Read one source's account and positions, pricing what the connector omits.
Args:
source: The enabled source to read.
rates: Currency units per USD available for this refresh.
Returns:
``{"source_id", "profile_id", "label", "broker", "status",
"account", "positions"}`` for a successful read.
Raises:
RuntimeError: If the profile is not read-only, or either connector
read returns a non-success status.
"""
if source.source_kind == "extraetf":
return self.settings_store.import_store.collect(source)
connection, profile = self._connection_profile(source)
if not profile.readonly:
raise RuntimeError("portfolio sources must use a read-only connector profile")
broker = profile.connector
profile_id = profile.id
longbridge_batch: dict[str, Any] | None = None
if broker == "longbridge" and profile_id == "longbridge-live-sdk-readonly" and self._use_longbridge_worker:
isolated = self._read_longbridge_isolated(connection.id)
account = isolated["account"]
positions_payload = isolated["positions"]
longbridge_batch = isolated.get("quotes")
else:
read_options: dict[str, Any] = {}
if profile.transport == "remote_mcp":
# A dashboard refresh must never open a browser. reconnect_source()
# is the one explicit interactive path.
read_options["interactive_oauth"] = False
if requires_account_selection(profile):
# Checked before any call: one OAuth grant serves every
# account, so a read without one must not fall back to the
# broker's default account.
if not connection.account_ref:
raise RuntimeError(
f"Select a {broker} account for this connection before refreshing"
)
read_options["account"] = connection.account_ref
elif profile.transport in {"broker_sdk", "local_plugin", "local_tws"} and (
profile.transport == "local_plugin" or self._use_connection_scoped_reads
):
read_options["connection_id"] = connection.id
account = self._get_account(profile_id, **read_options)
positions_payload = self._get_positions(profile_id, **read_options)
for label, payload in (("account", account), ("positions", positions_payload)):
if str(payload.get("status") or "ok").lower() not in {"ok", "success"}:
raise RuntimeError(
str(payload.get("error") or f"{broker} {label} read failed")
)
account, positions_payload = adapt_and_validate_payloads(broker, account, positions_payload)
normalized_rows = [normalize_position(broker, raw) for raw in positions_payload.get("positions", [])]
for row in normalized_rows:
row.update(
source_id=source.id,
profile_id=connection.profile_id,
source_label=source.label,
)
longbridge_prices: dict[str, Decimal] | None = None
longbridge_price_error: str | None = None
if broker == "longbridge" and longbridge_batch is not None:
longbridge_prices = {
str(item.get("symbol") or "").upper(): _decimal(item.get("last"))
for item in longbridge_batch.get("quotes", [])
if _decimal(item.get("last")) > 0
}
if str(longbridge_batch.get("status") or "ok").lower() not in {
"ok",
"success",
}:
longbridge_price_error = str(longbridge_batch.get("error") or "Longbridge quote read failed")[:160]
elif broker == "longbridge" and self._get_longbridge_quotes is not None:
try:
batch = self._get_longbridge_quotes([str(row["quote_symbol"]) for row in normalized_rows])
longbridge_prices = {
str(item.get("symbol") or "").upper(): _decimal(item.get("last"))
for item in batch.get("quotes", [])
if _decimal(item.get("last")) > 0
}
except Exception as exc:
longbridge_prices = {}
longbridge_price_error = str(exc)[:160]
rows = []
for normalized in normalized_rows:
quantity = _decimal(normalized["quantity"])
if quantity == 0:
continue
try:
if broker in {"binance", "okx"} and normalized["symbol"] in STABLECOINS:
normalized["market_price"] = 1.0
normalized["price_currency"] = "USD"
normalized["pricing_basis"] = "USDT/USD proxy"
elif _decimal(normalized.get("market_price")) > 0:
normalized["price_currency"] = normalized["currency"]
normalized["pricing_basis"] = f"{broker} position snapshot"
elif broker == "longbridge" and longbridge_prices is not None:
price = longbridge_prices.get(str(normalized["quote_symbol"]).upper())
normalized["market_price"] = _number(price) if price else None
normalized["price_currency"] = normalized["currency"]
normalized["pricing_basis"] = (
"Longbridge official quote" if longbridge_batch is not None else "Fallback market quote"
)
if not price and longbridge_price_error:
normalized["price_error"] = longbridge_price_error
elif profile.transport == "remote_mcp" and "quotes.read" not in profile.capabilities:
normalized["market_price"] = None
normalized["price_currency"] = normalized["currency"]
normalized["price_error"] = f"{broker} quotes are not mapped for this connection"
else:
quote_symbol = normalized["quote_symbol"]
quote = self._get_quote(
quote_symbol,
profile_id,
exchange=normalized.get("exchange") or "SMART",
currency=normalized["currency"],
sec_type="STK",
# Same rule as the reads above: a dashboard refresh
# must never open a browser (reconnect_source() is the
# one explicit interactive path).
interactive_oauth=False,
**(
{"connection_id": connection.id}
if profile.transport == "local_plugin"
or (profile.transport in {"broker_sdk", "local_tws"} and self._use_connection_scoped_quotes)
else {}
),
)
price = _quote_price(quote)
normalized["market_price"] = _number(price) if price else None
normalized["price_currency"] = normalized["currency"]
quote_view = quote.get("quote") or {}
if broker == "robinhood":
if quote_view.get("price_as_of"):
normalized["price_as_of"] = quote_view["price_as_of"]
normalized["pricing_basis"] = f"Robinhood quote as of {quote_view['price_as_of']}"
if quote_view.get("price_error"):
normalized["price_error"] = str(quote_view["price_error"])[:160]
if isinstance(quote, dict) and quote.get("mapping_error"):
# The broker answered but the reply did not match its
# mapped shape: the row stays unpriced with the reason
# on it, never silently.
normalized["price_error"] = str(quote["mapping_error"])[:160]
except Exception as exc:
normalized["price_error"] = str(exc)[:160]
rows.append(normalized)
ensure_supported_currencies(rows, account, rates)
return {
"source_id": source.id,
"profile_id": connection.profile_id,
"label": source.label,
"broker": broker,
"status": "ok",
"account": account,
"positions": rows,
}
@staticmethod
def _read_longbridge_isolated(connection_id: str) -> dict[str, Any]:
"""Read Longbridge in a throwaway subprocess and parse its payload.
Returns:
The worker's account/positions/quotes payload.
Raises:
RuntimeError: If the worker emitted no payload, reported an error,
or returned an incomplete payload.
"""
marker = "VIBE_PORTFOLIO_JSON="
process = subprocess.run(
[
sys.executable,
"-m",
"src.portfolio.longbridge_worker",
connection_id,
],
check=False,
capture_output=True,
text=True,
timeout=40,
)
payload: dict[str, Any] | None = None
for line in reversed(process.stdout.splitlines()):
if line.startswith(marker):
parsed = json.loads(line[len(marker) :])
if isinstance(parsed, dict):
payload = parsed
break
if payload is None:
raise RuntimeError("Longbridge isolated reader returned no payload")
if payload.get("error"):
raise RuntimeError(f"Longbridge isolated reader failed: {payload['error']}")
if "account" not in payload or "positions" not in payload:
raise RuntimeError("Longbridge isolated reader returned an incomplete payload")
return payload
def _rates(self) -> tuple[FxQuote, bool]:
"""Fetch a validated quote or fall back to one coherent cached quote."""
try:
result = self._fx_fetcher()
if isinstance(result, FxQuote):
quote = result.validated()
else:
cny, hkd, fetched = result # Backwards-compatible injected readers.
quote = FxQuote(build_rates(cny, hkd), fetched).validated()
self.store.save_fx_quote(quote.to_dict())
return quote, False
except Exception:
cached = self.store.load_fx_quote()
if cached is not None:
return FxQuote(**cached).validated(), True
cny, hkd = self.store.load_fx("USD", "CNY"), self.store.load_fx("USD", "HKD")
if cny is None or hkd is None:
raise RuntimeError("FX service unavailable and no prior USD/CNY + USD/HKD cache exists")
return FxQuote(build_rates(Decimal(cny[0]), Decimal(hkd[0])), min(cny[1], hkd[1]),
source="legacy_cache").validated(), True
@staticmethod
def _fetch_fx() -> FxQuote:
"""Read Frankfurter v1's dated reference rates; no intraday FX claim."""
url = "https://api.frankfurter.app/latest?from=USD&to=CNY,HKD,EUR"
request = urllib.request.Request(url, headers={"User-Agent": "Vibe-Trading/portfolio"})
with urllib.request.urlopen(request, timeout=8) as response: # noqa: S310 - fixed HTTPS host
payload = json.load(response)
if payload.get("base") != "USD" or payload.get("amount") != 1:
raise RuntimeError("FX response does not declare a one-USD base")
observed = date.fromisoformat(payload["date"])
if observed > datetime.now(timezone.utc).date():
raise RuntimeError("FX response date is in the future")
rates = {code: Decimal(str(payload["rates"][code])) for code in ("CNY", "HKD", "EUR")}
return FxQuote({"USD": Decimal(1), **rates}, _now(), observed.isoformat(),
"Frankfurter v1 reference rates").validated()
@staticmethod
def _warnings(accounts: list[dict[str, Any]], positions: list[dict[str, Any]], fx_stale: bool) -> list[str]:
"""Describe every reason this snapshot must be read with caution.
Args:
accounts: The per-source account rows of this snapshot.
positions: The valued positions that were actually aggregated.
fx_stale: Whether cached FX rates were used instead of fresh ones.
Returns:
English warning strings, safe to render verbatim in any UI.
"""
warnings = []
for account in accounts:
metadata = account.get("import_metadata")
if metadata:
warnings.append(
f"extraETF file modified {metadata['source_mtime']}; file mtime is not a price observation. "
f"Expires {metadata['expires_at']} under your freshness policy. "
"Totals cover exported holdings only; cash and unexported assets are not included. "
"ISIN/venue identity is unresolved, so these holdings are excluded from risk-series tools."
)
if any(row.get("source_kind") == "extraetf" for row in accounts) and len(accounts) > 1:
warnings.append("Imported views may overlap other selected sources; duplicate accounts cannot be detected. Select disjoint sources only.")
if fx_stale:
warnings.append(
"Exchange-rate service unavailable; valued with the last "
"successfully fetched reference rates; see the recorded FX date."
)
failed = [str(row.get("label") or row["broker"]) for row in accounts if row["status"] != "ok"]
if failed:
warnings.append(
"These sources failed to refresh and their value is excluded "
"from the totals, positions and combined holdings, so the "
"portfolio shown here is incomplete: " + ", ".join(failed) + "."
)
unpriced = [f"{row['broker']}:{row['symbol']}" for row in positions if not row.get("priced")]
if unpriced:
warnings.append(
"No price available for these positions: "
+ ", ".join(unpriced[:20])
)
incomplete_unpriced = [
f"{row['broker']}:{row['symbol']}"
for row in positions
if row.get("broker") == "binance" and not row.get("priced")
]
if incomplete_unpriced:
warnings.append(
"Binance account totals are built from priced positions; these "
"holdings are excluded, so this snapshot is incomplete: "
+ ", ".join(incomplete_unpriced[:20])
)
experimental = [
str(row.get("label") or row.get("broker"))
for row in accounts
if (row.get("portfolio_compatibility") or {}).get("level") == "experimental"
]
if experimental:
warnings.append(
"Experimental portfolio connectors require broker-specific "
"verification: " + ", ".join(experimental) + "."
)
crypto_proxies = sorted(
{str(row.get("broker")) for row in positions if row.get("broker") in {"binance", "okx"}}
)
if crypto_proxies:
warnings.append(", ".join(crypto_proxies) + " spot balances are valued with USDT treated as 1 USD.")
return warnings