"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
feat(portfolio): add a read-only extraETF export reader
cgycorey79df088d0b2f
diff --git a/agent/src/portfolio/extraetf.py b/agent/src/portfolio/extraetf.pynew file mode 100644index 000000000..88ad56490--- /dev/null+++ b/agent/src/portfolio/extraetf.py@@ -0,0 +1,648 @@+"""Read-only reader for extraETF portfolio exports.++extraETF exports two semicolon-delimited, German-locale files:++``holdings`` — the positions of one view at export time::++ ISIN;Name;Typ;Anzahl;Kaufpreis;Aktueller Kurs;Aktueller Wert;Währung;Wechselkurs;Region;Land;Sektor;Portfolioname;Portfolio ID++``transactions`` — the movement history of one view::++ Datum;ISIN;Name;Typ;Transaktion;Anzahl;Preis;Gebühren;Steuern;Währung;Wechselkurs;Portfolioname;Portfolio ID;Stückzinsen++Three properties are deliberate and load-bearing.++**A transactions export can never be read as a portfolio.** Its rows are+movements — ``Kauf``/``Verkauf``/``Dividende``/``Einbuchung``/``Ausbuchung`` —+not holdings. An allocation computed from trades looks perfectly plausible and+is wrong, so the two formats are told apart by their header and+:meth:`ExtraEtfExport.require_positions` refuses anything that is not a+holdings export. Movements carry an explicit ``movement_kind`` so a dividend or+a transfer is never mistaken for a trade.++**A holdings export carries no date.** The format has no snapshot column, so+freshness can only come from the file's own modification time, and the export+says so through ``as_of_source``. This reader never claims an observation time+it did not read.++**Validation fails closed.** A row this reader does not recognise raises+:class:`ExtraEtfFormatError` instead of being skipped: an import computed from+90% of a portfolio reads as entirely plausible, whereas a refused import is+merely annoying. Numbers are parsed as German — ``1.386,608`` is 1386.608 — and+a value that cannot be read unambiguously is refused rather than guessed.+``1,234.56`` is US-formatted; reading it as German would be wrong by three+orders of magnitude, so it is refused instead.++Rows are read by column name, so a reordered export cannot mis-align a value+into the wrong column. Rows are **not** merged or de-duplicated: two rows for+the same instrument in one portfolio are returned as two positions, because+summing them silently and dropping one silently are both wrong, and which one+the user meant is not this module's call.++The module writes nothing, holds no credentials, and touches no network. It+only reads a local file the user exported themselves, and it has no path to any+order surface.+"""++from __future__ import annotations++import csv+import io+import os+import re+from dataclasses import dataclass+from datetime import datetime, timezone+from decimal import Decimal, InvalidOperation+from pathlib import Path+from typing import Any, Literal++BROKER = "extraetf"++_HOLDINGS_COLUMNS: tuple[str, ...] = (+ "ISIN",+ "Name",+ "Typ",+ "Anzahl",+ "Kaufpreis",+ "Aktueller Kurs",+ "Aktueller Wert",+ "Währung",+ "Wechselkurs",+ "Region",+ "Land",+ "Sektor",+ "Portfolioname",+ "Portfolio ID",+)++_TRANSACTIONS_COLUMNS: tuple[str, ...] = (+ "Datum",+ "ISIN",+ "Name",+ "Typ",+ "Transaktion",+ "Anzahl",+ "Preis",+ "Gebühren",+ "Steuern",+ "Währung",+ "Wechselkurs",+ "Portfolioname",+ "Portfolio ID",+ "Stückzinsen",+)++ExportKind = Literal["holdings", "transactions"]++_ISIN_PATTERN = re.compile(r"^[A-Z]{2}[A-Z0-9]{9}[0-9]$")+_CRYPTO_PAIR_PATTERN = re.compile(r"^[A-Z0-9]{2,12}_to_[A-Z0-9]{2,12}$", re.IGNORECASE)+_CURRENCY_PATTERN = re.compile(r"^[A-Z]{3}$")+_DATE_PATTERN = re.compile(r"^(\d{2})\.(\d{2})\.(\d{4})$")+_NUMERAL_PATTERN = re.compile(r"^[0-9.,]+$")++#: ``Typ`` is an instrument class, not a row shape: a class this reader does not+#: know still describes a real position, so it is carried through with+#: ``asset_type`` left ``None`` rather than refused or guessed at.+_ASSET_TYPE_BY_EXPORT_TYPE = {+ "etf": "etf",+ "aktie": "stock",+ "währung / krypto": "crypto",+}++_MOVEMENT_KIND_BY_TRANSACTION = {+ "kauf": "buy",+ "verkauf": "sell",+ "dividende": "dividend",+ "einbuchung": "transfer_in",+ "ausbuchung": "transfer_out",+}+++class ExtraEtfFormatError(ValueError):+ """Raised when an export cannot be read without guessing."""+++@dataclass(frozen=True)+class ExtraEtfExport:+ """One parsed export, with its provenance attached.++ Attributes:+ kind: Which of the two extraETF formats this file is.+ positions: Holdings rows. Always empty for a transactions export.+ movements: Transactions rows. Always empty for a holdings export.+ as_of: ISO-8601 observation time, or ``None`` when it could not be read.+ as_of_source: Where ``as_of`` came from — ``"file_mtime"`` for a file+ read from disk, ``"unavailable"`` for bare text with no timestamp.+ A holdings export has no date column, so ``file_mtime`` is an+ inference from the file's own modification time and is labelled as+ one rather than presented as a recorded snapshot date.+ source_path: The file this export was read from, when there was one.+ """++ kind: ExportKind+ positions: tuple[dict[str, Any], ...]+ movements: tuple[dict[str, Any], ...]+ as_of: str | None+ as_of_source: Literal["file_mtime", "unavailable"]+ source_path: Path | None++ def require_positions(self) -> tuple[dict[str, Any], ...]:+ """Return holdings positions, refusing any other export outright.++ Returns:+ The parsed positions, possibly empty for an empty portfolio.++ Raises:+ ExtraEtfFormatError: If this export is a transactions export, whose+ rows are movements rather than holdings.+ """+ if self.kind != "holdings":+ raise ExtraEtfFormatError(+ "this is a transactions export and holds movements, not positions; "+ "importing it as a portfolio would build an allocation out of trades"+ )+ return self.positions+++def read_export(path: str | Path) -> ExtraEtfExport:+ """Read and parse one extraETF export file.++ The file's modification time becomes ``as_of`` with+ ``as_of_source="file_mtime"``, because neither export format carries a+ snapshot-observation column.++ Args:+ path: The exported ``.csv`` file.++ Returns:+ The parsed export.++ Raises:+ ExtraEtfFormatError: If the file cannot be read, decoded, or parsed.+ """+ resolved = Path(path)+ try:+ # Stat the same descriptor the bytes came from: a separate ``stat`` call+ # can observe a later write than the read did, which would let the import+ # claim a fresher observation time than the content it actually holds.+ with resolved.open("rb") as handle:+ data = handle.read()+ modified = os.fstat(handle.fileno()).st_mtime+ except OSError as exc:+ raise ExtraEtfFormatError(f"cannot read export {resolved}: {exc}") from exc+ as_of = datetime.fromtimestamp(modified, tz=timezone.utc).isoformat()+ return parse_export(+ data, source_path=resolved, as_of=as_of, as_of_source="file_mtime"+ )+++def parse_export(+ data: str | bytes,+ *,+ source_path: str | Path | None = None,+ as_of: str | None = None,+ as_of_source: Literal["file_mtime", "unavailable"] = "unavailable",+) -> ExtraEtfExport:+ """Parse one extraETF export from text or raw bytes.++ Args:+ data: The export contents, as text or as raw file bytes.+ source_path: The originating file, recorded for provenance only.+ as_of: ISO-8601 observation time, when the caller knows one.+ as_of_source: Where ``as_of`` came from.++ Returns:+ The parsed export. An export with a header and no data rows is valid and+ yields no positions — an empty portfolio is not an unreadable one.++ Raises:+ ExtraEtfFormatError: If the header is not one of the two known formats,+ or any row cannot be read without guessing.+ ValueError: If ``as_of_source`` claims an origin for an absent ``as_of``.+ """+ if as_of is None and as_of_source != "unavailable":+ raise ValueError(+ "as_of_source cannot claim where an observation time came from when "+ "there is no observation time"+ )+ text = _decode(data)+ kind, header, rows = _read_rows(text)+ if kind == "holdings":+ positions = tuple(+ _parse_holding(header, cells, number, as_of=as_of) for number, cells in rows+ )+ movements: tuple[dict[str, Any], ...] = ()+ else:+ positions = ()+ movements = tuple(+ _parse_movement(header, cells, number) for number, cells in rows+ )+ return ExtraEtfExport(+ kind=kind,+ positions=positions,+ movements=movements,+ as_of=as_of,+ as_of_source=as_of_source,+ source_path=Path(source_path) if source_path is not None else None,+ )+++def _decode(data: str | bytes) -> str:+ """Decode export bytes, preferring UTF-8 and falling back to cp1252.++ Args:+ data: Export contents as text or bytes.++ Returns:+ The decoded text.++ Raises:+ ExtraEtfFormatError: If neither encoding decodes the bytes.+ """+ if isinstance(data, str):+ return data+ for encoding in ("utf-8-sig", "cp1252"):+ try:+ return data.decode(encoding)+ except UnicodeDecodeError:+ continue+ raise ExtraEtfFormatError("export is decodable as neither UTF-8 nor cp1252")+++def _read_rows(+ text: str,+) -> tuple[ExportKind, tuple[str, ...], list[tuple[int, list[str]]]]:+ """Split an export into its format, header and data rows.++ Args:+ text: The decoded export.++ Returns:+ The detected format, the header columns, and ``(line number, cells)``+ pairs for every non-blank data row.++ Raises:+ ExtraEtfFormatError: If the export is empty, its header is not one of the+ two known formats, or a row cannot be aligned to that header.+ """+ reader = csv.reader(io.StringIO(text), delimiter=";")+ header: tuple[str, ...] | None = None+ number = 0+ for cells in reader:+ number += 1+ if not any(cell.strip() for cell in cells):+ continue+ header = tuple(cell.strip().lstrip("\ufeff") for cell in cells)+ break+ if header is None:+ raise ExtraEtfFormatError("export is empty")+ kind = _detect_kind(header)++ rows: list[tuple[int, list[str]]] = []+ for cells in reader:+ number += 1+ if not any(cell.strip() for cell in cells):+ continue+ if len(cells) != len(header):+ raise ExtraEtfFormatError(+ f"row {number}: expected {len(header)} fields to match the header, "+ f"found {len(cells)}; a row that cannot be aligned is refused rather "+ f"than skipped"+ )+ rows.append((number, [cell.strip() for cell in cells]))+ return kind, header, rows+++def _detect_kind(header: tuple[str, ...]) -> ExportKind:+ """Identify which extraETF format a header belongs to.++ Args:+ header: The export's column names.++ Returns:+ ``"holdings"`` or ``"transactions"``.++ Raises:+ ExtraEtfFormatError: If the header is neither known format exactly.+ """+ columns = set(header)+ if len(header) == len(_HOLDINGS_COLUMNS) and columns == set(_HOLDINGS_COLUMNS):+ return "holdings"+ if len(header) == len(_TRANSACTIONS_COLUMNS) and columns == set(+ _TRANSACTIONS_COLUMNS+ ):+ return "transactions"+ transactions_like = "Transaktion" in columns+ expected = _TRANSACTIONS_COLUMNS if transactions_like else _HOLDINGS_COLUMNS+ missing = [column for column in expected if column not in columns]+ unexpected = [column for column in header if column not in expected]+ raise ExtraEtfFormatError(+ "export header is not a recognised extraETF format; against the "+ f"{'transactions' if transactions_like else 'holdings'} format, missing "+ f"columns: {missing or 'none'}, unexpected columns: {unexpected or 'none'}"+ )+++def _parse_holding(+ header: tuple[str, ...], cells: list[str], number: int, *, as_of: str | None+) -> dict[str, Any]:+ """Convert one holdings row into a position.++ Args:+ header: The export's column names, which key the row's fields.+ cells: The row's fields.+ number: The row's line number, for error messages.+ as_of: The export's observation time, mirrored onto the position.++ Returns:+ One position in the portfolio wire shape.++ Raises:+ ExtraEtfFormatError: If the row cannot be read without guessing.+ """+ row = dict(zip(header, cells, strict=True))+ identifier, identifier_kind, checksum_ok = _describe_identifier(row["ISIN"])+ if identifier_kind not in {"isin", "crypto_pair"}:+ raise ExtraEtfFormatError(+ f"row {number}: {row['ISIN']!r} is neither an ISIN nor an extraETF crypto "+ f"pair such as BTC_to_EUR, so the holding cannot be attributed to an "+ f"instrument"+ )+ name = _required(row, "Name", number)+ instrument_type = _required(row, "Typ", number)+ currency = _required(row, "Währung", number).upper()+ if not _CURRENCY_PATTERN.fullmatch(currency):+ raise ExtraEtfFormatError(+ f"row {number}: Währung is not a three-letter currency code: "+ f"{row['Währung']!r}"+ )+ portfolio_id = _required(row, "Portfolio ID", number)+ quantity = _parse_decimal(row["Anzahl"], field="Anzahl", row_number=number)+ if quantity is None:+ raise ExtraEtfFormatError(f"row {number}: Anzahl is empty")+ return {+ "broker": BROKER,+ "source": "extraetf_export",+ "symbol": identifier,+ "quote_symbol": identifier,+ "instrument_id": identifier,+ "instrument_id_kind": identifier_kind,+ "instrument_id_checksum_ok": checksum_ok,+ "name": name,+ "instrument_type": instrument_type,+ "asset_type": _ASSET_TYPE_BY_EXPORT_TYPE.get(instrument_type.strip().lower()),+ "market": None,+ "currency": currency,+ "price_currency": currency,+ "quantity": _number(quantity),+ "cost_price": _optional_number(+ row["Kaufpreis"], field="Kaufpreis", row_number=number+ ),+ "market_price": _optional_number(+ row["Aktueller Kurs"], field="Aktueller Kurs", row_number=number+ ),+ "source_market_value": _optional_number(+ row["Aktueller Wert"], field="Aktueller Wert", row_number=number+ ),+ "fx_rate": _optional_number(+ row["Wechselkurs"], field="Wechselkurs", row_number=number+ ),+ "region": row["Region"] or None,+ "country": row["Land"] or None,+ "sector": row["Sektor"] or None,+ "portfolio_id": portfolio_id,+ "portfolio_name": row["Portfolioname"] or None,+ "updated_at": as_of,+ }+++def _parse_movement(+ header: tuple[str, ...], cells: list[str], number: int+) -> dict[str, Any]:+ """Convert one transactions row into a movement.++ Args:+ header: The export's column names, which key the row's fields.+ cells: The row's fields.+ number: The row's line number, for error messages.++ Returns:+ One movement, carrying the export's own transaction label plus a+ normalised ``movement_kind``.++ Raises:+ ExtraEtfFormatError: If the row cannot be read without guessing.+ """+ row = dict(zip(header, cells, strict=True))+ movement = _required(row, "Transaktion", number)+ transaction_date = _required(row, "Datum", number)+ match = _DATE_PATTERN.fullmatch(transaction_date)+ if match is None:+ raise ExtraEtfFormatError(+ f"row {number}: Datum is not a DD.MM.YYYY date: {transaction_date!r}"+ )+ day, month, year = (int(part) for part in match.groups())+ try:+ booked_on = datetime(year, month, day).date().isoformat()+ except ValueError as exc:+ raise ExtraEtfFormatError(+ f"row {number}: Datum is not a real date: {transaction_date!r}"+ ) from exc+ currency = _required(row, "Währung", number).upper()+ if not _CURRENCY_PATTERN.fullmatch(currency):+ raise ExtraEtfFormatError(+ f"row {number}: Währung is not a three-letter currency code: "+ f"{row['Währung']!r}"+ )+ identifier, identifier_kind, checksum_ok = _describe_identifier(row["ISIN"])+ quantity = _parse_decimal(row["Anzahl"], field="Anzahl", row_number=number)+ if quantity is None:+ raise ExtraEtfFormatError(f"row {number}: Anzahl is empty")+ return {+ "broker": BROKER,+ "source": "extraetf_export",+ "date": booked_on,+ "instrument_id": identifier or None,+ "instrument_id_kind": identifier_kind,+ "instrument_id_checksum_ok": checksum_ok,+ "name": _required(row, "Name", number),+ "instrument_type": _required(row, "Typ", number),+ "movement": movement,+ "movement_kind": _MOVEMENT_KIND_BY_TRANSACTION.get(movement.strip().lower()),+ "quantity": _number(quantity),+ "price": _optional_number(row["Preis"], field="Preis", row_number=number),+ "fees": _optional_number(row["Gebühren"], field="Gebühren", row_number=number),+ "taxes": _optional_number(row["Steuern"], field="Steuern", row_number=number),+ "currency": currency,+ "fx_rate": _optional_number(+ row["Wechselkurs"], field="Wechselkurs", row_number=number+ ),+ "accrued_interest": _optional_number(+ row["Stückzinsen"], field="Stückzinsen", row_number=number+ ),+ "portfolio_id": _required(row, "Portfolio ID", number),+ "portfolio_name": row["Portfolioname"] or None,+ }+++def _required(row: dict[str, str], column: str, number: int) -> str:+ """Return a column's value, refusing an empty one.++ Args:+ row: The aligned row.+ column: The column to read.+ number: The row's line number, for error messages.++ Returns:+ The non-empty value.++ Raises:+ ExtraEtfFormatError: If the value is empty.+ """+ value = row[column]+ if not value:+ raise ExtraEtfFormatError(f"row {number}: {column} is empty")+ return value+++def _optional_number(raw: str, *, field: str, row_number: int) -> float | None:+ """Parse an optional German-formatted number.++ Args:+ raw: The raw field text.+ field: The column name, for error messages.+ row_number: The row's line number, for error messages.++ Returns:+ The value, or ``None`` when the field is blank. A blank price is an+ absent observation rather than a zero, and is never guessed at.++ Raises:+ ExtraEtfFormatError: If the field is present but unreadable.+ """+ value = _parse_decimal(raw, field=field, row_number=row_number)+ return None if value is None else _number(value)+++def _parse_decimal(raw: str, *, field: str, row_number: int) -> Decimal | None:+ """Parse a German-formatted number, refusing anything ambiguous.++ A comma is the decimal separator and a period may only be a thousands+ separator: ``1.386,608`` is 1386.608 and ``12,3456`` is 12.3456. A value+ whose separators are US-ordered, or whose period grouping is not a plain+ thousands separator, is refused rather than guessed at, because guessing+ wrong changes a position's size by orders of magnitude.++ Args:+ raw: The raw field text.+ field: The column name, for error messages.+ row_number: The row's line number, for error messages.++ Returns:+ The parsed value, or ``None`` when the field is blank.++ Raises:+ ExtraEtfFormatError: If the field is unreadable or ambiguous.+ """+ text = raw.strip()+ if not text:+ return None+ sign = ""+ if text[0] in "+-":+ sign, text = text[0], text[1:]+ if not text or not _NUMERAL_PATTERN.fullmatch(text):+ raise ExtraEtfFormatError(f"row {row_number}: {field} is not a number: {raw!r}")+ whole, comma, fraction = text.rpartition(",")+ if not comma:+ whole, fraction = text, ""+ elif "." in fraction:+ raise ExtraEtfFormatError(+ f"row {row_number}: {field} reads as a US-formatted number: {raw!r}; "+ f"extraETF exports use German formatting such as 1.386,608"+ )+ groups = whole.split(".")+ if (+ any(not group.isdigit() for group in groups)+ or any(len(group) != 3 for group in groups[1:])+ or (comma and not fraction.isdigit())+ ):+ raise ExtraEtfFormatError(+ f"row {row_number}: {field} has ambiguous separators: {raw!r}; expected "+ f"German formatting such as 1.386,608"+ )+ digits = "".join(groups) + (f".{fraction}" if comma else "")+ try:+ return Decimal(f"{sign}{digits}")+ except InvalidOperation as exc: # pragma: no cover - guarded by the checks above+ raise ExtraEtfFormatError(+ f"row {row_number}: {field} is not a number: {raw!r}"+ ) from exc+++def _describe_identifier(raw: str) -> tuple[str, str, bool | None]:+ """Describe an export identifier without refusing anything.++ Args:+ raw: The raw ``ISIN`` column value.++ Returns:+ The identifier as the export spelled it, its kind — ``"isin"``,+ ``"crypto_pair"``, ``"raw_symbol"`` or ``"unknown"`` — and, for an+ ISIN-shaped value, whether its check digit is valid. extraETF stores+ crypto identifiers that are not ISINs, so the check digit is reported+ rather than enforced: refusing a whole import over one unverifiable+ identifier would block a user whose positions are otherwise perfectly+ readable.++ An ISIN is returned upper-cased, because an ISIN is canonical and+ case-insensitive. A crypto pair such as ``BTC_to_EUR`` is returned+ exactly as exported, because it is extraETF's own label and not a+ canonical code — rewriting its case would invent an identity we do not+ own.+ """+ identifier = raw.strip()+ if not identifier:+ return "", "unknown", None+ canonical = identifier.upper()+ if _ISIN_PATTERN.fullmatch(canonical):+ return canonical, "isin", _isin_checksum_ok(canonical)+ if _CRYPTO_PAIR_PATTERN.fullmatch(identifier):+ return identifier, "crypto_pair", None+ return identifier, "raw_symbol", None+++def _isin_checksum_ok(value: str) -> bool:+ """Check an ISIN's check digit.++ Args:+ value: A value already matching :data:`_ISIN_PATTERN`.++ Returns:+ ``True`` when the Luhn-style check digit is valid.+ """+ digits = "".join(+ str(ord(character) - 55) if character.isalpha() else character+ for character in value+ )+ total = 0+ for index, character in enumerate(reversed(digits)):+ digit = int(character)+ if index % 2:+ digit *= 2+ if digit > 9:+ digit -= 9+ total += digit+ return total % 10 == 0+++def _number(value: Decimal) -> float:+ """Return a wire-ready float, quantized as the connector path does.++ Args:+ value: The parsed value.++ Returns:+ The value as a float, quantized to eight decimal places to match+ :func:`src.portfolio.normalization.normalize_position`.+ """+ return float(value.quantize(Decimal("0.00000001")))diff --git a/agent/tests/test_extraetf_reader.py b/agent/tests/test_extraetf_reader.pynew file mode 100644index 000000000..928962846--- /dev/null+++ b/agent/tests/test_extraetf_reader.py@@ -0,0 +1,489 @@+"""Tests for the read-only extraETF export reader.++All fixtures are synthetic, shaped after the anonymized exports posted in+HKUDS/Vibe-Trading#1170. No real portfolio data is used.+"""++from __future__ import annotations++import os+from datetime import datetime, timezone++import pytest++from src.portfolio.extraetf import (+ ExtraEtfFormatError,+ parse_export,+ read_export,+)++HOLDINGS_COLUMNS = [+ "ISIN",+ "Name",+ "Typ",+ "Anzahl",+ "Kaufpreis",+ "Aktueller Kurs",+ "Aktueller Wert",+ "Währung",+ "Wechselkurs",+ "Region",+ "Land",+ "Sektor",+ "Portfolioname",+ "Portfolio ID",+]++TRANSACTIONS_COLUMNS = [+ "Datum",+ "ISIN",+ "Name",+ "Typ",+ "Transaktion",+ "Anzahl",+ "Preis",+ "Gebühren",+ "Steuern",+ "Währung",+ "Wechselkurs",+ "Portfolioname",+ "Portfolio ID",+ "Stückzinsen",+]++#: A real, checksum-valid ISIN, used so the happy path is realistic.+VALID_ISIN = "US0378331005"+#: The anonymized export's placeholder ISIN, which is shaped like an ISIN but has+#: an invalid check digit.+PLACEHOLDER_ISIN = "IE00AAAA0000"+++def _holding(+ *,+ isin: str = VALID_ISIN,+ name: str = "Example World ETF",+ instrument_type: str = "ETF",+ anzahl: str = "12,3456",+ kaufpreis: str = "101,234567",+ kurs: str = "112,345",+ wert: str = "1.386,608",+ waehrung: str = "EUR",+ wechselkurs: str = "1,00",+ region: str = "Welt",+ land: str = "",+ sektor: str = "",+ portfolio: str = "Example broker",+ portfolio_id: str = "1000001",+) -> dict[str, str]:+ return {+ "ISIN": isin,+ "Name": name,+ "Typ": instrument_type,+ "Anzahl": anzahl,+ "Kaufpreis": kaufpreis,+ "Aktueller Kurs": kurs,+ "Aktueller Wert": wert,+ "Währung": waehrung,+ "Wechselkurs": wechselkurs,+ "Region": region,+ "Land": land,+ "Sektor": sektor,+ "Portfolioname": portfolio,+ "Portfolio ID": portfolio_id,+ }+++def _movement(+ *,+ datum: str = "01.09.2026",+ isin: str = VALID_ISIN,+ name: str = "Example World ETF",+ instrument_type: str = "ETF",+ transaktion: str = "Kauf",+ anzahl: str = "1,2345",+ preis: str = "110,123456",+ gebuehren: str = "0,00",+ steuern: str = "0,00",+ waehrung: str = "EUR",+ wechselkurs: str = "1,00",+ portfolio: str = "Example broker",+ portfolio_id: str = "1000001",+ stueckzinsen: str = "0",+) -> dict[str, str]:+ return {+ "Datum": datum,+ "ISIN": isin,+ "Name": name,+ "Typ": instrument_type,+ "Transaktion": transaktion,+ "Anzahl": anzahl,+ "Preis": preis,+ "Gebühren": gebuehren,+ "Steuern": steuern,+ "Währung": waehrung,+ "Wechselkurs": wechselkurs,+ "Portfolioname": portfolio,+ "Portfolio ID": portfolio_id,+ "Stückzinsen": stueckzinsen,+ }+++def _export(columns: list[str], rows: list[dict[str, str]]) -> str:+ lines = [";".join(columns)]+ lines.extend(";".join(row[column] for column in columns) for row in rows)+ return "\n".join(lines) + "\n"+++def _holdings_export(*rows: dict[str, str]) -> str:+ return _export(HOLDINGS_COLUMNS, list(rows))+++def _transactions_export(*rows: dict[str, str]) -> str:+ return _export(TRANSACTIONS_COLUMNS, list(rows))+++def test_holdings_export_normalizes_a_german_locale_position():+ export = parse_export(_holdings_export(_holding()))++ assert export.kind == "holdings"+ (position,) = export.require_positions()+ assert position["instrument_id"] == VALID_ISIN+ assert position["instrument_id_kind"] == "isin"+ assert position["instrument_id_checksum_ok"] is True+ assert position["symbol"] == VALID_ISIN+ assert position["broker"] == "extraetf"+ assert position["quantity"] == pytest.approx(12.3456)+ assert position["cost_price"] == pytest.approx(101.234567)+ assert position["market_price"] == pytest.approx(112.345)+ assert position["source_market_value"] == pytest.approx(1386.608)+ assert position["currency"] == "EUR"+ assert position["price_currency"] == "EUR"+ assert position["asset_type"] == "etf"+ assert position["instrument_type"] == "ETF"+ assert position["portfolio_id"] == "1000001"+ assert position["portfolio_name"] == "Example broker"+ assert position["region"] == "Welt"+ assert position["country"] is None+ assert position["sector"] is None+++def test_german_separators_are_not_read_as_us_numbers():+ """``1.386,608`` is 1386.608, not 1.386 and not 1386608."""+ export = parse_export(+ _holdings_export(+ _holding(+ anzahl="12,3456",+ kaufpreis="1.386,608",+ kurs="59.999,99",+ wert="1.386.608,42",+ wechselkurs="1,0855",+ )+ )+ )++ (position,) = export.require_positions()+ assert position["quantity"] == pytest.approx(12.3456)+ assert position["cost_price"] == pytest.approx(1386.608)+ assert position["market_price"] == pytest.approx(59999.99)+ assert position["source_market_value"] == pytest.approx(1386608.42)+ assert position["fx_rate"] == pytest.approx(1.0855)+++def test_us_formatted_number_is_refused_rather_than_misread():+ with pytest.raises(ExtraEtfFormatError, match="US-formatted"):+ parse_export(_holdings_export(_holding(kaufpreis="1,234.56")))++[email protected]("ambiguous", ["1.23", "1234.56", "0.5"])+def test_ambiguous_separator_grouping_is_refused(ambiguous):+ """A period that is not a thousands separator cannot be guessed at."""+ with pytest.raises(ExtraEtfFormatError, match="ambiguous separators"):+ parse_export(_holdings_export(_holding(kurs=ambiguous)))++[email protected]("unreadable", ["zwei", "10,00 EUR", "--", ","])+def test_unreadable_number_is_refused(unreadable):+ with pytest.raises(ExtraEtfFormatError, match="is not a number|ambiguous|empty"):+ parse_export(_holdings_export(_holding(anzahl=unreadable)))+++def test_blank_optional_fields_are_none_rather_than_zero():+ export = parse_export(+ _holdings_export(_holding(kaufpreis="", kurs="", wert="", wechselkurs=""))+ )++ (position,) = export.require_positions()+ assert position["cost_price"] is None+ assert position["market_price"] is None+ assert position["source_market_value"] is None+ assert position["fx_rate"] is None+++def test_transactions_export_yields_movements_and_never_positions():+ """A transactions export is a movement history, not a portfolio."""+ export = parse_export(+ _transactions_export(+ _movement(transaktion="Kauf"),+ _movement(transaktion="Verkauf", anzahl="2,5000", preis="19,876543"),+ _movement(+ transaktion="Dividende",+ anzahl="0,5000",+ preis="2,345",+ steuern="0,45",+ ),+ _movement(transaktion="Einbuchung", anzahl="5,0000", preis="95,000000"),+ _movement(transaktion="Ausbuchung", anzahl="0,5000", preis="90,000000"),+ )+ )++ assert export.kind == "transactions"+ assert export.positions == ()+ assert export.movements[0]["movement"] == "Kauf"+ assert export.movements[0]["movement_kind"] == "buy"+ assert [m["movement_kind"] for m in export.movements] == [+ "buy",+ "sell",+ "dividend",+ "transfer_in",+ "transfer_out",+ ]+ # A dividend row carries a quantity of its own; it must never become a holding.+ assert export.movements[2]["quantity"] == pytest.approx(0.5)+ assert export.movements[2]["taxes"] == pytest.approx(0.45)+ assert all("asset_type" not in movement for movement in export.movements)+ assert all("accrued_interest" in movement for movement in export.movements)+ assert export.movements[0]["date"] == "2026-09-01"+ assert export.movements[0]["accrued_interest"] == pytest.approx(0.0)+++def test_a_transactions_export_refuses_to_pose_as_a_portfolio():+ export = parse_export(_transactions_export(_movement(transaktion="Kauf")))++ with pytest.raises(ExtraEtfFormatError, match="movements, not positions"):+ export.require_positions()+++def test_unknown_transaction_type_is_carried_through_not_invented():+ export = parse_export(_transactions_export(_movement(transaktion="Steuer")))++ assert export.positions == ()+ assert export.movements[0]["movement"] == "Steuer"+ assert export.movements[0]["movement_kind"] is None+++def test_invalid_transaction_date_is_refused():+ with pytest.raises(ExtraEtfFormatError, match="DD.MM.YYYY"):+ parse_export(_transactions_export(_movement(datum="2026-09-01")))++ with pytest.raises(ExtraEtfFormatError, match="not a real date"):+ parse_export(_transactions_export(_movement(datum="31.02.2026")))+++def test_crypto_identity_is_a_pair_not_an_isin():+ export = parse_export(+ _holdings_export(+ _holding(+ isin="BTC_to_EUR",+ name="Bitcoin",+ instrument_type="Währung / Krypto",+ region="Global",+ sektor="Diversifizierte Kapitalmärkte",+ portfolio="Example crypto wallet",+ portfolio_id="1000002",+ )+ )+ )++ (position,) = export.require_positions()+ assert position["instrument_id"] == "BTC_to_EUR"+ assert position["instrument_id_kind"] == "crypto_pair"+ assert position["instrument_id_checksum_ok"] is None+ assert position["asset_type"] == "crypto"+ assert position["portfolio_id"] == "1000002"++[email protected]("identifier", ["BTC", "", " ", "NOT*AN*ID"])+def test_unattributable_holding_identifier_is_refused(identifier):+ """A bare symbol cannot be resolved, so the import fails instead of guessing."""+ with pytest.raises(ExtraEtfFormatError, match="neither an ISIN nor"):+ parse_export(_holdings_export(_holding(isin=identifier)))+++def test_invalid_isin_check_digit_is_reported_not_enforced():+ export = parse_export(+ _holdings_export(_holding(isin=VALID_ISIN), _holding(isin=PLACEHOLDER_ISIN))+ )++ valid, placeholder = export.require_positions()+ assert valid["instrument_id_checksum_ok"] is True+ assert placeholder["instrument_id_checksum_ok"] is False+++def test_unknown_instrument_type_leaves_asset_type_unset():+ export = parse_export(_holdings_export(_holding(instrument_type="Anleihe")))++ (position,) = export.require_positions()+ assert position["instrument_type"] == "Anleihe"+ assert position["asset_type"] is None+++def test_missing_required_field_is_refused():+ for kwarg, message in (+ ("name", "Name is empty"),+ ("instrument_type", "Typ is empty"),+ ("waehrung", "Währung is empty"),+ ("portfolio_id", "Portfolio ID is empty"),+ ):+ with pytest.raises(ExtraEtfFormatError, match=message):+ parse_export(_holdings_export(_holding(**{kwarg: ""})))+++def test_invalid_currency_is_refused():+ with pytest.raises(ExtraEtfFormatError, match="three-letter currency"):+ parse_export(_holdings_export(_holding(waehrung="EURO")))+++def test_unknown_column_makes_the_export_unrecognisable():+ row = _holding()+ header = ";".join([*HOLDINGS_COLUMNS, "Neue Spalte"])+ body = ";".join([*(row[column] for column in HOLDINGS_COLUMNS), "x"])++ with pytest.raises(+ ExtraEtfFormatError, match="unexpected columns: \\['Neue Spalte'\\]"+ ):+ parse_export(f"{header}\n{body}\n")+++def test_missing_column_makes_the_export_unrecognisable():+ columns = [column for column in HOLDINGS_COLUMNS if column != "Anzahl"]+ row = _holding()++ with pytest.raises(ExtraEtfFormatError, match="missing columns: \\['Anzahl'\\]"):+ parse_export(_export(columns, [row]))+++def test_ragged_row_is_refused_rather_than_partially_imported():+ """One malformed row fails the import; it never yields a short portfolio."""+ row = _holding()+ short = ";".join(row[column] for column in HOLDINGS_COLUMNS[:-1])+ long = ";".join([*(row[column] for column in HOLDINGS_COLUMNS), "extra"])++ for ragged in (short, long):+ with pytest.raises(ExtraEtfFormatError, match="cannot be aligned"):+ parse_export(_holdings_export(_holding(), _holding()) + ragged + "\n")+++def test_reordered_columns_do_not_mis_align_values():+ """Rows are keyed by column name, so a reordered export stays correct."""+ columns = list(reversed(HOLDINGS_COLUMNS))++ export = parse_export(_export(columns, [_holding()]))++ (position,) = export.require_positions()+ assert position["currency"] == "EUR"+ assert position["quantity"] == pytest.approx(12.3456)+ assert position["name"] == "Example World ETF"+ assert position["portfolio_id"] == "1000001"+++def test_duplicate_rows_are_preserved_rather_than_merged():+ export = parse_export(+ _holdings_export(_holding(anzahl="1,0000"), _holding(anzahl="2,0000"))+ )++ assert [position["quantity"] for position in export.require_positions()] == [+ pytest.approx(1.0),+ pytest.approx(2.0),+ ]+++def test_empty_holdings_export_is_valid_and_not_an_error():+ """An empty portfolio and an unreadable file are different states."""+ export = parse_export(_holdings_export())++ assert export.kind == "holdings"+ assert export.require_positions() == ()+++def test_empty_file_is_refused():+ for empty in ("", "\n\n \n"):+ with pytest.raises(ExtraEtfFormatError, match="export is empty"):+ parse_export(empty)+++def test_as_of_comes_from_the_file_mtime_and_is_labelled(tmp_path):+ path = tmp_path / "holdings.csv"+ path.write_text(_holdings_export(_holding()), encoding="utf-8")+ stamp = 1_760_000_000+ os.utime(path, (stamp, stamp))++ export = read_export(path)++ expected = datetime.fromtimestamp(stamp, tz=timezone.utc).isoformat()+ assert export.as_of_source == "file_mtime"+ assert export.as_of == expected+ assert export.source_path == path+ assert export.require_positions()[0]["updated_at"] == expected+++def test_bare_text_reports_no_observation_time():+ export = parse_export(_holdings_export(_holding()))++ assert export.as_of is None+ assert export.as_of_source == "unavailable"+ assert export.require_positions()[0]["updated_at"] is None+++def test_an_absent_as_of_cannot_claim_an_origin():+ with pytest.raises(ValueError, match="no observation time"):+ parse_export(_holdings_export(_holding()), as_of_source="file_mtime")+++def test_bom_and_umlauts_decode_from_utf8_and_cp1252(tmp_path):+ document = _holdings_export(_holding())++ utf8_path = tmp_path / "utf8.csv"+ utf8_path.write_bytes(("\ufeff" + document).encode("utf-8"))+ assert read_export(utf8_path).require_positions()[0]["currency"] == "EUR"++ cp1252_path = tmp_path / "cp1252.csv"+ cp1252_path.write_bytes(document.encode("cp1252"))+ assert read_export(cp1252_path).require_positions()[0]["currency"] == "EUR"+++def test_missing_file_is_refused(tmp_path):+ with pytest.raises(ExtraEtfFormatError, match="cannot read export"):+ read_export(tmp_path / "absent.csv")+++def test_reader_imports_only_the_standard_library():+ """No network, no broker SDK, no order path can be reached from here."""+ import ast+ from pathlib import Path++ import src.portfolio.extraetf as module++ source = Path(module.__file__).read_text(encoding="utf-8")+ roots = set()+ for node in ast.walk(ast.parse(source)):+ if isinstance(node, ast.Import):+ roots.update(alias.name.split(".")[0] for alias in node.names)+ elif (+ isinstance(node, ast.ImportFrom)+ and node.level == 0+ and node.module+ and node.module != "__future__"+ ):+ roots.add(node.module.split(".")[0])++ assert roots == {+ "csv",+ "dataclasses",+ "datetime",+ "decimal",+ "io",+ "os",+ "pathlib",+ "re",+ "typing",+ }