Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(portfolio): add a read-only extraETF export reader

cgycorey79df088d0b2f
diff --git a/agent/src/portfolio/extraetf.py b/agent/src/portfolio/extraetf.py
new file mode 100644
index 000000000..88ad56490
--- /dev/null
+++ b/agent/src/portfolio/extraetf.py
@@ -0,0 +1,648 @@
+"""Read-only reader for extraETF portfolio exports.
+
+extraETF exports two semicolon-delimited, German-locale files:
+
+``holdings`` — the positions of one view at export time::
+
+ ISIN;Name;Typ;Anzahl;Kaufpreis;Aktueller Kurs;Aktueller Wert;Währung;Wechselkurs;Region;Land;Sektor;Portfolioname;Portfolio ID
+
+``transactions`` — the movement history of one view::
+
+ Datum;ISIN;Name;Typ;Transaktion;Anzahl;Preis;Gebühren;Steuern;Währung;Wechselkurs;Portfolioname;Portfolio ID;Stückzinsen
+
+Three properties are deliberate and load-bearing.
+
+**A transactions export can never be read as a portfolio.** Its rows are
+movements — ``Kauf``/``Verkauf``/``Dividende``/``Einbuchung``/``Ausbuchung`` —
+not holdings. An allocation computed from trades looks perfectly plausible and
+is wrong, so the two formats are told apart by their header and
+:meth:`ExtraEtfExport.require_positions` refuses anything that is not a
+holdings export. Movements carry an explicit ``movement_kind`` so a dividend or
+a transfer is never mistaken for a trade.
+
+**A holdings export carries no date.** The format has no snapshot column, so
+freshness can only come from the file's own modification time, and the export
+says so through ``as_of_source``. This reader never claims an observation time
+it did not read.
+
+**Validation fails closed.** A row this reader does not recognise raises
+:class:`ExtraEtfFormatError` instead of being skipped: an import computed from
+90% of a portfolio reads as entirely plausible, whereas a refused import is
+merely annoying. Numbers are parsed as German — ``1.386,608`` is 1386.608 — and
+a value that cannot be read unambiguously is refused rather than guessed.
+``1,234.56`` is US-formatted; reading it as German would be wrong by three
+orders of magnitude, so it is refused instead.
+
+Rows are read by column name, so a reordered export cannot mis-align a value
+into the wrong column. Rows are **not** merged or de-duplicated: two rows for
+the same instrument in one portfolio are returned as two positions, because
+summing them silently and dropping one silently are both wrong, and which one
+the user meant is not this module's call.
+
+The module writes nothing, holds no credentials, and touches no network. It
+only reads a local file the user exported themselves, and it has no path to any
+order surface.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import os
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from decimal import Decimal, InvalidOperation
+from pathlib import Path
+from typing import Any, Literal
+
+BROKER = "extraetf"
+
+_HOLDINGS_COLUMNS: tuple[str, ...] = (
+ "ISIN",
+ "Name",
+ "Typ",
+ "Anzahl",
+ "Kaufpreis",
+ "Aktueller Kurs",
+ "Aktueller Wert",
+ "Währung",
+ "Wechselkurs",
+ "Region",
+ "Land",
+ "Sektor",
+ "Portfolioname",
+ "Portfolio ID",
+)
+
+_TRANSACTIONS_COLUMNS: tuple[str, ...] = (
+ "Datum",
+ "ISIN",
+ "Name",
+ "Typ",
+ "Transaktion",
+ "Anzahl",
+ "Preis",
+ "Gebühren",
+ "Steuern",
+ "Währung",
+ "Wechselkurs",
+ "Portfolioname",
+ "Portfolio ID",
+ "Stückzinsen",
+)
+
+ExportKind = Literal["holdings", "transactions"]
+
+_ISIN_PATTERN = re.compile(r"^[A-Z]{2}[A-Z0-9]{9}[0-9]$")
+_CRYPTO_PAIR_PATTERN = re.compile(r"^[A-Z0-9]{2,12}_to_[A-Z0-9]{2,12}$", re.IGNORECASE)
+_CURRENCY_PATTERN = re.compile(r"^[A-Z]{3}$")
+_DATE_PATTERN = re.compile(r"^(\d{2})\.(\d{2})\.(\d{4})$")
+_NUMERAL_PATTERN = re.compile(r"^[0-9.,]+$")
+
+#: ``Typ`` is an instrument class, not a row shape: a class this reader does not
+#: know still describes a real position, so it is carried through with
+#: ``asset_type`` left ``None`` rather than refused or guessed at.
+_ASSET_TYPE_BY_EXPORT_TYPE = {
+ "etf": "etf",
+ "aktie": "stock",
+ "währung / krypto": "crypto",
+}
+
+_MOVEMENT_KIND_BY_TRANSACTION = {
+ "kauf": "buy",
+ "verkauf": "sell",
+ "dividende": "dividend",
+ "einbuchung": "transfer_in",
+ "ausbuchung": "transfer_out",
+}
+
+
+class ExtraEtfFormatError(ValueError):
+ """Raised when an export cannot be read without guessing."""
+
+
+@dataclass(frozen=True)
+class ExtraEtfExport:
+ """One parsed export, with its provenance attached.
+
+ Attributes:
+ kind: Which of the two extraETF formats this file is.
+ positions: Holdings rows. Always empty for a transactions export.
+ movements: Transactions rows. Always empty for a holdings export.
+ as_of: ISO-8601 observation time, or ``None`` when it could not be read.
+ as_of_source: Where ``as_of`` came from — ``"file_mtime"`` for a file
+ read from disk, ``"unavailable"`` for bare text with no timestamp.
+ A holdings export has no date column, so ``file_mtime`` is an
+ inference from the file's own modification time and is labelled as
+ one rather than presented as a recorded snapshot date.
+ source_path: The file this export was read from, when there was one.
+ """
+
+ kind: ExportKind
+ positions: tuple[dict[str, Any], ...]
+ movements: tuple[dict[str, Any], ...]
+ as_of: str | None
+ as_of_source: Literal["file_mtime", "unavailable"]
+ source_path: Path | None
+
+ def require_positions(self) -> tuple[dict[str, Any], ...]:
+ """Return holdings positions, refusing any other export outright.
+
+ Returns:
+ The parsed positions, possibly empty for an empty portfolio.
+
+ Raises:
+ ExtraEtfFormatError: If this export is a transactions export, whose
+ rows are movements rather than holdings.
+ """
+ if self.kind != "holdings":
+ raise ExtraEtfFormatError(
+ "this is a transactions export and holds movements, not positions; "
+ "importing it as a portfolio would build an allocation out of trades"
+ )
+ return self.positions
+
+
+def read_export(path: str | Path) -> ExtraEtfExport:
+ """Read and parse one extraETF export file.
+
+ The file's modification time becomes ``as_of`` with
+ ``as_of_source="file_mtime"``, because neither export format carries a
+ snapshot-observation column.
+
+ Args:
+ path: The exported ``.csv`` file.
+
+ Returns:
+ The parsed export.
+
+ Raises:
+ ExtraEtfFormatError: If the file cannot be read, decoded, or parsed.
+ """
+ resolved = Path(path)
+ try:
+ # Stat the same descriptor the bytes came from: a separate ``stat`` call
+ # can observe a later write than the read did, which would let the import
+ # claim a fresher observation time than the content it actually holds.
+ with resolved.open("rb") as handle:
+ data = handle.read()
+ modified = os.fstat(handle.fileno()).st_mtime
+ except OSError as exc:
+ raise ExtraEtfFormatError(f"cannot read export {resolved}: {exc}") from exc
+ as_of = datetime.fromtimestamp(modified, tz=timezone.utc).isoformat()
+ return parse_export(
+ data, source_path=resolved, as_of=as_of, as_of_source="file_mtime"
+ )
+
+
+def parse_export(
+ data: str | bytes,
+ *,
+ source_path: str | Path | None = None,
+ as_of: str | None = None,
+ as_of_source: Literal["file_mtime", "unavailable"] = "unavailable",
+) -> ExtraEtfExport:
+ """Parse one extraETF export from text or raw bytes.
+
+ Args:
+ data: The export contents, as text or as raw file bytes.
+ source_path: The originating file, recorded for provenance only.
+ as_of: ISO-8601 observation time, when the caller knows one.
+ as_of_source: Where ``as_of`` came from.
+
+ Returns:
+ The parsed export. An export with a header and no data rows is valid and
+ yields no positions — an empty portfolio is not an unreadable one.
+
+ Raises:
+ ExtraEtfFormatError: If the header is not one of the two known formats,
+ or any row cannot be read without guessing.
+ ValueError: If ``as_of_source`` claims an origin for an absent ``as_of``.
+ """
+ if as_of is None and as_of_source != "unavailable":
+ raise ValueError(
+ "as_of_source cannot claim where an observation time came from when "
+ "there is no observation time"
+ )
+ text = _decode(data)
+ kind, header, rows = _read_rows(text)
+ if kind == "holdings":
+ positions = tuple(
+ _parse_holding(header, cells, number, as_of=as_of) for number, cells in rows
+ )
+ movements: tuple[dict[str, Any], ...] = ()
+ else:
+ positions = ()
+ movements = tuple(
+ _parse_movement(header, cells, number) for number, cells in rows
+ )
+ return ExtraEtfExport(
+ kind=kind,
+ positions=positions,
+ movements=movements,
+ as_of=as_of,
+ as_of_source=as_of_source,
+ source_path=Path(source_path) if source_path is not None else None,
+ )
+
+
+def _decode(data: str | bytes) -> str:
+ """Decode export bytes, preferring UTF-8 and falling back to cp1252.
+
+ Args:
+ data: Export contents as text or bytes.
+
+ Returns:
+ The decoded text.
+
+ Raises:
+ ExtraEtfFormatError: If neither encoding decodes the bytes.
+ """
+ if isinstance(data, str):
+ return data
+ for encoding in ("utf-8-sig", "cp1252"):
+ try:
+ return data.decode(encoding)
+ except UnicodeDecodeError:
+ continue
+ raise ExtraEtfFormatError("export is decodable as neither UTF-8 nor cp1252")
+
+
+def _read_rows(
+ text: str,
+) -> tuple[ExportKind, tuple[str, ...], list[tuple[int, list[str]]]]:
+ """Split an export into its format, header and data rows.
+
+ Args:
+ text: The decoded export.
+
+ Returns:
+ The detected format, the header columns, and ``(line number, cells)``
+ pairs for every non-blank data row.
+
+ Raises:
+ ExtraEtfFormatError: If the export is empty, its header is not one of the
+ two known formats, or a row cannot be aligned to that header.
+ """
+ reader = csv.reader(io.StringIO(text), delimiter=";")
+ header: tuple[str, ...] | None = None
+ number = 0
+ for cells in reader:
+ number += 1
+ if not any(cell.strip() for cell in cells):
+ continue
+ header = tuple(cell.strip().lstrip("\ufeff") for cell in cells)
+ break
+ if header is None:
+ raise ExtraEtfFormatError("export is empty")
+ kind = _detect_kind(header)
+
+ rows: list[tuple[int, list[str]]] = []
+ for cells in reader:
+ number += 1
+ if not any(cell.strip() for cell in cells):
+ continue
+ if len(cells) != len(header):
+ raise ExtraEtfFormatError(
+ f"row {number}: expected {len(header)} fields to match the header, "
+ f"found {len(cells)}; a row that cannot be aligned is refused rather "
+ f"than skipped"
+ )
+ rows.append((number, [cell.strip() for cell in cells]))
+ return kind, header, rows
+
+
+def _detect_kind(header: tuple[str, ...]) -> ExportKind:
+ """Identify which extraETF format a header belongs to.
+
+ Args:
+ header: The export's column names.
+
+ Returns:
+ ``"holdings"`` or ``"transactions"``.
+
+ Raises:
+ ExtraEtfFormatError: If the header is neither known format exactly.
+ """
+ columns = set(header)
+ if len(header) == len(_HOLDINGS_COLUMNS) and columns == set(_HOLDINGS_COLUMNS):
+ return "holdings"
+ if len(header) == len(_TRANSACTIONS_COLUMNS) and columns == set(
+ _TRANSACTIONS_COLUMNS
+ ):
+ return "transactions"
+ transactions_like = "Transaktion" in columns
+ expected = _TRANSACTIONS_COLUMNS if transactions_like else _HOLDINGS_COLUMNS
+ missing = [column for column in expected if column not in columns]
+ unexpected = [column for column in header if column not in expected]
+ raise ExtraEtfFormatError(
+ "export header is not a recognised extraETF format; against the "
+ f"{'transactions' if transactions_like else 'holdings'} format, missing "
+ f"columns: {missing or 'none'}, unexpected columns: {unexpected or 'none'}"
+ )
+
+
+def _parse_holding(
+ header: tuple[str, ...], cells: list[str], number: int, *, as_of: str | None
+) -> dict[str, Any]:
+ """Convert one holdings row into a position.
+
+ Args:
+ header: The export's column names, which key the row's fields.
+ cells: The row's fields.
+ number: The row's line number, for error messages.
+ as_of: The export's observation time, mirrored onto the position.
+
+ Returns:
+ One position in the portfolio wire shape.
+
+ Raises:
+ ExtraEtfFormatError: If the row cannot be read without guessing.
+ """
+ row = dict(zip(header, cells, strict=True))
+ identifier, identifier_kind, checksum_ok = _describe_identifier(row["ISIN"])
+ if identifier_kind not in {"isin", "crypto_pair"}:
+ raise ExtraEtfFormatError(
+ f"row {number}: {row['ISIN']!r} is neither an ISIN nor an extraETF crypto "
+ f"pair such as BTC_to_EUR, so the holding cannot be attributed to an "
+ f"instrument"
+ )
+ name = _required(row, "Name", number)
+ instrument_type = _required(row, "Typ", number)
+ currency = _required(row, "Währung", number).upper()
+ if not _CURRENCY_PATTERN.fullmatch(currency):
+ raise ExtraEtfFormatError(
+ f"row {number}: Währung is not a three-letter currency code: "
+ f"{row['Währung']!r}"
+ )
+ portfolio_id = _required(row, "Portfolio ID", number)
+ quantity = _parse_decimal(row["Anzahl"], field="Anzahl", row_number=number)
+ if quantity is None:
+ raise ExtraEtfFormatError(f"row {number}: Anzahl is empty")
+ return {
+ "broker": BROKER,
+ "source": "extraetf_export",
+ "symbol": identifier,
+ "quote_symbol": identifier,
+ "instrument_id": identifier,
+ "instrument_id_kind": identifier_kind,
+ "instrument_id_checksum_ok": checksum_ok,
+ "name": name,
+ "instrument_type": instrument_type,
+ "asset_type": _ASSET_TYPE_BY_EXPORT_TYPE.get(instrument_type.strip().lower()),
+ "market": None,
+ "currency": currency,
+ "price_currency": currency,
+ "quantity": _number(quantity),
+ "cost_price": _optional_number(
+ row["Kaufpreis"], field="Kaufpreis", row_number=number
+ ),
+ "market_price": _optional_number(
+ row["Aktueller Kurs"], field="Aktueller Kurs", row_number=number
+ ),
+ "source_market_value": _optional_number(
+ row["Aktueller Wert"], field="Aktueller Wert", row_number=number
+ ),
+ "fx_rate": _optional_number(
+ row["Wechselkurs"], field="Wechselkurs", row_number=number
+ ),
+ "region": row["Region"] or None,
+ "country": row["Land"] or None,
+ "sector": row["Sektor"] or None,
+ "portfolio_id": portfolio_id,
+ "portfolio_name": row["Portfolioname"] or None,
+ "updated_at": as_of,
+ }
+
+
+def _parse_movement(
+ header: tuple[str, ...], cells: list[str], number: int
+) -> dict[str, Any]:
+ """Convert one transactions row into a movement.
+
+ Args:
+ header: The export's column names, which key the row's fields.
+ cells: The row's fields.
+ number: The row's line number, for error messages.
+
+ Returns:
+ One movement, carrying the export's own transaction label plus a
+ normalised ``movement_kind``.
+
+ Raises:
+ ExtraEtfFormatError: If the row cannot be read without guessing.
+ """
+ row = dict(zip(header, cells, strict=True))
+ movement = _required(row, "Transaktion", number)
+ transaction_date = _required(row, "Datum", number)
+ match = _DATE_PATTERN.fullmatch(transaction_date)
+ if match is None:
+ raise ExtraEtfFormatError(
+ f"row {number}: Datum is not a DD.MM.YYYY date: {transaction_date!r}"
+ )
+ day, month, year = (int(part) for part in match.groups())
+ try:
+ booked_on = datetime(year, month, day).date().isoformat()
+ except ValueError as exc:
+ raise ExtraEtfFormatError(
+ f"row {number}: Datum is not a real date: {transaction_date!r}"
+ ) from exc
+ currency = _required(row, "Währung", number).upper()
+ if not _CURRENCY_PATTERN.fullmatch(currency):
+ raise ExtraEtfFormatError(
+ f"row {number}: Währung is not a three-letter currency code: "
+ f"{row['Währung']!r}"
+ )
+ identifier, identifier_kind, checksum_ok = _describe_identifier(row["ISIN"])
+ quantity = _parse_decimal(row["Anzahl"], field="Anzahl", row_number=number)
+ if quantity is None:
+ raise ExtraEtfFormatError(f"row {number}: Anzahl is empty")
+ return {
+ "broker": BROKER,
+ "source": "extraetf_export",
+ "date": booked_on,
+ "instrument_id": identifier or None,
+ "instrument_id_kind": identifier_kind,
+ "instrument_id_checksum_ok": checksum_ok,
+ "name": _required(row, "Name", number),
+ "instrument_type": _required(row, "Typ", number),
+ "movement": movement,
+ "movement_kind": _MOVEMENT_KIND_BY_TRANSACTION.get(movement.strip().lower()),
+ "quantity": _number(quantity),
+ "price": _optional_number(row["Preis"], field="Preis", row_number=number),
+ "fees": _optional_number(row["Gebühren"], field="Gebühren", row_number=number),
+ "taxes": _optional_number(row["Steuern"], field="Steuern", row_number=number),
+ "currency": currency,
+ "fx_rate": _optional_number(
+ row["Wechselkurs"], field="Wechselkurs", row_number=number
+ ),
+ "accrued_interest": _optional_number(
+ row["Stückzinsen"], field="Stückzinsen", row_number=number
+ ),
+ "portfolio_id": _required(row, "Portfolio ID", number),
+ "portfolio_name": row["Portfolioname"] or None,
+ }
+
+
+def _required(row: dict[str, str], column: str, number: int) -> str:
+ """Return a column's value, refusing an empty one.
+
+ Args:
+ row: The aligned row.
+ column: The column to read.
+ number: The row's line number, for error messages.
+
+ Returns:
+ The non-empty value.
+
+ Raises:
+ ExtraEtfFormatError: If the value is empty.
+ """
+ value = row[column]
+ if not value:
+ raise ExtraEtfFormatError(f"row {number}: {column} is empty")
+ return value
+
+
+def _optional_number(raw: str, *, field: str, row_number: int) -> float | None:
+ """Parse an optional German-formatted number.
+
+ Args:
+ raw: The raw field text.
+ field: The column name, for error messages.
+ row_number: The row's line number, for error messages.
+
+ Returns:
+ The value, or ``None`` when the field is blank. A blank price is an
+ absent observation rather than a zero, and is never guessed at.
+
+ Raises:
+ ExtraEtfFormatError: If the field is present but unreadable.
+ """
+ value = _parse_decimal(raw, field=field, row_number=row_number)
+ return None if value is None else _number(value)
+
+
+def _parse_decimal(raw: str, *, field: str, row_number: int) -> Decimal | None:
+ """Parse a German-formatted number, refusing anything ambiguous.
+
+ A comma is the decimal separator and a period may only be a thousands
+ separator: ``1.386,608`` is 1386.608 and ``12,3456`` is 12.3456. A value
+ whose separators are US-ordered, or whose period grouping is not a plain
+ thousands separator, is refused rather than guessed at, because guessing
+ wrong changes a position's size by orders of magnitude.
+
+ Args:
+ raw: The raw field text.
+ field: The column name, for error messages.
+ row_number: The row's line number, for error messages.
+
+ Returns:
+ The parsed value, or ``None`` when the field is blank.
+
+ Raises:
+ ExtraEtfFormatError: If the field is unreadable or ambiguous.
+ """
+ text = raw.strip()
+ if not text:
+ return None
+ sign = ""
+ if text[0] in "+-":
+ sign, text = text[0], text[1:]
+ if not text or not _NUMERAL_PATTERN.fullmatch(text):
+ raise ExtraEtfFormatError(f"row {row_number}: {field} is not a number: {raw!r}")
+ whole, comma, fraction = text.rpartition(",")
+ if not comma:
+ whole, fraction = text, ""
+ elif "." in fraction:
+ raise ExtraEtfFormatError(
+ f"row {row_number}: {field} reads as a US-formatted number: {raw!r}; "
+ f"extraETF exports use German formatting such as 1.386,608"
+ )
+ groups = whole.split(".")
+ if (
+ any(not group.isdigit() for group in groups)
+ or any(len(group) != 3 for group in groups[1:])
+ or (comma and not fraction.isdigit())
+ ):
+ raise ExtraEtfFormatError(
+ f"row {row_number}: {field} has ambiguous separators: {raw!r}; expected "
+ f"German formatting such as 1.386,608"
+ )
+ digits = "".join(groups) + (f".{fraction}" if comma else "")
+ try:
+ return Decimal(f"{sign}{digits}")
+ except InvalidOperation as exc: # pragma: no cover - guarded by the checks above
+ raise ExtraEtfFormatError(
+ f"row {row_number}: {field} is not a number: {raw!r}"
+ ) from exc
+
+
+def _describe_identifier(raw: str) -> tuple[str, str, bool | None]:
+ """Describe an export identifier without refusing anything.
+
+ Args:
+ raw: The raw ``ISIN`` column value.
+
+ Returns:
+ The identifier as the export spelled it, its kind — ``"isin"``,
+ ``"crypto_pair"``, ``"raw_symbol"`` or ``"unknown"`` — and, for an
+ ISIN-shaped value, whether its check digit is valid. extraETF stores
+ crypto identifiers that are not ISINs, so the check digit is reported
+ rather than enforced: refusing a whole import over one unverifiable
+ identifier would block a user whose positions are otherwise perfectly
+ readable.
+
+ An ISIN is returned upper-cased, because an ISIN is canonical and
+ case-insensitive. A crypto pair such as ``BTC_to_EUR`` is returned
+ exactly as exported, because it is extraETF's own label and not a
+ canonical code — rewriting its case would invent an identity we do not
+ own.
+ """
+ identifier = raw.strip()
+ if not identifier:
+ return "", "unknown", None
+ canonical = identifier.upper()
+ if _ISIN_PATTERN.fullmatch(canonical):
+ return canonical, "isin", _isin_checksum_ok(canonical)
+ if _CRYPTO_PAIR_PATTERN.fullmatch(identifier):
+ return identifier, "crypto_pair", None
+ return identifier, "raw_symbol", None
+
+
+def _isin_checksum_ok(value: str) -> bool:
+ """Check an ISIN's check digit.
+
+ Args:
+ value: A value already matching :data:`_ISIN_PATTERN`.
+
+ Returns:
+ ``True`` when the Luhn-style check digit is valid.
+ """
+ digits = "".join(
+ str(ord(character) - 55) if character.isalpha() else character
+ for character in value
+ )
+ total = 0
+ for index, character in enumerate(reversed(digits)):
+ digit = int(character)
+ if index % 2:
+ digit *= 2
+ if digit > 9:
+ digit -= 9
+ total += digit
+ return total % 10 == 0
+
+
+def _number(value: Decimal) -> float:
+ """Return a wire-ready float, quantized as the connector path does.
+
+ Args:
+ value: The parsed value.
+
+ Returns:
+ The value as a float, quantized to eight decimal places to match
+ :func:`src.portfolio.normalization.normalize_position`.
+ """
+ return float(value.quantize(Decimal("0.00000001")))
diff --git a/agent/tests/test_extraetf_reader.py b/agent/tests/test_extraetf_reader.py
new file mode 100644
index 000000000..928962846
--- /dev/null
+++ b/agent/tests/test_extraetf_reader.py
@@ -0,0 +1,489 @@
+"""Tests for the read-only extraETF export reader.
+
+All fixtures are synthetic, shaped after the anonymized exports posted in
+HKUDS/Vibe-Trading#1170. No real portfolio data is used.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+
+import pytest
+
+from src.portfolio.extraetf import (
+ ExtraEtfFormatError,
+ parse_export,
+ read_export,
+)
+
+HOLDINGS_COLUMNS = [
+ "ISIN",
+ "Name",
+ "Typ",
+ "Anzahl",
+ "Kaufpreis",
+ "Aktueller Kurs",
+ "Aktueller Wert",
+ "Währung",
+ "Wechselkurs",
+ "Region",
+ "Land",
+ "Sektor",
+ "Portfolioname",
+ "Portfolio ID",
+]
+
+TRANSACTIONS_COLUMNS = [
+ "Datum",
+ "ISIN",
+ "Name",
+ "Typ",
+ "Transaktion",
+ "Anzahl",
+ "Preis",
+ "Gebühren",
+ "Steuern",
+ "Währung",
+ "Wechselkurs",
+ "Portfolioname",
+ "Portfolio ID",
+ "Stückzinsen",
+]
+
+#: A real, checksum-valid ISIN, used so the happy path is realistic.
+VALID_ISIN = "US0378331005"
+#: The anonymized export's placeholder ISIN, which is shaped like an ISIN but has
+#: an invalid check digit.
+PLACEHOLDER_ISIN = "IE00AAAA0000"
+
+
+def _holding(
+ *,
+ isin: str = VALID_ISIN,
+ name: str = "Example World ETF",
+ instrument_type: str = "ETF",
+ anzahl: str = "12,3456",
+ kaufpreis: str = "101,234567",
+ kurs: str = "112,345",
+ wert: str = "1.386,608",
+ waehrung: str = "EUR",
+ wechselkurs: str = "1,00",
+ region: str = "Welt",
+ land: str = "",
+ sektor: str = "",
+ portfolio: str = "Example broker",
+ portfolio_id: str = "1000001",
+) -> dict[str, str]:
+ return {
+ "ISIN": isin,
+ "Name": name,
+ "Typ": instrument_type,
+ "Anzahl": anzahl,
+ "Kaufpreis": kaufpreis,
+ "Aktueller Kurs": kurs,
+ "Aktueller Wert": wert,
+ "Währung": waehrung,
+ "Wechselkurs": wechselkurs,
+ "Region": region,
+ "Land": land,
+ "Sektor": sektor,
+ "Portfolioname": portfolio,
+ "Portfolio ID": portfolio_id,
+ }
+
+
+def _movement(
+ *,
+ datum: str = "01.09.2026",
+ isin: str = VALID_ISIN,
+ name: str = "Example World ETF",
+ instrument_type: str = "ETF",
+ transaktion: str = "Kauf",
+ anzahl: str = "1,2345",
+ preis: str = "110,123456",
+ gebuehren: str = "0,00",
+ steuern: str = "0,00",
+ waehrung: str = "EUR",
+ wechselkurs: str = "1,00",
+ portfolio: str = "Example broker",
+ portfolio_id: str = "1000001",
+ stueckzinsen: str = "0",
+) -> dict[str, str]:
+ return {
+ "Datum": datum,
+ "ISIN": isin,
+ "Name": name,
+ "Typ": instrument_type,
+ "Transaktion": transaktion,
+ "Anzahl": anzahl,
+ "Preis": preis,
+ "Gebühren": gebuehren,
+ "Steuern": steuern,
+ "Währung": waehrung,
+ "Wechselkurs": wechselkurs,
+ "Portfolioname": portfolio,
+ "Portfolio ID": portfolio_id,
+ "Stückzinsen": stueckzinsen,
+ }
+
+
+def _export(columns: list[str], rows: list[dict[str, str]]) -> str:
+ lines = [";".join(columns)]
+ lines.extend(";".join(row[column] for column in columns) for row in rows)
+ return "\n".join(lines) + "\n"
+
+
+def _holdings_export(*rows: dict[str, str]) -> str:
+ return _export(HOLDINGS_COLUMNS, list(rows))
+
+
+def _transactions_export(*rows: dict[str, str]) -> str:
+ return _export(TRANSACTIONS_COLUMNS, list(rows))
+
+
+def test_holdings_export_normalizes_a_german_locale_position():
+ export = parse_export(_holdings_export(_holding()))
+
+ assert export.kind == "holdings"
+ (position,) = export.require_positions()
+ assert position["instrument_id"] == VALID_ISIN
+ assert position["instrument_id_kind"] == "isin"
+ assert position["instrument_id_checksum_ok"] is True
+ assert position["symbol"] == VALID_ISIN
+ assert position["broker"] == "extraetf"
+ assert position["quantity"] == pytest.approx(12.3456)
+ assert position["cost_price"] == pytest.approx(101.234567)
+ assert position["market_price"] == pytest.approx(112.345)
+ assert position["source_market_value"] == pytest.approx(1386.608)
+ assert position["currency"] == "EUR"
+ assert position["price_currency"] == "EUR"
+ assert position["asset_type"] == "etf"
+ assert position["instrument_type"] == "ETF"
+ assert position["portfolio_id"] == "1000001"
+ assert position["portfolio_name"] == "Example broker"
+ assert position["region"] == "Welt"
+ assert position["country"] is None
+ assert position["sector"] is None
+
+
+def test_german_separators_are_not_read_as_us_numbers():
+ """``1.386,608`` is 1386.608, not 1.386 and not 1386608."""
+ export = parse_export(
+ _holdings_export(
+ _holding(
+ anzahl="12,3456",
+ kaufpreis="1.386,608",
+ kurs="59.999,99",
+ wert="1.386.608,42",
+ wechselkurs="1,0855",
+ )
+ )
+ )
+
+ (position,) = export.require_positions()
+ assert position["quantity"] == pytest.approx(12.3456)
+ assert position["cost_price"] == pytest.approx(1386.608)
+ assert position["market_price"] == pytest.approx(59999.99)
+ assert position["source_market_value"] == pytest.approx(1386608.42)
+ assert position["fx_rate"] == pytest.approx(1.0855)
+
+
+def test_us_formatted_number_is_refused_rather_than_misread():
+ with pytest.raises(ExtraEtfFormatError, match="US-formatted"):
+ parse_export(_holdings_export(_holding(kaufpreis="1,234.56")))
+
+
[email protected]("ambiguous", ["1.23", "1234.56", "0.5"])
+def test_ambiguous_separator_grouping_is_refused(ambiguous):
+ """A period that is not a thousands separator cannot be guessed at."""
+ with pytest.raises(ExtraEtfFormatError, match="ambiguous separators"):
+ parse_export(_holdings_export(_holding(kurs=ambiguous)))
+
+
[email protected]("unreadable", ["zwei", "10,00 EUR", "--", ","])
+def test_unreadable_number_is_refused(unreadable):
+ with pytest.raises(ExtraEtfFormatError, match="is not a number|ambiguous|empty"):
+ parse_export(_holdings_export(_holding(anzahl=unreadable)))
+
+
+def test_blank_optional_fields_are_none_rather_than_zero():
+ export = parse_export(
+ _holdings_export(_holding(kaufpreis="", kurs="", wert="", wechselkurs=""))
+ )
+
+ (position,) = export.require_positions()
+ assert position["cost_price"] is None
+ assert position["market_price"] is None
+ assert position["source_market_value"] is None
+ assert position["fx_rate"] is None
+
+
+def test_transactions_export_yields_movements_and_never_positions():
+ """A transactions export is a movement history, not a portfolio."""
+ export = parse_export(
+ _transactions_export(
+ _movement(transaktion="Kauf"),
+ _movement(transaktion="Verkauf", anzahl="2,5000", preis="19,876543"),
+ _movement(
+ transaktion="Dividende",
+ anzahl="0,5000",
+ preis="2,345",
+ steuern="0,45",
+ ),
+ _movement(transaktion="Einbuchung", anzahl="5,0000", preis="95,000000"),
+ _movement(transaktion="Ausbuchung", anzahl="0,5000", preis="90,000000"),
+ )
+ )
+
+ assert export.kind == "transactions"
+ assert export.positions == ()
+ assert export.movements[0]["movement"] == "Kauf"
+ assert export.movements[0]["movement_kind"] == "buy"
+ assert [m["movement_kind"] for m in export.movements] == [
+ "buy",
+ "sell",
+ "dividend",
+ "transfer_in",
+ "transfer_out",
+ ]
+ # A dividend row carries a quantity of its own; it must never become a holding.
+ assert export.movements[2]["quantity"] == pytest.approx(0.5)
+ assert export.movements[2]["taxes"] == pytest.approx(0.45)
+ assert all("asset_type" not in movement for movement in export.movements)
+ assert all("accrued_interest" in movement for movement in export.movements)
+ assert export.movements[0]["date"] == "2026-09-01"
+ assert export.movements[0]["accrued_interest"] == pytest.approx(0.0)
+
+
+def test_a_transactions_export_refuses_to_pose_as_a_portfolio():
+ export = parse_export(_transactions_export(_movement(transaktion="Kauf")))
+
+ with pytest.raises(ExtraEtfFormatError, match="movements, not positions"):
+ export.require_positions()
+
+
+def test_unknown_transaction_type_is_carried_through_not_invented():
+ export = parse_export(_transactions_export(_movement(transaktion="Steuer")))
+
+ assert export.positions == ()
+ assert export.movements[0]["movement"] == "Steuer"
+ assert export.movements[0]["movement_kind"] is None
+
+
+def test_invalid_transaction_date_is_refused():
+ with pytest.raises(ExtraEtfFormatError, match="DD.MM.YYYY"):
+ parse_export(_transactions_export(_movement(datum="2026-09-01")))
+
+ with pytest.raises(ExtraEtfFormatError, match="not a real date"):
+ parse_export(_transactions_export(_movement(datum="31.02.2026")))
+
+
+def test_crypto_identity_is_a_pair_not_an_isin():
+ export = parse_export(
+ _holdings_export(
+ _holding(
+ isin="BTC_to_EUR",
+ name="Bitcoin",
+ instrument_type="Währung / Krypto",
+ region="Global",
+ sektor="Diversifizierte Kapitalmärkte",
+ portfolio="Example crypto wallet",
+ portfolio_id="1000002",
+ )
+ )
+ )
+
+ (position,) = export.require_positions()
+ assert position["instrument_id"] == "BTC_to_EUR"
+ assert position["instrument_id_kind"] == "crypto_pair"
+ assert position["instrument_id_checksum_ok"] is None
+ assert position["asset_type"] == "crypto"
+ assert position["portfolio_id"] == "1000002"
+
+
[email protected]("identifier", ["BTC", "", " ", "NOT*AN*ID"])
+def test_unattributable_holding_identifier_is_refused(identifier):
+ """A bare symbol cannot be resolved, so the import fails instead of guessing."""
+ with pytest.raises(ExtraEtfFormatError, match="neither an ISIN nor"):
+ parse_export(_holdings_export(_holding(isin=identifier)))
+
+
+def test_invalid_isin_check_digit_is_reported_not_enforced():
+ export = parse_export(
+ _holdings_export(_holding(isin=VALID_ISIN), _holding(isin=PLACEHOLDER_ISIN))
+ )
+
+ valid, placeholder = export.require_positions()
+ assert valid["instrument_id_checksum_ok"] is True
+ assert placeholder["instrument_id_checksum_ok"] is False
+
+
+def test_unknown_instrument_type_leaves_asset_type_unset():
+ export = parse_export(_holdings_export(_holding(instrument_type="Anleihe")))
+
+ (position,) = export.require_positions()
+ assert position["instrument_type"] == "Anleihe"
+ assert position["asset_type"] is None
+
+
+def test_missing_required_field_is_refused():
+ for kwarg, message in (
+ ("name", "Name is empty"),
+ ("instrument_type", "Typ is empty"),
+ ("waehrung", "Währung is empty"),
+ ("portfolio_id", "Portfolio ID is empty"),
+ ):
+ with pytest.raises(ExtraEtfFormatError, match=message):
+ parse_export(_holdings_export(_holding(**{kwarg: ""})))
+
+
+def test_invalid_currency_is_refused():
+ with pytest.raises(ExtraEtfFormatError, match="three-letter currency"):
+ parse_export(_holdings_export(_holding(waehrung="EURO")))
+
+
+def test_unknown_column_makes_the_export_unrecognisable():
+ row = _holding()
+ header = ";".join([*HOLDINGS_COLUMNS, "Neue Spalte"])
+ body = ";".join([*(row[column] for column in HOLDINGS_COLUMNS), "x"])
+
+ with pytest.raises(
+ ExtraEtfFormatError, match="unexpected columns: \\['Neue Spalte'\\]"
+ ):
+ parse_export(f"{header}\n{body}\n")
+
+
+def test_missing_column_makes_the_export_unrecognisable():
+ columns = [column for column in HOLDINGS_COLUMNS if column != "Anzahl"]
+ row = _holding()
+
+ with pytest.raises(ExtraEtfFormatError, match="missing columns: \\['Anzahl'\\]"):
+ parse_export(_export(columns, [row]))
+
+
+def test_ragged_row_is_refused_rather_than_partially_imported():
+ """One malformed row fails the import; it never yields a short portfolio."""
+ row = _holding()
+ short = ";".join(row[column] for column in HOLDINGS_COLUMNS[:-1])
+ long = ";".join([*(row[column] for column in HOLDINGS_COLUMNS), "extra"])
+
+ for ragged in (short, long):
+ with pytest.raises(ExtraEtfFormatError, match="cannot be aligned"):
+ parse_export(_holdings_export(_holding(), _holding()) + ragged + "\n")
+
+
+def test_reordered_columns_do_not_mis_align_values():
+ """Rows are keyed by column name, so a reordered export stays correct."""
+ columns = list(reversed(HOLDINGS_COLUMNS))
+
+ export = parse_export(_export(columns, [_holding()]))
+
+ (position,) = export.require_positions()
+ assert position["currency"] == "EUR"
+ assert position["quantity"] == pytest.approx(12.3456)
+ assert position["name"] == "Example World ETF"
+ assert position["portfolio_id"] == "1000001"
+
+
+def test_duplicate_rows_are_preserved_rather_than_merged():
+ export = parse_export(
+ _holdings_export(_holding(anzahl="1,0000"), _holding(anzahl="2,0000"))
+ )
+
+ assert [position["quantity"] for position in export.require_positions()] == [
+ pytest.approx(1.0),
+ pytest.approx(2.0),
+ ]
+
+
+def test_empty_holdings_export_is_valid_and_not_an_error():
+ """An empty portfolio and an unreadable file are different states."""
+ export = parse_export(_holdings_export())
+
+ assert export.kind == "holdings"
+ assert export.require_positions() == ()
+
+
+def test_empty_file_is_refused():
+ for empty in ("", "\n\n \n"):
+ with pytest.raises(ExtraEtfFormatError, match="export is empty"):
+ parse_export(empty)
+
+
+def test_as_of_comes_from_the_file_mtime_and_is_labelled(tmp_path):
+ path = tmp_path / "holdings.csv"
+ path.write_text(_holdings_export(_holding()), encoding="utf-8")
+ stamp = 1_760_000_000
+ os.utime(path, (stamp, stamp))
+
+ export = read_export(path)
+
+ expected = datetime.fromtimestamp(stamp, tz=timezone.utc).isoformat()
+ assert export.as_of_source == "file_mtime"
+ assert export.as_of == expected
+ assert export.source_path == path
+ assert export.require_positions()[0]["updated_at"] == expected
+
+
+def test_bare_text_reports_no_observation_time():
+ export = parse_export(_holdings_export(_holding()))
+
+ assert export.as_of is None
+ assert export.as_of_source == "unavailable"
+ assert export.require_positions()[0]["updated_at"] is None
+
+
+def test_an_absent_as_of_cannot_claim_an_origin():
+ with pytest.raises(ValueError, match="no observation time"):
+ parse_export(_holdings_export(_holding()), as_of_source="file_mtime")
+
+
+def test_bom_and_umlauts_decode_from_utf8_and_cp1252(tmp_path):
+ document = _holdings_export(_holding())
+
+ utf8_path = tmp_path / "utf8.csv"
+ utf8_path.write_bytes(("\ufeff" + document).encode("utf-8"))
+ assert read_export(utf8_path).require_positions()[0]["currency"] == "EUR"
+
+ cp1252_path = tmp_path / "cp1252.csv"
+ cp1252_path.write_bytes(document.encode("cp1252"))
+ assert read_export(cp1252_path).require_positions()[0]["currency"] == "EUR"
+
+
+def test_missing_file_is_refused(tmp_path):
+ with pytest.raises(ExtraEtfFormatError, match="cannot read export"):
+ read_export(tmp_path / "absent.csv")
+
+
+def test_reader_imports_only_the_standard_library():
+ """No network, no broker SDK, no order path can be reached from here."""
+ import ast
+ from pathlib import Path
+
+ import src.portfolio.extraetf as module
+
+ source = Path(module.__file__).read_text(encoding="utf-8")
+ roots = set()
+ for node in ast.walk(ast.parse(source)):
+ if isinstance(node, ast.Import):
+ roots.update(alias.name.split(".")[0] for alias in node.names)
+ elif (
+ isinstance(node, ast.ImportFrom)
+ and node.level == 0
+ and node.module
+ and node.module != "__future__"
+ ):
+ roots.add(node.module.split(".")[0])
+
+ assert roots == {
+ "csv",
+ "dataclasses",
+ "datetime",
+ "decimal",
+ "io",
+ "os",
+ "pathlib",
+ "re",
+ "typing",
+ }