Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

Create a standalone script that populates the database with realistic stress-test data for testing and performance benchmarking (#196)

Geovane Pereira001df4106533
diff --git a/backend/scripts/README.md b/backend/scripts/README.md
new file mode 100644
index 00000000..78963521
--- /dev/null
+++ b/backend/scripts/README.md
@@ -0,0 +1,95 @@
+# seed_perf.py
+
+Populates the database with realistic data for performance benchmarking and manual testing. Must be run inside the backend container since it imports app modules directly.
+
+## Prerequisites
+
+The stack must be running:
+
+```bash
+docker compose up -d
+```
+
+## Basic usage
+
+```bash
+# Full default seed (2024-01-01 through today)
+docker compose exec backend python scripts/seed_perf.py
+
+# Quick smoke run — 10% volume, same structure
+docker compose exec backend python scripts/seed_perf.py --scale 0.1
+```
+
+## What gets seeded
+
+At default settings the script creates the following data for one user:
+
+| Entity | Default count | Controlled by |
+|---|---|---|
+| Accounts | 5 | `--accounts` |
+| Categories | 30 (22 debit + 8 income) | `--categories` |
+| Assets | 15 | `--assets` |
+| Asset values | one per asset per day in range | `--start-date` |
+| Transactions | 100,000 | `--scale` |
+| Recurring transactions | 20 | `--scale` |
+| FX rates (EUR + BRL) | one per day in range | `--start-date` |
+
+Transactions are randomly distributed across the date range: 75% debit, 25% credit, with lognormal amounts.
+
+## Options
+
+| Flag | Default | Description |
+|---|---|---|
+| `--email` | `[email protected]` | Email of the seed user |
+| `--password` | `Securo123!` | Password of the seed user |
+| `--start-date` | `2024-01-01` | Earliest date for all seeded data (format: `YYYY-MM-DD`) |
+| `--scale` | `1.0` | Multiplier for transaction and recurring transaction counts |
+| `--accounts` | `5` | Number of accounts (max 5) |
+| `--categories` | `30` | Number of categories (max 30) |
+| `--assets` | `15` | Number of assets (max 15) |
+| `--no-reset` | _(unset)_ | Skip wiping existing data for the seed user before re-seeding |
+
+### `--scale` in detail
+
+`--scale` is a simple multiplier applied only to counts that can grow arbitrarily:
+
+- Transactions: `int(100_000 × scale)`
+- Recurring transactions: `int(20 × scale)`
+
+It does **not** affect the number of accounts, categories, assets, or the date range. Use `--scale 0.1` for a fast smoke run and `--scale 2.0` to stress-test with 200k transactions.
+
+### `--start-date` in detail
+
+All time-series data (transactions, asset values, FX rates) spans from `--start-date` to the day the script runs. The default is `2024-01-01`. Use an earlier date to generate longer histories:
+
+```bash
+# 5 years of history
+docker compose exec backend python scripts/seed_perf.py --start-date 2020-01-01
+```
+
+### `--no-reset`
+
+By default the script wipes all existing data for the seed user before inserting. Pass `--no-reset` to skip the wipe — if the user already has transactions the script exits immediately without inserting anything.
+
+## Examples
+
+```bash
+# Minimal dataset for a quick UI check
+docker compose exec backend python scripts/seed_perf.py \
+ --scale 0.1 --accounts 2 --categories 5 --assets 3
+
+# Large dataset for performance profiling
+docker compose exec backend python scripts/seed_perf.py \
+ --scale 2.0 --start-date 2019-01-01
+
+# Seed a second user without touching the default one
+docker compose exec backend python scripts/seed_perf.py \
+ --email [email protected] --password Securo123!
+
+# Re-seed only transactions (skip wipe, then add more — only works on a fresh user)
+docker compose exec backend python scripts/seed_perf.py --no-reset
+```
+
+## Reproducibility
+
+The script uses a fixed RNG seed (`random.Random(42)`), so the same arguments always produce the same data distribution. The only variable is `--start-date` relative to the current date, which determines the date range.
diff --git a/backend/scripts/seed_perf.py b/backend/scripts/seed_perf.py
new file mode 100644
index 00000000..5b1bc57a
--- /dev/null
+++ b/backend/scripts/seed_perf.py
@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+"""Seed stress-test data for performance benchmarking.
+
+Must run inside the backend container (needs app module access):
+ docker compose exec backend python scripts/seed_perf.py
+
+Options:
+ --scale FLOAT Data volume multiplier (default: 1.0; use 0.1 for a quick smoke run)
+ --email TEXT Seed user email (default: [email protected])
+ --password TEXT Seed user password (default: Securo123!)
+ --no-reset Skip wiping existing data (default: wipe and re-seed)
+ --start-date YYYY-MM-DD Earliest date for seeded data (default: 2024-01-01)
+
+At scale=1.0 seeds:
+ 5 accounts · 30 categories · 100 000 transactions · 15 assets w/ daily values
+ FX rates for EUR + BRL · 20 recurring transactions
+ (date range: --start-date through today)
+"""
+import argparse
+import asyncio
+import os
+import random
+import sys
+import uuid
+from datetime import date, timedelta
+from decimal import Decimal
+
+# Allow running from repo root or backend root
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from sqlalchemy import delete, select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from app.core.auth import UserManager
+from app.core.database import async_session_maker
+from app.models.account import Account
+from app.models.asset import Asset
+from app.models.asset_value import AssetValue
+from app.models.category import Category
+from app.models.fx_rate import FxRate
+from app.models.recurring_transaction import RecurringTransaction
+from app.models.transaction import Transaction
+from app.models.user import User
+from app.schemas.user import UserCreate
+from fastapi_users.db import SQLAlchemyUserDatabase
+from fastapi_users.exceptions import UserAlreadyExists
+
+# ---------------------------------------------------------------------------
+# Seed templates
+# ---------------------------------------------------------------------------
+
+ACCOUNT_TEMPLATES = [
+ {"name": "Main Checking", "type": "checking", "currency": "USD", "balance": Decimal("5000.00")},
+ {"name": "Savings", "type": "savings", "currency": "USD", "balance": Decimal("25000.00")},
+ {"name": "Visa Credit", "type": "credit_card", "currency": "USD", "balance": Decimal("0.00"), "credit_limit": Decimal("10000.00")},
+ {"name": "Euro Account", "type": "checking", "currency": "EUR", "balance": Decimal("3000.00")},
+ {"name": "BRL Account", "type": "checking", "currency": "BRL", "balance": Decimal("15000.00")},
+]
+
+# (name, color, icon) — first 22 are debit, last 8 are credit
+CATEGORY_TEMPLATES = [
+ ("Groceries", "#22c55e", "shopping-cart"),
+ ("Rent", "#ef4444", "home"),
+ ("Utilities", "#f97316", "zap"),
+ ("Transport", "#3b82f6", "car"),
+ ("Dining Out", "#ec4899", "utensils"),
+ ("Entertainment", "#8b5cf6", "film"),
+ ("Healthcare", "#06b6d4", "heart-pulse"),
+ ("Insurance", "#64748b", "shield"),
+ ("Education", "#f59e0b", "book"),
+ ("Clothing", "#84cc16", "shirt"),
+ ("Travel", "#0ea5e9", "plane"),
+ ("Electronics", "#6366f1", "smartphone"),
+ ("Sports", "#10b981", "dumbbell"),
+ ("Personal Care", "#f43f5e", "sparkles"),
+ ("Subscriptions", "#a855f7", "repeat"),
+ ("Home Improvement", "#d97706", "wrench"),
+ ("Gifts", "#e11d48", "gift"),
+ ("Pets", "#65a30d", "paw-print"),
+ ("Kids", "#0284c7", "baby"),
+ ("Taxes", "#dc2626", "landmark"),
+ ("Fees", "#475569", "circle-dollar-sign"),
+ ("Miscellaneous", "#6b7280", "circle-help"),
+ # Credit / income
+ ("Salary", "#16a34a", "briefcase"),
+ ("Freelance", "#15803d", "laptop"),
+ ("Investment Income", "#166534", "trending-up"),
+ ("Rental Income", "#14532d", "building"),
+ ("Bonus", "#065f46", "star"),
+ ("Refunds", "#0f766e", "undo"),
+ ("Other Income", "#1e40af", "plus"),
+ ("Gifts Received", "#0369a1", "gift"),
+]
+N_DEBIT_CATS = 22 # first N_DEBIT_CATS entries are debit categories
+
+PAYEES = [
+ "Amazon", "Walmart", "Target", "Costco", "Whole Foods", "Trader Joe's",
+ "Netflix", "Spotify", "Apple", "Google", "Microsoft", "Adobe",
+ "Shell", "BP", "Uber", "Lyft", "Delta Airlines", "United Airlines",
+ "Marriott", "Hilton", "Home Depot", "IKEA", "CVS", "Walgreens",
+ "Doctor Smith", "City Hospital", "LA Fitness", "Planet Fitness",
+ "PG&E", "Comcast", "AT&T", "Verizon", "State Farm", "Geico",
+]
+
+ASSET_TEMPLATES = [
+ {"name": "Primary Residence", "type": "real_estate", "currency": "USD", "purchase": Decimal("450000"), "base": Decimal("520000"), "daily_pct": Decimal("0.00110")},
+ {"name": "Investment Property", "type": "real_estate", "currency": "USD", "purchase": Decimal("280000"), "base": Decimal("310000"), "daily_pct": Decimal("0.00082")},
+ {"name": "Tesla Model 3", "type": "vehicle", "currency": "USD", "purchase": Decimal("45000"), "base": Decimal("28000"), "daily_pct": Decimal("-0.00219")},
+ {"name": "Rolex Watch", "type": "valuable", "currency": "USD", "purchase": Decimal("8000"), "base": Decimal("9500"), "daily_pct": Decimal("0.00055")},
+ {"name": "S&P 500 Index", "type": "investment", "currency": "USD", "purchase": Decimal("50000"), "base": Decimal("75000"), "daily_pct": Decimal("0.00219")},
+ {"name": "Tech Stock Portfolio","type": "investment", "currency": "USD", "purchase": Decimal("30000"), "base": Decimal("42000"), "daily_pct": Decimal("0.00274")},
+ {"name": "Bond Portfolio", "type": "investment", "currency": "USD", "purchase": Decimal("20000"), "base": Decimal("22000"), "daily_pct": Decimal("0.00082")},
+ {"name": "Crypto Portfolio", "type": "investment", "currency": "USD", "purchase": Decimal("10000"), "base": Decimal("15000"), "daily_pct": Decimal("0.00548")},
+ {"name": "Vacation Cabin", "type": "real_estate", "currency": "USD", "purchase": Decimal("180000"), "base": Decimal("210000"), "daily_pct": Decimal("0.00137")},
+ {"name": "BMW 5 Series", "type": "vehicle", "currency": "USD", "purchase": Decimal("55000"), "base": Decimal("35000"), "daily_pct": Decimal("-0.00247")},
+ {"name": "Art Collection", "type": "valuable", "currency": "USD", "purchase": Decimal("15000"), "base": Decimal("18000"), "daily_pct": Decimal("0.00082")},
+ {"name": "Gold Bars", "type": "valuable", "currency": "USD", "purchase": Decimal("25000"), "base": Decimal("30000"), "daily_pct": Decimal("0.00110")},
+ {"name": "EU Real Estate", "type": "real_estate", "currency": "EUR", "purchase": Decimal("300000"), "base": Decimal("340000"), "daily_pct": Decimal("0.00110")},
+ {"name": "Emerging Markets ETF","type": "investment", "currency": "USD", "purchase": Decimal("15000"), "base": Decimal("18000"), "daily_pct": Decimal("0.00164")},
+ {"name": "Private Equity Fund", "type": "investment", "currency": "USD", "purchase": Decimal("100000"), "base": Decimal("125000"), "daily_pct": Decimal("0.00192")},
+]
+
+RECURRING_DESCRIPTIONS = [
+ "Netflix", "Spotify", "Rent Payment", "Gym Membership", "Car Insurance",
+ "Phone Bill", "Internet", "Electric Bill", "Water Bill", "Gas Bill",
+ "Amazon Prime", "iCloud Storage", "LinkedIn Premium", "YouTube Premium", "Adobe Creative Cloud",
+ "Mortgage", "Student Loan", "Car Payment", "Health Insurance", "Life Insurance",
+]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _make_date_range(days: int) -> list[date]:
+ today = date.today()
+ return [today - timedelta(days=d) for d in range(days, -1, -1)]
+
+
+async def _wipe_user_data(session, user_id: uuid.UUID) -> None:
+ await session.execute(delete(RecurringTransaction).where(RecurringTransaction.user_id == user_id))
+ await session.execute(
+ delete(AssetValue).where(
+ AssetValue.asset_id.in_(select(Asset.id).where(Asset.user_id == user_id))
+ )
+ )
+ await session.execute(delete(Asset).where(Asset.user_id == user_id))
+ await session.execute(delete(Transaction).where(Transaction.user_id == user_id))
+ await session.execute(delete(Category).where(Category.user_id == user_id))
+ await session.execute(delete(Account).where(Account.user_id == user_id))
+ await session.commit()
+
+
+# ---------------------------------------------------------------------------
+# Main seeder
+# ---------------------------------------------------------------------------
+
+async def seed(
+ email: str,
+ password: str,
+ scale: float,
+ reset: bool,
+ start_date: date,
+ n_accounts: int,
+ n_categories: int,
+ n_assets: int,
+) -> None:
+ rng = random.Random(42)
+ today = date.today()
+
+ async with async_session_maker() as session:
+
+ # ── 1. User ──────────────────────────────────────────────────────────
+ user = await session.scalar(select(User).where(User.email == email))
+
+ if user and not reset:
+ tx_count = await session.scalar(
+ select(Transaction).where(Transaction.user_id == user.id).limit(1)
+ )
+ if tx_count is not None:
+ print(f"User {email} already has data and --no-reset is set. Nothing to do.")
+ return
+ elif user and reset:
+ print(f"Wiping existing data for {email} …")
+ await _wipe_user_data(session, user.id)
+
+ if not user:
+ print(f"Creating user {email} …")
+ user_db = SQLAlchemyUserDatabase(session, User)
+ manager = UserManager(user_db)
+ try:
+ user = await manager.create(UserCreate(email=email, password=password))
+ except UserAlreadyExists:
+ user = await session.scalar(select(User).where(User.email == email))
+ await session.commit()
+ print(f" Created {user.id}")
+ else:
+ print(f"Using existing user {user.id}")
+
+ uid = user.id
+
+ # ── 2. Accounts ──────────────────────────────────────────────────────
+ print("Creating accounts …")
+ accounts: list[Account] = []
+ for tmpl in ACCOUNT_TEMPLATES[:n_accounts]:
+ acc = Account(
+ user_id=uid,
+ name=tmpl["name"],
+ type=tmpl["type"],
+ currency=tmpl["currency"],
+ balance=tmpl["balance"],
+ credit_limit=tmpl.get("credit_limit"),
+ )
+ session.add(acc)
+ accounts.append(acc)
+ await session.flush()
+ print(f" {len(accounts)} accounts")
+
+ # ── 3. Categories ────────────────────────────────────────────────────
+ print("Creating categories …")
+ categories: list[Category] = []
+ for name, color, icon in CATEGORY_TEMPLATES[:n_categories]:
+ cat = Category(user_id=uid, name=name, color=color, icon=icon)
+ session.add(cat)
+ categories.append(cat)
+ await session.flush()
+ debit_cats = categories[:N_DEBIT_CATS]
+ credit_cats = categories[N_DEBIT_CATS:] or debit_cats # fallback if slice has no credit cats
+ print(f" {len(categories)} categories")
+
+ # ── 4. FX Rates ──────────────────────────────────────────────────────
+ n_days = max(1, (today - start_date).days)
+ print(f"Seeding FX rates for {n_days} days ({start_date} → {today}) …")
+ all_dates = _make_date_range(n_days)
+
+ # rates are USD-quoted: how many of quote_currency per 1 USD
+ eur_rate = Decimal("0.9200") # ~0.92 EUR per USD
+ brl_rate = Decimal("5.0000") # ~5.0 BRL per USD
+ fx_rows = []
+ for d in all_dates:
+ eur_rate = max(
+ Decimal("0.75"),
+ min(Decimal("1.05"), eur_rate + Decimal(str(round(rng.gauss(0, 0.002), 6)))),
+ )
+ brl_rate = max(
+ Decimal("4.00"),
+ min(Decimal("6.50"), brl_rate + Decimal(str(round(rng.gauss(0, 0.02), 6)))),
+ )
+ fx_rows.append({"base_currency": "USD", "quote_currency": "EUR", "date": d,
+ "rate": eur_rate.quantize(Decimal("0.000001")), "source": "seed"})
+ fx_rows.append({"base_currency": "USD", "quote_currency": "BRL", "date": d,
+ "rate": brl_rate.quantize(Decimal("0.000001")), "source": "seed"})
+
+ for i in range(0, len(fx_rows), 2000):
+ stmt = pg_insert(FxRate).values(fx_rows[i : i + 2000])
+ stmt = stmt.on_conflict_do_update(
+ constraint="uq_fx_rate_base_quote_date",
+ set_={"rate": stmt.excluded.rate, "source": stmt.excluded.source},
+ )
+ await session.execute(stmt)
+ await session.commit()
+ print(f" {len(fx_rows)} FX rate rows")
+
+ # ── 5. Transactions ──────────────────────────────────────────────────
+ n_tx = max(1, int(100_000 * scale))
+ print(f"Inserting {n_tx:,} transactions …")
+ tx_rows = []
+
+ for _ in range(n_tx):
+ acc = rng.choice(accounts)
+ is_debit = rng.random() < 0.75
+ cat = rng.choice(debit_cats if is_debit else credit_cats)
+ tx_date = start_date + timedelta(days=rng.randint(0, n_days))
+ if is_debit:
+ amount = Decimal(str(round(abs(rng.lognormvariate(3.5, 1.2)), 2)))
+ amount = min(amount, Decimal("5000.00"))
+ else:
+ amount = Decimal(str(round(abs(rng.lognormvariate(6.5, 0.8)), 2)))
+
+ tx_rows.append({
+ "id": uuid.uuid4(),
+ "user_id": uid,
+ "account_id": acc.id,
+ "category_id": cat.id,
+ "description": rng.choice(PAYEES),
+ "amount": amount,
+ "currency": acc.currency,
+ "date": tx_date,
+ "effective_date": tx_date,
+ "type": "debit" if is_debit else "credit",
+ "source": "manual",
+ "status": "posted",
+ "payee": rng.choice(PAYEES),
+ })
+
+ chunk = 2000 # asyncpg limit: 32767 params / 14 cols per tx = 2340 max
+ for i in range(0, len(tx_rows), chunk):
+ await session.execute(pg_insert(Transaction).values(tx_rows[i : i + chunk]))
+ done = min(i + chunk, n_tx)
+ print(f" {done:,}/{n_tx:,}", end="\r", flush=True)
+ await session.commit()
+ print(f" {n_tx:,} transactions done ")
+
+ # ── 6. Assets + AssetValues ──────────────────────────────────────────
+ print(f"Creating {len(ASSET_TEMPLATES[:n_assets])} assets with {n_days + 1} daily values each …")
+ asset_days = _make_date_range(n_days)
+
+ for tmpl in ASSET_TEMPLATES[:n_assets]:
+ asset = Asset(
+ user_id=uid,
+ name=tmpl["name"],
+ type=tmpl["type"],
+ currency=tmpl["currency"],
+ purchase_price=tmpl["purchase"],
+ purchase_date=asset_days[0],
+ valuation_method="manual",
+ )
+ session.add(asset)
+ await session.flush()
+
+ daily_pct = tmpl["daily_pct"]
+ value_rows = []
+ # Walk backwards so oldest value starts at base and grows to today
+ value_sequence = []
+ v = tmpl["base"]
+ for _ in asset_days:
+ noise = Decimal(str(round(rng.gauss(0, float(v) * 0.004), 2)))
+ value_sequence.append(max(Decimal("1.00"), (v + noise).quantize(Decimal("0.01"))))
+ v = v * (1 + daily_pct)
+
+ for d, v in zip(asset_days, value_sequence):
+ value_rows.append({
+ "id": uuid.uuid4(),
+ "asset_id": asset.id,
+ "amount": v,
+ "date": d,
+ "source": "manual",
+ })
+
+ for i in range(0, len(value_rows), 2000):
+ await session.execute(pg_insert(AssetValue).values(value_rows[i : i + 2000]))
+
+ await session.commit()
+ total_av = len(ASSET_TEMPLATES[:n_assets]) * (n_days + 1)
+ print(f" {len(ASSET_TEMPLATES)} assets, {total_av:,} asset values")
+
+ # ── 7. Recurring transactions ────────────────────────────────────────
+ n_rec = max(1, int(20 * scale))
+ print(f"Creating {n_rec} recurring transactions …")
+ for i, desc in enumerate(RECURRING_DESCRIPTIONS[:n_rec]):
+ acc = accounts[i % len(accounts)]
+ cat = debit_cats[i % len(debit_cats)]
+ session.add(RecurringTransaction(
+ user_id=uid,
+ account_id=acc.id,
+ category_id=cat.id,
+ description=desc,
+ amount=Decimal(str(round(rng.uniform(10.0, 500.0), 2))),
+ currency=acc.currency,
+ type="debit",
+ frequency="monthly",
+ day_of_month=rng.randint(1, 28),
+ start_date=today - timedelta(days=365),
+ next_occurrence=today + timedelta(days=rng.randint(1, 30)),
+ is_active=True,
+ ))
+ await session.commit()
+ print(f" {n_rec} recurring transactions")
+
+ print("\nSeed complete.")
+ print(f" Login : {email}")
+ print(f" Passwd: {password}")
+
+
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Seed performance stress-test data")
+ parser.add_argument("--scale", type=float, default=1.0,
+ help="Volume multiplier (default 1.0; 0.1 for quick smoke run)")
+ parser.add_argument("--email", default="[email protected]")
+ parser.add_argument("--password", default="Securo123!")
+ parser.add_argument("--no-reset", dest="reset", action="store_false", default=True,
+ help="Skip wiping existing seed data for this user")
+ parser.add_argument("--start-date", type=date.fromisoformat, default=date(2024, 1, 1),
+ metavar="YYYY-MM-DD", help="Earliest date for seeded data (default: 2024-01-01)")
+ parser.add_argument("--accounts", type=int, default=len(ACCOUNT_TEMPLATES),
+ help=f"Number of accounts to create (default: {len(ACCOUNT_TEMPLATES)}, max: {len(ACCOUNT_TEMPLATES)})")
+ parser.add_argument("--categories", type=int, default=len(CATEGORY_TEMPLATES),
+ help=f"Number of categories to create (default: {len(CATEGORY_TEMPLATES)}, max: {len(CATEGORY_TEMPLATES)})")
+ parser.add_argument("--assets", type=int, default=len(ASSET_TEMPLATES),
+ help=f"Number of assets to create (default: {len(ASSET_TEMPLATES)}, max: {len(ASSET_TEMPLATES)})")
+ args = parser.parse_args()
+ asyncio.run(seed(
+ args.email, args.password, args.scale, args.reset, args.start_date,
+ args.accounts, args.categories, args.assets,
+ ))
+
+
+if __name__ == "__main__":
+ main()