Open-source personal finance manager. Self-hosted, privacy-first.
fix: limit celery concurrency and dispose SQLAlchemy engines to reduce RAM usage (#208)
Luis Gustavo Couto Wacker7186dfd8969c
diff --git a/backend/app/tasks/asset_tasks.py b/backend/app/tasks/asset_tasks.pyindex 52efbafc..38086e8a 100644--- a/backend/app/tasks/asset_tasks.py+++ b/backend/app/tasks/asset_tasks.py@@ -19,7 +19,7 @@ def _make_session_maker():"""Create a fresh engine+session for the Celery worker event loop."""settings = get_settings()engine = create_async_engine(settings.database_url)- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)def _next_due_date(last_date: date, frequency: str) -> date:@@ -43,82 +43,85 @@ def _next_due_date(last_date: date, frequency: str) -> date:async def _apply_growth_rules() -> int:"""Apply growth rules for all assets that have valuation_method='growth_rule'."""- session_maker = _make_session_maker()- today = date.today()- total = 0-- async with session_maker() as session:- result = await session.execute(- select(Asset).where(- Asset.valuation_method == "growth_rule",- Asset.growth_type.isnot(None),- Asset.growth_rate.isnot(None),- Asset.growth_frequency.isnot(None),- Asset.is_archived == False,- Asset.sell_date.is_(None),- )- )- assets = list(result.scalars().all())-- for asset in assets:- try:- async with session_maker() as session:- # Get latest value- val_result = await session.execute(- select(AssetValue)- .where(AssetValue.asset_id == asset.id)- .order_by(AssetValue.date.desc(), AssetValue.id.desc())- .limit(1)+ engine, session_maker = _make_session_maker()+ try:+ today = date.today()+ total = 0++ async with session_maker() as session:+ result = await session.execute(+ select(Asset).where(+ Asset.valuation_method == "growth_rule",+ Asset.growth_type.isnot(None),+ Asset.growth_rate.isnot(None),+ Asset.growth_frequency.isnot(None),+ Asset.is_archived == False,+ Asset.sell_date.is_(None),)- latest = val_result.scalar_one_or_none()- if not latest:- continue-- # Check if growth should start- if asset.growth_start_date and today < asset.growth_start_date:- continue-- # Generate all missed periods in a loop- current_date = latest.date- current_amount = float(latest.amount)- created = 0-- while True:- next_due = _next_due_date(current_date, asset.growth_frequency)- if next_due > today:- break-- if asset.growth_type == "percentage":- current_amount = current_amount * (1 + float(asset.growth_rate) / 100)- elif asset.growth_type == "absolute":- current_amount = current_amount + float(asset.growth_rate)- else:- break-- new_value = AssetValue(- asset_id=asset.id,- amount=Decimal(str(round(current_amount, 6))),- date=next_due,- source="rule",- )- session.add(new_value)- current_date = next_due- created += 1-- # Safety limit to avoid infinite loops- if created >= 1000:- break-- if created > 0:- await session.commit()- total += created- logger.info(- "Growth rule applied for asset %s: %d values created, latest=%.2f",- asset.id, created, current_amount,+ )+ assets = list(result.scalars().all())++ for asset in assets:+ try:+ async with session_maker() as session:+ # Get latest value+ val_result = await session.execute(+ select(AssetValue)+ .where(AssetValue.asset_id == asset.id)+ .order_by(AssetValue.date.desc(), AssetValue.id.desc())+ .limit(1))- except Exception:- logger.exception("Failed to apply growth rule for asset %s", asset.id)-+ latest = val_result.scalar_one_or_none()+ if not latest:+ continue++ # Check if growth should start+ if asset.growth_start_date and today < asset.growth_start_date:+ continue++ # Generate all missed periods in a loop+ current_date = latest.date+ current_amount = float(latest.amount)+ created = 0++ while True:+ next_due = _next_due_date(current_date, asset.growth_frequency)+ if next_due > today:+ break++ if asset.growth_type == "percentage":+ current_amount = current_amount * (1 + float(asset.growth_rate) / 100)+ elif asset.growth_type == "absolute":+ current_amount = current_amount + float(asset.growth_rate)+ else:+ break++ new_value = AssetValue(+ asset_id=asset.id,+ amount=Decimal(str(round(current_amount, 6))),+ date=next_due,+ source="rule",+ )+ session.add(new_value)+ current_date = next_due+ created += 1++ # Safety limit to avoid infinite loops+ if created >= 1000:+ break++ if created > 0:+ await session.commit()+ total += created+ logger.info(+ "Growth rule applied for asset %s: %d values created, latest=%.2f",+ asset.id, created, current_amount,+ )+ except Exception:+ logger.exception("Failed to apply growth rule for asset %s", asset.id)++ finally:+ await engine.dispose()return total@@ -136,9 +139,12 @@ async def _refresh_market_prices() -> dict[str, int]:Uses a fresh async session per task run (same pattern as the growth-ruletask above) to avoid sharing engines with the FastAPI request lifecycle."""- session_maker = _make_session_maker()- async with session_maker() as session:- return await refresh_all_market_prices(session)+ engine, session_maker = _make_session_maker()+ try:+ async with session_maker() as session:+ return await refresh_all_market_prices(session)+ finally:+ await engine.dispose()@celery_app.task(name="app.tasks.asset_tasks.refresh_market_prices")diff --git a/backend/app/tasks/fx_backfill_tasks.py b/backend/app/tasks/fx_backfill_tasks.pyindex 8f696084..a75fc31e 100644--- a/backend/app/tasks/fx_backfill_tasks.py+++ b/backend/app/tasks/fx_backfill_tasks.py@@ -19,102 +19,106 @@ logger = logging.getLogger(__name__)def _make_session_maker():settings = get_settings()engine = create_async_engine(settings.database_url)- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)async def _backfill_primary_amounts() -> dict:"""One-time backfill: stamp amount_primary on all rows where it's NULL."""from app.services.fx_rate_service import sync_rates, convert- session_maker = _make_session_maker()- stats = {"transactions": 0, "recurring": 0, "assets": 0, "rates_synced": 0}-- async with session_maker() as session:- # 1. Collect all unique (year, month) pairs from transactions needing backfill- month_result = await session.execute(- select(- distinct(func.to_char(Transaction.date, 'YYYY-MM'))- ).where(Transaction.amount_primary.is_(None))- )- months = [row[0] for row in month_result.all()]-- # Sync historical rates for each month- for month_str in months:- try:- year, mon = month_str.split("-")- # Use last day of month for historical rate- if int(mon) == 12:- target = date(int(year) + 1, 1, 1)- else:- target = date(int(year), int(mon) + 1, 1)- from datetime import timedelta- target = target - timedelta(days=1)- count = await sync_rates(session, target)- stats["rates_synced"] += count- except Exception:- logger.exception("Failed to sync rates for %s", month_str)-- # 2. Get all users- users_result = await session.execute(select(User))- settings = get_settings()- users = {u.id: (u.preferences or {}).get("currency_display", settings.default_currency) for u in users_result.scalars().all()}-- # 3. Backfill transactions- tx_result = await session.execute(- select(Transaction).where(Transaction.amount_primary.is_(None))- )- for tx in tx_result.scalars().all():- primary_currency = users.get(tx.user_id, settings.default_currency)- try:- converted, rate = await convert(- session, Decimal(str(tx.amount)),- tx.currency, primary_currency, tx.date,- )- tx.amount_primary = converted- tx.fx_rate_used = rate- stats["transactions"] += 1- except Exception:- logger.exception("Failed to backfill tx %s", tx.id)- await session.commit()-- # 4. Backfill recurring transactions- rec_result = await session.execute(- select(RecurringTransaction).where(RecurringTransaction.amount_primary.is_(None))- )- for rec in rec_result.scalars().all():- primary_currency = users.get(rec.user_id, settings.default_currency)- try:- converted, rate = await convert(- session, Decimal(str(rec.amount)),- rec.currency, primary_currency, rec.start_date,+ engine, session_maker = _make_session_maker()+ try:+ stats = {"transactions": 0, "recurring": 0, "assets": 0, "rates_synced": 0}++ async with session_maker() as session:+ # 1. Collect all unique (year, month) pairs from transactions needing backfill+ month_result = await session.execute(+ select(distinct(func.to_char(Transaction.date, "YYYY-MM"))).where(+ Transaction.amount_primary.is_(None))- rec.amount_primary = converted- rec.fx_rate_used = rate- stats["recurring"] += 1- except Exception:- logger.exception("Failed to backfill recurring %s", rec.id)- await session.commit()-- # 5. Backfill assets- asset_result = await session.execute(- select(Asset).where(- Asset.purchase_price.isnot(None),- Asset.purchase_price_primary.is_(None),)- )- for asset in asset_result.scalars().all():- primary_currency = users.get(asset.user_id, settings.default_currency)- try:- converted, _ = await convert(- session, Decimal(str(asset.purchase_price)),- asset.currency, primary_currency, asset.purchase_date,+ months = [row[0] for row in month_result.all()]++ # Sync historical rates for each month+ for month_str in months:+ try:+ year, mon = month_str.split("-")+ # Use last day of month for historical rate+ if int(mon) == 12:+ target = date(int(year) + 1, 1, 1)+ else:+ target = date(int(year), int(mon) + 1, 1)+ from datetime import timedelta++ target = target - timedelta(days=1)+ count = await sync_rates(session, target)+ stats["rates_synced"] += count+ except Exception:+ logger.exception("Failed to sync rates for %s", month_str)++ # 2. Get all users+ users_result = await session.execute(select(User))+ settings = get_settings()+ users = {u.id: (u.preferences or {}).get("currency_display", settings.default_currency) for u in users_result.scalars().all()}++ # 3. Backfill transactions+ tx_result = await session.execute(+ select(Transaction).where(Transaction.amount_primary.is_(None))+ )+ for tx in tx_result.scalars().all():+ primary_currency = users.get(tx.user_id, settings.default_currency)+ try:+ converted, rate = await convert(+ session, Decimal(str(tx.amount)),+ tx.currency, primary_currency, tx.date,+ )+ tx.amount_primary = converted+ tx.fx_rate_used = rate+ stats["transactions"] += 1+ except Exception:+ logger.exception("Failed to backfill tx %s", tx.id)+ await session.commit()++ # 4. Backfill recurring transactions+ rec_result = await session.execute(+ select(RecurringTransaction).where(RecurringTransaction.amount_primary.is_(None))+ )+ for rec in rec_result.scalars().all():+ primary_currency = users.get(rec.user_id, settings.default_currency)+ try:+ converted, rate = await convert(+ session, Decimal(str(rec.amount)),+ rec.currency, primary_currency, rec.start_date,+ )+ rec.amount_primary = converted+ rec.fx_rate_used = rate+ stats["recurring"] += 1+ except Exception:+ logger.exception("Failed to backfill recurring %s", rec.id)+ await session.commit()++ # 5. Backfill assets+ asset_result = await session.execute(+ select(Asset).where(+ Asset.purchase_price.isnot(None),+ Asset.purchase_price_primary.is_(None),)- asset.purchase_price_primary = converted- stats["assets"] += 1- except Exception:- logger.exception("Failed to backfill asset %s", asset.id)- await session.commit()-+ )+ for asset in asset_result.scalars().all():+ primary_currency = users.get(asset.user_id, settings.default_currency)+ try:+ converted, _ = await convert(+ session, Decimal(str(asset.purchase_price)),+ asset.currency, primary_currency, asset.purchase_date,+ )+ asset.purchase_price_primary = converted+ stats["assets"] += 1+ except Exception:+ logger.exception("Failed to backfill asset %s", asset.id)+ await session.commit()++ finally:+ await engine.dispose()return statsdiff --git a/backend/app/tasks/fx_rate_tasks.py b/backend/app/tasks/fx_rate_tasks.pyindex cab2a62d..a50ef748 100644--- a/backend/app/tasks/fx_rate_tasks.py+++ b/backend/app/tasks/fx_rate_tasks.py@@ -13,17 +13,20 @@ def _make_session_maker():"""Create a fresh engine+session for the Celery worker event loop."""settings = get_settings()engine = create_async_engine(settings.database_url)- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)async def _sync_fx_rates() -> int:"""Fetch latest FX rates and store them."""from app.services.fx_rate_service import sync_rates- session_maker = _make_session_maker()- async with session_maker() as session:- count = await sync_rates(session)- return count+ engine, session_maker = _make_session_maker()+ try:+ async with session_maker() as session:+ count = await sync_rates(session)+ return count+ finally:+ await engine.dispose()async def _restamp_recurring_fx() -> int:@@ -34,9 +37,10 @@ async def _restamp_recurring_fx() -> int:from app.models.user import Userfrom app.services.fx_rate_service import stamp_primary_amount- session_maker = _make_session_maker()- async with session_maker() as session:- users = (await session.execute(select(User))).scalars().all()+ engine, session_maker = _make_session_maker()+ try:+ async with session_maker() as session:+ users = (await session.execute(select(User))).scalars().all()count = 0for user in users:primary = user.primary_currency@@ -53,6 +57,8 @@ async def _restamp_recurring_fx() -> int:)count += 1await session.commit()+ finally:+ await engine.dispose()return countdiff --git a/backend/app/tasks/recurring_tasks.py b/backend/app/tasks/recurring_tasks.pyindex f9fd84d5..e88497be 100644--- a/backend/app/tasks/recurring_tasks.py+++ b/backend/app/tasks/recurring_tasks.py@@ -16,28 +16,31 @@ def _make_session_maker():"""Create a fresh engine+session for the Celery worker event loop."""settings = get_settings()engine = create_async_engine(settings.database_url)- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)async def _generate_all() -> int:"""Generate pending recurring transactions for all users."""- session_maker = _make_session_maker()- total = 0-- async with session_maker() as session:- result = await session.execute(select(User.id))- user_ids = [row[0] for row in result.all()]-- for user_id in user_ids:- try:- async with session_maker() as session:- count = await recurring_transaction_service.generate_pending(session, user_id)- if count:- logger.info("Generated %d recurring transactions for user %s", count, user_id)- total += count- except Exception:- logger.exception("Failed to generate recurring transactions for user %s", user_id)-+ engine, session_maker = _make_session_maker()+ try:+ total = 0++ async with session_maker() as session:+ result = await session.execute(select(User.id))+ user_ids = [row[0] for row in result.all()]++ for user_id in user_ids:+ try:+ async with session_maker() as session:+ count = await recurring_transaction_service.generate_pending(session, user_id)+ if count:+ logger.info("Generated %d recurring transactions for user %s", count, user_id)+ total += count+ except Exception:+ logger.exception("Failed to generate recurring transactions for user %s", user_id)++ finally:+ await engine.dispose()return totaldiff --git a/backend/app/tasks/sync_tasks.py b/backend/app/tasks/sync_tasks.pyindex 05feb7e5..5422a1b0 100644--- a/backend/app/tasks/sync_tasks.py+++ b/backend/app/tasks/sync_tasks.py@@ -4,9 +4,10 @@ import uuidfrom datetime import datetime, timezone, timedeltafrom sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_enginefrom app.worker import celery_app-from app.core.database import async_session_maker+from app.core.config import get_settingsfrom app.models.bank_connection import BankConnectionfrom app.services import connection_service@@ -15,40 +16,53 @@ logger = logging.getLogger(__name__)STALE_THRESHOLD = timedelta(hours=4)+def _make_session_maker():+ """Create a fresh engine+session for the Celery worker event loop."""+ engine = create_async_engine(get_settings().database_url)+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)++async def _sync_all() -> int:"""Find stale connections and sync each one."""- cutoff = datetime.now(timezone.utc) - STALE_THRESHOLD- synced = 0-- async with async_session_maker() as session:- result = await session.execute(- select(BankConnection.id, BankConnection.user_id, BankConnection.last_sync_at).where(- BankConnection.status.in_(["active", "error"]),- (BankConnection.last_sync_at < cutoff) | (BankConnection.last_sync_at.is_(None)),+ engine, session_maker = _make_session_maker()+ try:+ cutoff = datetime.now(timezone.utc) - STALE_THRESHOLD+ synced = 0++ async with session_maker() as session:+ result = await session.execute(+ select(+ BankConnection.id, BankConnection.user_id, BankConnection.last_sync_at+ ).where(+ BankConnection.status.in_(["active", "error"]),+ (BankConnection.last_sync_at < cutoff)+ | (BankConnection.last_sync_at.is_(None)),+ ))- )- connections = result.all()+ connections = result.all()- logger.info(- "Sync check: found %d stale connections (cutoff=%s)",- len(connections),- cutoff.isoformat(),- )+ logger.info(+ "Sync check: found %d stale connections (cutoff=%s)",+ len(connections),+ cutoff.isoformat(),+ )- for conn_id, user_id, last_sync in connections:- try:- logger.info("Syncing connection %s (last_sync=%s)", conn_id, last_sync)- await _sync_one(conn_id, user_id)- synced += 1- except Exception:- logger.exception("Background sync failed for connection %s", conn_id)+ for conn_id, user_id, last_sync in connections:+ try:+ logger.info("Syncing connection %s (last_sync=%s)", conn_id, last_sync)+ await _sync_one(session_maker, conn_id, user_id)+ synced += 1+ except Exception:+ logger.exception("Background sync failed for connection %s", conn_id)- return synced+ return synced+ finally:+ await engine.dispose()-async def _sync_one(connection_id: uuid.UUID, user_id: uuid.UUID) -> None:+async def _sync_one(session_maker, connection_id: uuid.UUID, user_id: uuid.UUID) -> None:"""Sync a single connection. Error status is set by sync_connection itself."""- async with async_session_maker() as session:+ async with session_maker() as session:await connection_service.sync_connection(session, connection_id, user_id)@@ -64,8 +78,17 @@ def sync_all_connections() -> dict:def sync_single_connection(connection_id: str, user_id: str) -> dict:"""Celery task: sync a single connection (used for on-demand dispatch)."""try:- asyncio.run(_sync_one(uuid.UUID(connection_id), uuid.UUID(user_id)))+ asyncio.run(_sync_one_celery(connection_id, user_id))return {"status": "ok", "connection_id": connection_id}except Exception as e:logger.exception("Sync task failed for connection %s", connection_id)return {"status": "error", "connection_id": connection_id, "error": str(e)}+++async def _sync_one_celery(connection_id: str, user_id: str) -> None:+ engine, session_maker = _make_session_maker()+ try:+ async with session_maker() as session:+ await connection_service.sync_connection(session, uuid.UUID(connection_id), uuid.UUID(user_id))+ finally:+ await engine.dispose()diff --git a/docker-compose.prod.yml b/docker-compose.prod.ymlindex 249fd6c4..0b625a69 100644--- a/docker-compose.prod.yml+++ b/docker-compose.prod.yml@@ -67,7 +67,8 @@ services:celery-worker:<<: *backend-base- command: celery -A app.worker worker --loglevel=info+ # Limit to 2 workers to save RAM (default = number of CPUs).+ command: celery -A app.worker worker --loglevel=info --concurrency=2celery-beat:<<: *backend-basediff --git a/docker-compose.yml b/docker-compose.ymlindex 0b9379d5..e95d9630 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -100,7 +100,8 @@ services:celery-worker:<<: *backend-base- command: celery -A app.worker worker --loglevel=info+ # Limit to 2 workers to save RAM (default = number of CPUs).+ command: celery -A app.worker worker --loglevel=info --concurrency=2celery-beat:<<: *backend-base