Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: limit celery concurrency and dispose SQLAlchemy engines to reduce RAM usage (#208)

Luis Gustavo Couto Wacker7186dfd8969c
diff --git a/backend/app/tasks/asset_tasks.py b/backend/app/tasks/asset_tasks.py
index 52efbafc..38086e8a 100644
--- a/backend/app/tasks/asset_tasks.py
+++ b/backend/app/tasks/asset_tasks.py
@@ -19,7 +19,7 @@ def _make_session_maker():
"""Create a fresh engine+session for the Celery worker event loop."""
settings = get_settings()
engine = create_async_engine(settings.database_url)
- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
def _next_due_date(last_date: date, frequency: str) -> date:
@@ -43,82 +43,85 @@ def _next_due_date(last_date: date, frequency: str) -> date:
async def _apply_growth_rules() -> int:
"""Apply growth rules for all assets that have valuation_method='growth_rule'."""
- session_maker = _make_session_maker()
- today = date.today()
- total = 0
-
- async with session_maker() as session:
- result = await session.execute(
- select(Asset).where(
- Asset.valuation_method == "growth_rule",
- Asset.growth_type.isnot(None),
- Asset.growth_rate.isnot(None),
- Asset.growth_frequency.isnot(None),
- Asset.is_archived == False,
- Asset.sell_date.is_(None),
- )
- )
- assets = list(result.scalars().all())
-
- for asset in assets:
- try:
- async with session_maker() as session:
- # Get latest value
- val_result = await session.execute(
- select(AssetValue)
- .where(AssetValue.asset_id == asset.id)
- .order_by(AssetValue.date.desc(), AssetValue.id.desc())
- .limit(1)
+ engine, session_maker = _make_session_maker()
+ try:
+ today = date.today()
+ total = 0
+
+ async with session_maker() as session:
+ result = await session.execute(
+ select(Asset).where(
+ Asset.valuation_method == "growth_rule",
+ Asset.growth_type.isnot(None),
+ Asset.growth_rate.isnot(None),
+ Asset.growth_frequency.isnot(None),
+ Asset.is_archived == False,
+ Asset.sell_date.is_(None),
)
- latest = val_result.scalar_one_or_none()
- if not latest:
- continue
-
- # Check if growth should start
- if asset.growth_start_date and today < asset.growth_start_date:
- continue
-
- # Generate all missed periods in a loop
- current_date = latest.date
- current_amount = float(latest.amount)
- created = 0
-
- while True:
- next_due = _next_due_date(current_date, asset.growth_frequency)
- if next_due > today:
- break
-
- if asset.growth_type == "percentage":
- current_amount = current_amount * (1 + float(asset.growth_rate) / 100)
- elif asset.growth_type == "absolute":
- current_amount = current_amount + float(asset.growth_rate)
- else:
- break
-
- new_value = AssetValue(
- asset_id=asset.id,
- amount=Decimal(str(round(current_amount, 6))),
- date=next_due,
- source="rule",
- )
- session.add(new_value)
- current_date = next_due
- created += 1
-
- # Safety limit to avoid infinite loops
- if created >= 1000:
- break
-
- if created > 0:
- await session.commit()
- total += created
- logger.info(
- "Growth rule applied for asset %s: %d values created, latest=%.2f",
- asset.id, created, current_amount,
+ )
+ assets = list(result.scalars().all())
+
+ for asset in assets:
+ try:
+ async with session_maker() as session:
+ # Get latest value
+ val_result = await session.execute(
+ select(AssetValue)
+ .where(AssetValue.asset_id == asset.id)
+ .order_by(AssetValue.date.desc(), AssetValue.id.desc())
+ .limit(1)
)
- except Exception:
- logger.exception("Failed to apply growth rule for asset %s", asset.id)
-
+ latest = val_result.scalar_one_or_none()
+ if not latest:
+ continue
+
+ # Check if growth should start
+ if asset.growth_start_date and today < asset.growth_start_date:
+ continue
+
+ # Generate all missed periods in a loop
+ current_date = latest.date
+ current_amount = float(latest.amount)
+ created = 0
+
+ while True:
+ next_due = _next_due_date(current_date, asset.growth_frequency)
+ if next_due > today:
+ break
+
+ if asset.growth_type == "percentage":
+ current_amount = current_amount * (1 + float(asset.growth_rate) / 100)
+ elif asset.growth_type == "absolute":
+ current_amount = current_amount + float(asset.growth_rate)
+ else:
+ break
+
+ new_value = AssetValue(
+ asset_id=asset.id,
+ amount=Decimal(str(round(current_amount, 6))),
+ date=next_due,
+ source="rule",
+ )
+ session.add(new_value)
+ current_date = next_due
+ created += 1
+
+ # Safety limit to avoid infinite loops
+ if created >= 1000:
+ break
+
+ if created > 0:
+ await session.commit()
+ total += created
+ logger.info(
+ "Growth rule applied for asset %s: %d values created, latest=%.2f",
+ asset.id, created, current_amount,
+ )
+ except Exception:
+ logger.exception("Failed to apply growth rule for asset %s", asset.id)
+
+ finally:
+ await engine.dispose()
return total
@@ -136,9 +139,12 @@ async def _refresh_market_prices() -> dict[str, int]:
Uses a fresh async session per task run (same pattern as the growth-rule
task above) to avoid sharing engines with the FastAPI request lifecycle.
"""
- session_maker = _make_session_maker()
- async with session_maker() as session:
- return await refresh_all_market_prices(session)
+ engine, session_maker = _make_session_maker()
+ try:
+ async with session_maker() as session:
+ return await refresh_all_market_prices(session)
+ finally:
+ await engine.dispose()
@celery_app.task(name="app.tasks.asset_tasks.refresh_market_prices")
diff --git a/backend/app/tasks/fx_backfill_tasks.py b/backend/app/tasks/fx_backfill_tasks.py
index 8f696084..a75fc31e 100644
--- a/backend/app/tasks/fx_backfill_tasks.py
+++ b/backend/app/tasks/fx_backfill_tasks.py
@@ -19,102 +19,106 @@ logger = logging.getLogger(__name__)
def _make_session_maker():
settings = get_settings()
engine = create_async_engine(settings.database_url)
- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
async def _backfill_primary_amounts() -> dict:
"""One-time backfill: stamp amount_primary on all rows where it's NULL."""
from app.services.fx_rate_service import sync_rates, convert
- session_maker = _make_session_maker()
- stats = {"transactions": 0, "recurring": 0, "assets": 0, "rates_synced": 0}
-
- async with session_maker() as session:
- # 1. Collect all unique (year, month) pairs from transactions needing backfill
- month_result = await session.execute(
- select(
- distinct(func.to_char(Transaction.date, 'YYYY-MM'))
- ).where(Transaction.amount_primary.is_(None))
- )
- months = [row[0] for row in month_result.all()]
-
- # Sync historical rates for each month
- for month_str in months:
- try:
- year, mon = month_str.split("-")
- # Use last day of month for historical rate
- if int(mon) == 12:
- target = date(int(year) + 1, 1, 1)
- else:
- target = date(int(year), int(mon) + 1, 1)
- from datetime import timedelta
- target = target - timedelta(days=1)
- count = await sync_rates(session, target)
- stats["rates_synced"] += count
- except Exception:
- logger.exception("Failed to sync rates for %s", month_str)
-
- # 2. Get all users
- users_result = await session.execute(select(User))
- settings = get_settings()
- users = {u.id: (u.preferences or {}).get("currency_display", settings.default_currency) for u in users_result.scalars().all()}
-
- # 3. Backfill transactions
- tx_result = await session.execute(
- select(Transaction).where(Transaction.amount_primary.is_(None))
- )
- for tx in tx_result.scalars().all():
- primary_currency = users.get(tx.user_id, settings.default_currency)
- try:
- converted, rate = await convert(
- session, Decimal(str(tx.amount)),
- tx.currency, primary_currency, tx.date,
- )
- tx.amount_primary = converted
- tx.fx_rate_used = rate
- stats["transactions"] += 1
- except Exception:
- logger.exception("Failed to backfill tx %s", tx.id)
- await session.commit()
-
- # 4. Backfill recurring transactions
- rec_result = await session.execute(
- select(RecurringTransaction).where(RecurringTransaction.amount_primary.is_(None))
- )
- for rec in rec_result.scalars().all():
- primary_currency = users.get(rec.user_id, settings.default_currency)
- try:
- converted, rate = await convert(
- session, Decimal(str(rec.amount)),
- rec.currency, primary_currency, rec.start_date,
+ engine, session_maker = _make_session_maker()
+ try:
+ stats = {"transactions": 0, "recurring": 0, "assets": 0, "rates_synced": 0}
+
+ async with session_maker() as session:
+ # 1. Collect all unique (year, month) pairs from transactions needing backfill
+ month_result = await session.execute(
+ select(distinct(func.to_char(Transaction.date, "YYYY-MM"))).where(
+ Transaction.amount_primary.is_(None)
)
- rec.amount_primary = converted
- rec.fx_rate_used = rate
- stats["recurring"] += 1
- except Exception:
- logger.exception("Failed to backfill recurring %s", rec.id)
- await session.commit()
-
- # 5. Backfill assets
- asset_result = await session.execute(
- select(Asset).where(
- Asset.purchase_price.isnot(None),
- Asset.purchase_price_primary.is_(None),
)
- )
- for asset in asset_result.scalars().all():
- primary_currency = users.get(asset.user_id, settings.default_currency)
- try:
- converted, _ = await convert(
- session, Decimal(str(asset.purchase_price)),
- asset.currency, primary_currency, asset.purchase_date,
+ months = [row[0] for row in month_result.all()]
+
+ # Sync historical rates for each month
+ for month_str in months:
+ try:
+ year, mon = month_str.split("-")
+ # Use last day of month for historical rate
+ if int(mon) == 12:
+ target = date(int(year) + 1, 1, 1)
+ else:
+ target = date(int(year), int(mon) + 1, 1)
+ from datetime import timedelta
+
+ target = target - timedelta(days=1)
+ count = await sync_rates(session, target)
+ stats["rates_synced"] += count
+ except Exception:
+ logger.exception("Failed to sync rates for %s", month_str)
+
+ # 2. Get all users
+ users_result = await session.execute(select(User))
+ settings = get_settings()
+ users = {u.id: (u.preferences or {}).get("currency_display", settings.default_currency) for u in users_result.scalars().all()}
+
+ # 3. Backfill transactions
+ tx_result = await session.execute(
+ select(Transaction).where(Transaction.amount_primary.is_(None))
+ )
+ for tx in tx_result.scalars().all():
+ primary_currency = users.get(tx.user_id, settings.default_currency)
+ try:
+ converted, rate = await convert(
+ session, Decimal(str(tx.amount)),
+ tx.currency, primary_currency, tx.date,
+ )
+ tx.amount_primary = converted
+ tx.fx_rate_used = rate
+ stats["transactions"] += 1
+ except Exception:
+ logger.exception("Failed to backfill tx %s", tx.id)
+ await session.commit()
+
+ # 4. Backfill recurring transactions
+ rec_result = await session.execute(
+ select(RecurringTransaction).where(RecurringTransaction.amount_primary.is_(None))
+ )
+ for rec in rec_result.scalars().all():
+ primary_currency = users.get(rec.user_id, settings.default_currency)
+ try:
+ converted, rate = await convert(
+ session, Decimal(str(rec.amount)),
+ rec.currency, primary_currency, rec.start_date,
+ )
+ rec.amount_primary = converted
+ rec.fx_rate_used = rate
+ stats["recurring"] += 1
+ except Exception:
+ logger.exception("Failed to backfill recurring %s", rec.id)
+ await session.commit()
+
+ # 5. Backfill assets
+ asset_result = await session.execute(
+ select(Asset).where(
+ Asset.purchase_price.isnot(None),
+ Asset.purchase_price_primary.is_(None),
)
- asset.purchase_price_primary = converted
- stats["assets"] += 1
- except Exception:
- logger.exception("Failed to backfill asset %s", asset.id)
- await session.commit()
-
+ )
+ for asset in asset_result.scalars().all():
+ primary_currency = users.get(asset.user_id, settings.default_currency)
+ try:
+ converted, _ = await convert(
+ session, Decimal(str(asset.purchase_price)),
+ asset.currency, primary_currency, asset.purchase_date,
+ )
+ asset.purchase_price_primary = converted
+ stats["assets"] += 1
+ except Exception:
+ logger.exception("Failed to backfill asset %s", asset.id)
+ await session.commit()
+
+ finally:
+ await engine.dispose()
return stats
diff --git a/backend/app/tasks/fx_rate_tasks.py b/backend/app/tasks/fx_rate_tasks.py
index cab2a62d..a50ef748 100644
--- a/backend/app/tasks/fx_rate_tasks.py
+++ b/backend/app/tasks/fx_rate_tasks.py
@@ -13,17 +13,20 @@ def _make_session_maker():
"""Create a fresh engine+session for the Celery worker event loop."""
settings = get_settings()
engine = create_async_engine(settings.database_url)
- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
async def _sync_fx_rates() -> int:
"""Fetch latest FX rates and store them."""
from app.services.fx_rate_service import sync_rates
- session_maker = _make_session_maker()
- async with session_maker() as session:
- count = await sync_rates(session)
- return count
+ engine, session_maker = _make_session_maker()
+ try:
+ async with session_maker() as session:
+ count = await sync_rates(session)
+ return count
+ finally:
+ await engine.dispose()
async def _restamp_recurring_fx() -> int:
@@ -34,9 +37,10 @@ async def _restamp_recurring_fx() -> int:
from app.models.user import User
from app.services.fx_rate_service import stamp_primary_amount
- session_maker = _make_session_maker()
- async with session_maker() as session:
- users = (await session.execute(select(User))).scalars().all()
+ engine, session_maker = _make_session_maker()
+ try:
+ async with session_maker() as session:
+ users = (await session.execute(select(User))).scalars().all()
count = 0
for user in users:
primary = user.primary_currency
@@ -53,6 +57,8 @@ async def _restamp_recurring_fx() -> int:
)
count += 1
await session.commit()
+ finally:
+ await engine.dispose()
return count
diff --git a/backend/app/tasks/recurring_tasks.py b/backend/app/tasks/recurring_tasks.py
index f9fd84d5..e88497be 100644
--- a/backend/app/tasks/recurring_tasks.py
+++ b/backend/app/tasks/recurring_tasks.py
@@ -16,28 +16,31 @@ def _make_session_maker():
"""Create a fresh engine+session for the Celery worker event loop."""
settings = get_settings()
engine = create_async_engine(settings.database_url)
- return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
async def _generate_all() -> int:
"""Generate pending recurring transactions for all users."""
- session_maker = _make_session_maker()
- total = 0
-
- async with session_maker() as session:
- result = await session.execute(select(User.id))
- user_ids = [row[0] for row in result.all()]
-
- for user_id in user_ids:
- try:
- async with session_maker() as session:
- count = await recurring_transaction_service.generate_pending(session, user_id)
- if count:
- logger.info("Generated %d recurring transactions for user %s", count, user_id)
- total += count
- except Exception:
- logger.exception("Failed to generate recurring transactions for user %s", user_id)
-
+ engine, session_maker = _make_session_maker()
+ try:
+ total = 0
+
+ async with session_maker() as session:
+ result = await session.execute(select(User.id))
+ user_ids = [row[0] for row in result.all()]
+
+ for user_id in user_ids:
+ try:
+ async with session_maker() as session:
+ count = await recurring_transaction_service.generate_pending(session, user_id)
+ if count:
+ logger.info("Generated %d recurring transactions for user %s", count, user_id)
+ total += count
+ except Exception:
+ logger.exception("Failed to generate recurring transactions for user %s", user_id)
+
+ finally:
+ await engine.dispose()
return total
diff --git a/backend/app/tasks/sync_tasks.py b/backend/app/tasks/sync_tasks.py
index 05feb7e5..5422a1b0 100644
--- a/backend/app/tasks/sync_tasks.py
+++ b/backend/app/tasks/sync_tasks.py
@@ -4,9 +4,10 @@ import uuid
from datetime import datetime, timezone, timedelta
from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.worker import celery_app
-from app.core.database import async_session_maker
+from app.core.config import get_settings
from app.models.bank_connection import BankConnection
from app.services import connection_service
@@ -15,40 +16,53 @@ logger = logging.getLogger(__name__)
STALE_THRESHOLD = timedelta(hours=4)
+def _make_session_maker():
+ """Create a fresh engine+session for the Celery worker event loop."""
+ engine = create_async_engine(get_settings().database_url)
+ return engine, async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
async def _sync_all() -> int:
"""Find stale connections and sync each one."""
- cutoff = datetime.now(timezone.utc) - STALE_THRESHOLD
- synced = 0
-
- async with async_session_maker() as session:
- result = await session.execute(
- select(BankConnection.id, BankConnection.user_id, BankConnection.last_sync_at).where(
- BankConnection.status.in_(["active", "error"]),
- (BankConnection.last_sync_at < cutoff) | (BankConnection.last_sync_at.is_(None)),
+ engine, session_maker = _make_session_maker()
+ try:
+ cutoff = datetime.now(timezone.utc) - STALE_THRESHOLD
+ synced = 0
+
+ async with session_maker() as session:
+ result = await session.execute(
+ select(
+ BankConnection.id, BankConnection.user_id, BankConnection.last_sync_at
+ ).where(
+ BankConnection.status.in_(["active", "error"]),
+ (BankConnection.last_sync_at < cutoff)
+ | (BankConnection.last_sync_at.is_(None)),
+ )
)
- )
- connections = result.all()
+ connections = result.all()
- logger.info(
- "Sync check: found %d stale connections (cutoff=%s)",
- len(connections),
- cutoff.isoformat(),
- )
+ logger.info(
+ "Sync check: found %d stale connections (cutoff=%s)",
+ len(connections),
+ cutoff.isoformat(),
+ )
- for conn_id, user_id, last_sync in connections:
- try:
- logger.info("Syncing connection %s (last_sync=%s)", conn_id, last_sync)
- await _sync_one(conn_id, user_id)
- synced += 1
- except Exception:
- logger.exception("Background sync failed for connection %s", conn_id)
+ for conn_id, user_id, last_sync in connections:
+ try:
+ logger.info("Syncing connection %s (last_sync=%s)", conn_id, last_sync)
+ await _sync_one(session_maker, conn_id, user_id)
+ synced += 1
+ except Exception:
+ logger.exception("Background sync failed for connection %s", conn_id)
- return synced
+ return synced
+ finally:
+ await engine.dispose()
-async def _sync_one(connection_id: uuid.UUID, user_id: uuid.UUID) -> None:
+async def _sync_one(session_maker, connection_id: uuid.UUID, user_id: uuid.UUID) -> None:
"""Sync a single connection. Error status is set by sync_connection itself."""
- async with async_session_maker() as session:
+ async with session_maker() as session:
await connection_service.sync_connection(session, connection_id, user_id)
@@ -64,8 +78,17 @@ def sync_all_connections() -> dict:
def sync_single_connection(connection_id: str, user_id: str) -> dict:
"""Celery task: sync a single connection (used for on-demand dispatch)."""
try:
- asyncio.run(_sync_one(uuid.UUID(connection_id), uuid.UUID(user_id)))
+ asyncio.run(_sync_one_celery(connection_id, user_id))
return {"status": "ok", "connection_id": connection_id}
except Exception as e:
logger.exception("Sync task failed for connection %s", connection_id)
return {"status": "error", "connection_id": connection_id, "error": str(e)}
+
+
+async def _sync_one_celery(connection_id: str, user_id: str) -> None:
+ engine, session_maker = _make_session_maker()
+ try:
+ async with session_maker() as session:
+ await connection_service.sync_connection(session, uuid.UUID(connection_id), uuid.UUID(user_id))
+ finally:
+ await engine.dispose()
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 249fd6c4..0b625a69 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -67,7 +67,8 @@ services:
celery-worker:
<<: *backend-base
- command: celery -A app.worker worker --loglevel=info
+ # Limit to 2 workers to save RAM (default = number of CPUs).
+ command: celery -A app.worker worker --loglevel=info --concurrency=2
celery-beat:
<<: *backend-base
diff --git a/docker-compose.yml b/docker-compose.yml
index 0b9379d5..e95d9630 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -100,7 +100,8 @@ services:
celery-worker:
<<: *backend-base
- command: celery -A app.worker worker --loglevel=info
+ # Limit to 2 workers to save RAM (default = number of CPUs).
+ command: celery -A app.worker worker --loglevel=info --concurrency=2
celery-beat:
<<: *backend-base