Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

41.9 KB
Raw Blame
import pytest
from unittest.mock import patch, MagicMock
from datetime import datetime, timezone, timedelta
from fastapi import HTTPException
import stripe

from agentops.opsboard.services.billing_service import billing_service


def get_org_owner_id(org):
    """Helper to get the owner user ID from an organization."""
    for user_org in org.users:
        if user_org.role == OrgRoles.owner:
            return user_org.user_id
    return None


def setup_mock_request_auth(mock_request, user_id):
    """Helper to ensure mock_request has proper authentication setup."""
    if not hasattr(mock_request, 'state'):
        mock_request.state = MagicMock()
    if not hasattr(mock_request.state, 'session'):
        mock_request.state.session = MagicMock()
    mock_request.state.session.user_id = user_id


from agentops.opsboard.views.billing import BillingDashboardView
from agentops.opsboard.views.orgs import update_member_licenses, UpdateMemberLicensesBody
from agentops.opsboard.models import (
    OrgModel,
    UserOrgModel,
    BillingPeriod,
    BillingAuditLog,
    PremStatus,
    OrgRoles,
    ProjectModel,
)

# Import shared billing fixtures
pytest_plugins = ["tests._conftest.billing"]


# Mock stripe at module level to prevent API key errors
stripe.api_key = 'sk_test_mock'


@pytest.fixture(autouse=True)
def ensure_clean_session(orm_session):
    """Ensure clean session state before each test."""
    try:
        # Check if session has pending rollback
        if orm_session.in_transaction() and orm_session.is_active:
            if hasattr(orm_session, '_transaction') and orm_session._transaction.is_active:
                # Session is in a good state
                pass
        yield
    except Exception:
        orm_session.rollback()
        raise
    finally:
        # Cleanup after test
        try:
            if orm_session.in_transaction():
                orm_session.rollback()
        except Exception:
            pass


@pytest.fixture
def test_billing_members(orm_session, test_pro_org, test_user2, test_user3):
    """Create test members for billing integration tests."""
    members = []

    # Use existing test users to avoid foreign key constraints
    test_users = [test_user2, test_user3]

    for i, user in enumerate(test_users):
        member = UserOrgModel(
            user_id=user.id,
            org_id=test_pro_org.id,
            role=OrgRoles.developer,
            user_email=user.email,
            is_paid=i < 1,  # First one is paid, second one is not
        )
        orm_session.add(member)
        members.append(member)

    orm_session.flush()
    return members


@pytest.fixture(autouse=True)
def mock_stripe_config():
    """Mock Stripe configuration for all tests."""
    with (
        patch('agentops.opsboard.views.orgs.STRIPE_SECRET_KEY', 'sk_test_123'),
        patch('agentops.opsboard.views.orgs.STRIPE_SUBSCRIPTION_PRICE_ID', 'price_test123'),
        patch('agentops.api.environment.STRIPE_SECRET_KEY', 'sk_test_123'),
        patch('agentops.api.environment.STRIPE_SUBSCRIPTION_PRICE_ID', 'price_test123'),
        patch.dict(
            'os.environ',
            {'STRIPE_SECRET_KEY': 'sk_test_123', 'STRIPE_SUBSCRIPTION_PRICE_ID': 'price_test123'},
        ),
    ):
        yield


class TestBillingIntegration:
    """Integration tests for the billing system components."""

    @patch('stripe.Subscription.modify')
    @patch('stripe.Subscription.retrieve')
    async def test_full_billing_workflow_new_member_join(
        self,
        mock_stripe_retrieve,
        mock_stripe_modify,
        mock_request,
        orm_session,
        test_pro_org,
        test_billing_members,
        mock_stripe_subscription,
    ):
        """Test complete billing workflow when a new member joins and gets licensed."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        mock_stripe_retrieve.return_value = mock_stripe_subscription

        unlicensed_member = next(m for m in test_billing_members if not m.is_paid)

        # Step 1: License the member
        body = UpdateMemberLicensesBody(add=[str(unlicensed_member.user_id)], remove=[])

        license_result = await update_member_licenses(
            request=mock_request, org_id=str(test_pro_org.id), body=body, orm=orm_session
        )

        # Verify licensing worked
        assert license_result.paid_members_count == 3  # Owner + 1 existing + newly added

        # Step 2: Check audit log was created
        audit_logs = (
            orm_session.query(BillingAuditLog)
            .filter_by(org_id=test_pro_org.id, action='member_licensed')
            .all()
        )
        assert len(audit_logs) == 1
        assert audit_logs[0].details['member_id'] == str(unlicensed_member.user_id)

        # Step 3: Verify Stripe was called
        mock_stripe_modify.assert_called_once()
        call_args = mock_stripe_modify.call_args
        assert call_args[1]['items'][0]['quantity'] == 3

    @patch('stripe.Subscription.modify')
    @patch('stripe.Subscription.retrieve')
    async def test_full_billing_workflow_member_removal(
        self,
        mock_stripe_retrieve,
        mock_stripe_modify,
        mock_request,
        orm_session,
        test_pro_org,
        test_billing_members,
        mock_stripe_subscription,
    ):
        """Test complete billing workflow when a member is removed and unlicensed."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        mock_stripe_retrieve.return_value = mock_stripe_subscription

        licensed_member = next(m for m in test_billing_members if m.is_paid)

        # Step 1: Remove member license
        body = UpdateMemberLicensesBody(add=[], remove=[str(licensed_member.user_id)])

        license_result = await update_member_licenses(
            request=mock_request, org_id=str(test_pro_org.id), body=body, orm=orm_session
        )

        # Verify unlicensing worked
        assert license_result.paid_members_count == 1  # Only owner remains

        # Step 2: Check audit log was created
        audit_logs = (
            orm_session.query(BillingAuditLog)
            .filter_by(org_id=test_pro_org.id, action='member_unlicensed')
            .all()
        )
        assert len(audit_logs) == 1

        # Step 3: Verify member is marked as unpaid (need to refresh from DB)
        orm_session.refresh(licensed_member)
        assert licensed_member.is_paid is False

    async def test_billing_period_snapshot_creation_integration(
        self, orm_session, test_pro_org, test_billing_members
    ):
        """Test creating billing period snapshots integrates with usage tracking."""
        # Create some projects for usage data
        project = ProjectModel(name="Test Project", org_id=test_pro_org.id)
        orm_session.add(project)
        orm_session.flush()

        period_start = datetime(2024, 1, 1, tzinfo=timezone.utc)
        period_end = datetime(2024, 1, 31, tzinfo=timezone.utc)

        with (
            patch.object(billing_service, 'get_usage_for_period') as mock_get_usage,
            patch.object(billing_service, 'calculate_usage_costs') as mock_calc_costs,
            patch.object(billing_service, 'get_seat_price') as mock_seat_price,
        ):
            mock_get_usage.return_value = {"tokens": 1000000, "spans": 500}
            mock_calc_costs.return_value = {"tokens": 20, "spans": 50}
            mock_seat_price.return_value = 4000

            billing_period = await billing_service.create_billing_period_snapshot(
                orm_session, test_pro_org, period_start, period_end
            )

        # Verify the snapshot
        assert billing_period.org_id == test_pro_org.id
        assert billing_period.seat_count == 2  # Owner + 1 paid member
        assert billing_period.seat_cost == 8000  # 2 * 4000
        assert billing_period.usage_quantities == {"tokens": 1000000, "spans": 500}
        assert billing_period.usage_costs == {"tokens": 20, "spans": 50}
        assert billing_period.total_cost == 8070  # 8000 + 20 + 50

        # Verify it's in the database
        saved_period = orm_session.query(BillingPeriod).filter_by(id=billing_period.id).first()
        assert saved_period is not None

    async def test_billing_dashboard_reflects_member_changes(
        self, mock_request, orm_session, test_pro_org, test_billing_members
    ):
        """Test billing dashboard shows updated costs after member licensing changes."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        dashboard_view = BillingDashboardView(mock_request)

        with (
            patch.object(billing_service, 'get_usage_for_period', return_value={"tokens": 500000}),
            patch.object(billing_service, 'calculate_usage_costs', return_value={"tokens": 100}),
            patch.object(billing_service, 'get_seat_price', return_value=5000),
        ):
            result = await dashboard_view(org_id=str(test_pro_org.id), orm=orm_session)

        # Extract response data if wrapped in JSONResponse
        if hasattr(result, 'body'):
            import json

            result = json.loads(result.body.decode())

        # Verify dashboard reflects current state
        assert result['current_period']['seat_count'] == 2  # Owner + 1 paid member
        assert result['current_period']['seat_cost'] == 10000  # 2 * 5000
        assert result['current_period']['usage_costs'] == {"tokens": 100}
        assert result['current_period']['total_cost'] == 10100  # 10000 + 100

    async def test_billing_audit_logs_track_all_changes(
        self, mock_request, orm_session, test_pro_org, test_billing_members, billing_period_factory
    ):
        """Test audit logs are created for all billing-related changes."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        # Create billing period
        billing_period = billing_period_factory(
            test_pro_org.id,
            total_cost=8000,
        )
        orm_session.add(billing_period)
        orm_session.commit()

        # Create audit log for period creation
        audit_log = BillingAuditLog(
            org_id=test_pro_org.id,
            user_id=mock_request.state.session.user_id,
            action='billing_period_created',
            details={'period_id': str(billing_period.id), 'total_cost': billing_period.total_cost},
        )
        orm_session.add(audit_log)
        orm_session.commit()

        # Verify audit trail
        all_logs = orm_session.query(BillingAuditLog).filter_by(org_id=test_pro_org.id).all()
        assert len(all_logs) >= 1

        period_creation_log = next((log for log in all_logs if log.action == 'billing_period_created'), None)
        assert period_creation_log is not None
        assert period_creation_log.details['period_id'] == str(billing_period.id)

    @patch('stripe.Subscription.modify')
    @patch('stripe.Subscription.retrieve')
    async def test_stripe_integration_member_licensing(
        self,
        mock_stripe_retrieve,
        mock_stripe_modify,
        mock_request,
        orm_session,
        test_pro_org,
        test_billing_members,
        mock_stripe_subscription,
    ):
        """Test Stripe subscription updates when member licensing changes."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        mock_stripe_retrieve.return_value = mock_stripe_subscription

        # Test multiple operations
        unlicensed_member = next(m for m in test_billing_members if not m.is_paid)
        licensed_member = next(m for m in test_billing_members if m.is_paid)

        # Operation 1: Add member
        body1 = UpdateMemberLicensesBody(add=[str(unlicensed_member.user_id)], remove=[])

        await update_member_licenses(
            request=mock_request, org_id=str(test_pro_org.id), body=body1, orm=orm_session
        )

        # Operation 2: Remove member
        body2 = UpdateMemberLicensesBody(add=[], remove=[str(licensed_member.user_id)])

        await update_member_licenses(
            request=mock_request, org_id=str(test_pro_org.id), body=body2, orm=orm_session
        )

        # Verify Stripe was called twice with correct quantities
        assert mock_stripe_modify.call_count == 2

        # First call should increase quantity
        first_call = mock_stripe_modify.call_args_list[0]
        assert first_call[1]['items'][0]['quantity'] == 3  # Added one

        # Second call should decrease quantity
        second_call = mock_stripe_modify.call_args_list[1]
        assert second_call[1]['items'][0]['quantity'] == 2  # Removed one

    async def test_usage_cost_calculation_integration(self, orm_session, test_pro_org, test_billing_members):
        """Test usage cost calculation integrates with billing service and ClickHouse."""
        # Create project for usage
        project = ProjectModel(name="Usage Test Project", org_id=test_pro_org.id)
        orm_session.add(project)
        orm_session.flush()

        # Mock ClickHouse response
        with patch('agentops.opsboard.services.billing_service.get_clickhouse') as mock_clickhouse:
            mock_client = MagicMock()
            mock_result = MagicMock()
            mock_result.result_rows = [(1000, 82000)]  # span_count, total_tokens (50000+25000+5000+2000)
            mock_client.query.return_value = mock_result
            mock_clickhouse.return_value = mock_client

            period_start = datetime(2024, 1, 1)
            period_end = datetime(2024, 1, 31)

            # Get usage data
            usage_data = await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), period_start, period_end
            )

            # Calculate costs
            costs = await billing_service.calculate_usage_costs(usage_data)

        # Verify integration
        assert usage_data == {'tokens': 82000, 'spans': 1000}  # Total from mock data
        assert isinstance(costs, dict)
        assert 'tokens' in costs or 'spans' in costs  # At least one cost type

    async def test_billing_error_handling_integration(
        self, mock_request, orm_session, test_pro_org, test_billing_members, mock_stripe_subscription
    ):
        """Test error handling across billing service, views, and models."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        # Test scenario: Stripe fails during member licensing
        with (
            patch('stripe.Subscription.retrieve') as mock_retrieve,
            patch('stripe.Subscription.modify') as mock_modify,
        ):
            # Ensure the mock subscription has the correct price ID to pass legacy check
            mock_stripe_subscription.items.data[0]['price']['id'] = 'price_test123'
            mock_retrieve.return_value = mock_stripe_subscription

            # Make Stripe fail
            mock_modify.side_effect = stripe.error.StripeError("Card declined")

            unlicensed_member = next(m for m in test_billing_members if not m.is_paid)

            body = UpdateMemberLicensesBody(add=[str(unlicensed_member.user_id)], remove=[])

            # Should raise HTTPException from Stripe error
            with pytest.raises(HTTPException) as excinfo:
                await update_member_licenses(
                    request=mock_request, org_id=str(test_pro_org.id), body=body, orm=orm_session
                )

            # Verify error is properly handled
            assert excinfo.value.status_code == 500
            assert "Failed to update subscription" in excinfo.value.detail

            # Verify the error was logged
            # Since the transaction raises an exception, the database changes are rolled back
            # and we can't easily verify the state without complex session management

    async def test_billing_cache_integration(self, orm_session, test_pro_org, test_billing_members):
        """Test billing service caching works correctly with real data."""
        # Create project
        project = ProjectModel(name="Cache Test Project", org_id=test_pro_org.id)
        orm_session.add(project)
        orm_session.flush()

        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        with patch('agentops.opsboard.services.billing_service.get_clickhouse') as mock_clickhouse:
            mock_client = MagicMock()
            mock_result = MagicMock()
            mock_result.result_rows = [(500, 40000)]  # span_count, total_tokens (25000+12000+2000+1000)
            mock_client.query.return_value = mock_result
            mock_clickhouse.return_value = mock_client

            # First call should hit ClickHouse
            usage_data1 = await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), period_start, period_end
            )

            # Second call should use cache
            usage_data2 = await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), period_start, period_end
            )

        # Verify caching worked
        assert usage_data1 == usage_data2
        assert mock_client.query.call_count == 1  # Only called once due to caching

    @patch('stripe.Subscription.modify')
    @patch('stripe.Subscription.retrieve')
    async def test_concurrent_member_licensing_operations(
        self,
        mock_stripe_retrieve,
        mock_stripe_modify,
        mock_request,
        orm_session,
        test_pro_org,
        test_billing_members,
        mock_stripe_subscription,
    ):
        """Test concurrent member licensing operations don't cause data corruption."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        mock_stripe_retrieve.return_value = mock_stripe_subscription

        # This test simulates what would happen with concurrent operations
        # In practice, the with_for_update() lock should prevent issues

        unlicensed_member = next(m for m in test_billing_members if not m.is_paid)

        body = UpdateMemberLicensesBody(add=[str(unlicensed_member.user_id)], remove=[])

        # Simulate first operation
        result1 = await update_member_licenses(
            request=mock_request, org_id=str(test_pro_org.id), body=body, orm=orm_session
        )

        # Verify state is consistent
        assert result1.paid_members_count == 3

        # Verify database state
        updated_member = orm_session.query(UserOrgModel).filter_by(user_id=unlicensed_member.user_id).first()
        assert updated_member.is_paid is True


class TestBillingWorkflows:
    """Test common billing workflows end-to-end."""

    async def test_org_upgrade_to_pro_workflow(self, mock_request, orm_session, test_user):
        """Test complete workflow of upgrading an org to pro status."""
        # Create free org
        free_org = OrgModel(name="Free Org", prem_status=PremStatus.free)
        orm_session.add(free_org)
        orm_session.flush()

        # Add user as owner
        user_org = UserOrgModel(
            user_id=test_user.id,
            org_id=free_org.id,
            role=OrgRoles.owner,
            user_email=test_user.email,
            is_paid=False,
        )
        orm_session.add(user_org)
        orm_session.flush()

        # Simulate upgrade process
        free_org.prem_status = PremStatus.pro
        free_org.subscription_id = "sub_new_upgrade"
        user_org.is_paid = True
        orm_session.commit()

        # Verify upgrade
        assert free_org.prem_status == PremStatus.pro
        assert free_org.subscription_id is not None
        assert user_org.is_paid is True

    async def test_monthly_billing_period_closure_workflow(
        self, orm_session, test_pro_org, test_billing_members
    ):
        """Test monthly billing period closure and snapshot creation."""
        # Create project for usage
        project = ProjectModel(name="Monthly Test Project", org_id=test_pro_org.id)
        orm_session.add(project)
        orm_session.flush()

        # Set up billing period
        period_start = datetime(2024, 1, 1, tzinfo=timezone.utc)
        period_end = datetime(2024, 1, 31, tzinfo=timezone.utc)

        with (
            patch.object(billing_service, 'get_usage_for_period') as mock_usage,
            patch.object(billing_service, 'calculate_usage_costs') as mock_costs,
            patch.object(billing_service, 'get_seat_price') as mock_price,
        ):
            mock_usage.return_value = {"tokens": 2000000, "spans": 1000}
            mock_costs.return_value = {"tokens": 40, "spans": 100}
            mock_price.return_value = 4000

            # Create snapshot (simulates monthly closure)
            snapshot = await billing_service.create_billing_period_snapshot(
                orm_session, test_pro_org, period_start, period_end
            )

        # Verify snapshot
        assert snapshot.period_start == period_start
        assert snapshot.period_end == period_end
        assert snapshot.seat_count == 2  # Owner + 1 paid member
        assert snapshot.total_cost == 8140  # (2 * 4000) + 40 + 100
        assert snapshot.status == 'pending'

    async def test_member_invitation_auto_licensing_workflow(
        self, mock_request, orm_session, test_pro_org, test_user, test_user3
    ):
        """Test workflow of inviting a member and auto-licensing them."""
        # Use existing test_user3 instead of creating new user
        # Simulate invitation acceptance and auto-licensing
        new_member = UserOrgModel(
            user_id=test_user3.id,
            org_id=test_pro_org.id,
            role=OrgRoles.developer,
            user_email=test_user3.email,
            is_paid=True,  # Auto-licensed
        )
        orm_session.add(new_member)
        orm_session.commit()

        # Create audit log for auto-licensing
        audit_log = BillingAuditLog(
            org_id=test_pro_org.id,
            user_id=test_user3.id,
            action='member_auto_licensed_on_invite_accept',
            details={
                'member_id': str(test_user3.id),
                'member_email': test_user3.email,
                'invite_role': 'developer',
            },
        )
        orm_session.add(audit_log)
        orm_session.commit()

        # Verify workflow
        assert new_member.is_paid is True

        # Verify audit log
        audit_logs = (
            orm_session.query(BillingAuditLog)
            .filter_by(org_id=test_pro_org.id, action='member_auto_licensed_on_invite_accept')
            .all()
        )
        assert len(audit_logs) == 1

    async def test_legacy_to_new_billing_migration_workflow(
        self, mock_request, orm_session, test_pro_org, test_billing_members
    ):
        """Test migration from legacy billing to new seat-based billing."""
        # Simulate legacy billing state
        test_pro_org.subscription_id = "sub_legacy_migration"

        # Set all members as unpaid initially (legacy state)
        for member in test_billing_members:
            member.is_paid = False

        # Get owner
        owner = orm_session.query(UserOrgModel).filter_by(org_id=test_pro_org.id, role=OrgRoles.owner).first()
        owner.is_paid = False

        orm_session.commit()

        # Simulate migration: auto-license all existing members
        all_members = orm_session.query(UserOrgModel).filter_by(org_id=test_pro_org.id).all()

        for member in all_members:
            member.is_paid = True

        orm_session.commit()

        # Verify migration
        paid_members = orm_session.query(UserOrgModel).filter_by(org_id=test_pro_org.id, is_paid=True).count()
        assert paid_members == 3  # Owner + 2 members

    @patch('stripe.Subscription.retrieve')
    async def test_subscription_cancellation_workflow(
        self, mock_stripe_retrieve, mock_request, orm_session, test_pro_org, test_billing_members
    ):
        """Test workflow when subscription is cancelled."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        # Mock cancelled subscription for member licensing check
        cancelled_subscription = MagicMock()
        cancelled_subscription.cancel_at_period_end = True
        cancelled_subscription.current_period_end = int((datetime.now() + timedelta(days=7)).timestamp())

        def mock_get(key, default=None):
            if key == 'cancel_at_period_end':
                return True
            elif key == 'current_period_end':
                return cancelled_subscription.current_period_end
            elif key == 'items':
                return {'data': [{'price': {'id': 'price_legacy_123'}}]}
            return default

        cancelled_subscription.get = mock_get
        mock_stripe_retrieve.return_value = cancelled_subscription

        # Test that member licensing is blocked for cancelled subscription
        body = UpdateMemberLicensesBody(add=[], remove=[])

        with pytest.raises(HTTPException) as excinfo:
            await update_member_licenses(
                request=mock_request, org_id=str(test_pro_org.id), body=body, orm=orm_session
            )

        assert excinfo.value.status_code == 400
        assert "subscription is scheduled to cancel" in excinfo.value.detail

        # This test primarily verifies that cancelled subscriptions prevent seat management,
        # which is the core functionality we care about for the cancellation workflow


class TestBillingDataConsistency:
    """Test data consistency across billing components."""

    async def test_seat_count_consistency_across_components(
        self, orm_session, test_pro_org, test_billing_members
    ):
        """Test seat counts are consistent between service, models, and Stripe."""
        # Count paid members in database
        db_paid_count = (
            orm_session.query(UserOrgModel).filter_by(org_id=test_pro_org.id, is_paid=True).count()
        )

        # Should be consistent with owner + paid members
        assert db_paid_count >= 1  # At least the owner

    async def test_usage_data_consistency_billing_dashboard(
        self, orm_session, test_pro_org, test_billing_members
    ):
        """Test usage data consistency between service and dashboard view."""
        # Create project for usage
        project = ProjectModel(name="Consistency Test", org_id=test_pro_org.id)
        orm_session.add(project)
        orm_session.flush()

        test_usage = {"tokens": 1500000, "spans": 750}
        test_costs = {"tokens": 30, "spans": 75}

        with (
            patch.object(billing_service, 'get_usage_for_period', return_value=test_usage),
            patch.object(billing_service, 'calculate_usage_costs', return_value=test_costs),
            patch.object(billing_service, 'get_seat_price', return_value=4000),
        ):
            # Get data from service directly
            period_start = datetime(2024, 1, 1)
            period_end = datetime(2024, 1, 31)

            service_usage = await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), period_start, period_end
            )
            service_costs = await billing_service.calculate_usage_costs(service_usage)

            # Get data from dashboard view
            dashboard_view = BillingDashboardView(MagicMock())
            dashboard_view.request.state.session.user_id = "00000000-0000-0000-0000-000000000000"

            dashboard_result = await dashboard_view(org_id=str(test_pro_org.id), orm=orm_session)

            # Extract response data if wrapped in JSONResponse
            if hasattr(dashboard_result, 'body'):
                import json

                dashboard_result = json.loads(dashboard_result.body.decode())

        # Verify consistency
        assert service_usage == test_usage
        assert service_costs == test_costs
        assert dashboard_result['current_period']['usage_quantities'] == test_usage
        assert dashboard_result['current_period']['usage_costs'] == test_costs

    async def test_cost_calculation_consistency(self, orm_session, test_pro_org, test_billing_members):
        """Test cost calculations are consistent across all billing components."""
        # Test data
        usage_quantities = {"tokens": 2000000, "spans": 1000}

        # Calculate costs multiple times
        with patch.object(billing_service, 'get_usage_pricing') as mock_pricing:
            from decimal import Decimal

            mock_pricing.return_value = {
                'tokens': {'price_per_unit': Decimal('0.00002'), 'unit_size': 1000},
                'spans': {'price_per_unit': Decimal('0.001'), 'unit_size': 1},
            }

            costs1 = await billing_service.calculate_usage_costs(usage_quantities)
            costs2 = await billing_service.calculate_usage_costs(usage_quantities)
            costs3 = await billing_service.calculate_usage_costs(usage_quantities)

        # All calculations should be identical
        assert costs1 == costs2 == costs3

        # Verify expected calculations
        # Tokens: 2M / 1000 * 0.00002 = 0.04 = 4 cents
        # Spans: 1000 * 0.001 = 1.00 = 100 cents
        assert costs1.get('tokens', 0) == 4
        assert costs1.get('spans', 0) == 100

    async def test_audit_log_completeness(
        self, mock_request, orm_session, test_pro_org, test_billing_members, billing_period_factory
    ):
        """Test all billing actions generate appropriate audit log entries."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        initial_log_count = orm_session.query(BillingAuditLog).filter_by(org_id=test_pro_org.id).count()

        # Perform several billing operations

        # 1. Create billing period
        billing_period = billing_period_factory(
            test_pro_org.id,
            total_cost=5000,
        )
        orm_session.add(billing_period)
        orm_session.commit()

        # Add audit log for period creation
        period_log = BillingAuditLog(
            org_id=test_pro_org.id,
            user_id=mock_request.state.session.user_id,
            action='billing_period_created',
            details={'period_id': str(billing_period.id)},
        )
        orm_session.add(period_log)

        # 2. Create member licensing log
        member_log = BillingAuditLog(
            org_id=test_pro_org.id,
            user_id=mock_request.state.session.user_id,
            action='member_licensed',
            details={'member_id': str(test_billing_members[0].user_id)},
        )
        orm_session.add(member_log)

        orm_session.commit()

        # Verify audit logs were created
        final_log_count = orm_session.query(BillingAuditLog).filter_by(org_id=test_pro_org.id).count()
        assert final_log_count == initial_log_count + 2

        # Verify log content
        logs = orm_session.query(BillingAuditLog).filter_by(org_id=test_pro_org.id).all()
        actions = [log.action for log in logs]
        assert 'billing_period_created' in actions
        assert 'member_licensed' in actions


class TestBillingPerformance:
    """Test performance aspects of the billing system."""

    async def test_billing_dashboard_query_performance(self, orm_session, test_pro_org, test_billing_members):
        """Test billing dashboard queries perform efficiently with large datasets."""
        # Create multiple billing periods
        periods = []
        import time

        base_day = int(time.time() % 20) + 1

        for i in range(20):  # Simulate 20 months of data
            unique_day = base_day + (i % 5)  # Vary days to avoid conflicts
            period = BillingPeriod(
                org_id=test_pro_org.id,
                period_start=datetime(2024, 1, unique_day, tzinfo=timezone.utc) + timedelta(days=30 * i),
                period_end=datetime(2024, 1, unique_day + 1, tzinfo=timezone.utc) + timedelta(days=30 * i),
                seat_cost=4000,
                total_cost=4200,
                status='paid',
            )
            periods.append(period)

        orm_session.add_all(periods)
        orm_session.commit()

        # Test dashboard performance
        dashboard_view = BillingDashboardView(MagicMock())
        dashboard_view.request.state.session.user_id = "00000000-0000-0000-0000-000000000000"

        with (
            patch.object(billing_service, 'get_usage_for_period', return_value={}),
            patch.object(billing_service, 'calculate_usage_costs', return_value={}),
            patch.object(billing_service, 'get_seat_price', return_value=4000),
        ):
            start_time = datetime.now()
            result = await dashboard_view(org_id=str(test_pro_org.id), orm=orm_session)
            end_time = datetime.now()

        # Verify reasonable performance (should be under 1 second for this dataset)
        execution_time = (end_time - start_time).total_seconds()
        assert execution_time < 1.0

        # Extract response data if wrapped in JSONResponse
        if hasattr(result, 'body'):
            import json

            result = json.loads(result.body.decode())

        # Verify correct data limiting (only 12 past periods returned)
        assert len(result['past_periods']) <= 12

    async def test_usage_calculation_performance(self, orm_session, test_pro_org, test_billing_members):
        """Test usage calculation performance with large amounts of data."""
        # Create multiple projects
        projects = []
        for i in range(10):
            project = ProjectModel(name=f"Performance Test Project {i}", org_id=test_pro_org.id)
            projects.append(project)

        orm_session.add_all(projects)
        orm_session.flush()

        # Mock large dataset response
        with patch('agentops.opsboard.services.billing_service.get_clickhouse') as mock_clickhouse:
            mock_client = MagicMock()
            mock_result = MagicMock()
            # Simulate large usage numbers
            mock_result.result_rows = [
                (100000, 8200000)
            ]  # span_count, total_tokens (5000000+2500000+500000+200000)
            mock_client.query.return_value = mock_result
            mock_clickhouse.return_value = mock_client

            start_time = datetime.now()
            usage_data = await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), datetime(2024, 1, 1), datetime(2024, 1, 31)
            )
            end_time = datetime.now()

        # Verify performance
        execution_time = (end_time - start_time).total_seconds()
        assert execution_time < 0.5  # Should be very fast

        # Verify data
        assert usage_data['tokens'] == 8200000  # Sum of all token types
        assert usage_data['spans'] == 100000

    async def test_billing_service_cache_performance(self, orm_session, test_pro_org, test_billing_members):
        """Test billing service caching improves performance."""
        # Create project
        project = ProjectModel(name="Cache Performance Test", org_id=test_pro_org.id)
        orm_session.add(project)
        orm_session.flush()

        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        with patch('agentops.opsboard.services.billing_service.get_clickhouse') as mock_clickhouse:
            mock_client = MagicMock()
            mock_result = MagicMock()
            mock_result.result_rows = [(1000, 82000)]  # span_count, total_tokens (50000+25000+5000+2000)
            mock_client.query.return_value = mock_result
            mock_clickhouse.return_value = mock_client

            # First call - should hit database
            start_time1 = datetime.now()
            await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), period_start, period_end
            )
            end_time1 = datetime.now()

            # Second call - should use cache
            start_time2 = datetime.now()
            await billing_service.get_usage_for_period(
                orm_session, str(test_pro_org.id), period_start, period_end
            )
            end_time2 = datetime.now()

        # Cache should be faster
        time1 = (end_time1 - start_time1).total_seconds()
        time2 = (end_time2 - start_time2).total_seconds()

        # Second call should be significantly faster (cache hit)
        assert time2 < time1
        assert mock_client.query.call_count == 1  # Only called once


class TestBillingEdgeCases:
    """Test edge cases in the billing system."""

    async def test_billing_with_zero_usage(self, orm_session, test_pro_org, test_billing_members):
        """Test billing calculations work correctly with zero usage."""
        with (
            patch.object(billing_service, 'get_usage_for_period', return_value={}),
            patch.object(billing_service, 'calculate_usage_costs', return_value={}),
            patch.object(billing_service, 'get_seat_price', return_value=4000),
        ):
            billing_period = await billing_service.create_billing_period_snapshot(
                orm_session,
                test_pro_org,
                datetime(2024, 1, 1, tzinfo=timezone.utc),
                datetime(2024, 1, 31, tzinfo=timezone.utc),
            )

        # Verify zero usage is handled correctly
        assert billing_period.usage_quantities == {}
        assert billing_period.usage_costs == {}
        assert billing_period.seat_cost == 8000  # 2 * 4000 (owner + 1 paid member)
        assert billing_period.total_cost == 8000  # Only seat cost

    async def test_billing_with_single_member_org(self, orm_session, test_user3):
        """Test billing works correctly for single-member organizations."""
        # Create single-member org
        single_org = OrgModel(
            name="Single Member Org", prem_status=PremStatus.pro, subscription_id="sub_single"
        )
        orm_session.add(single_org)
        orm_session.flush()

        owner = UserOrgModel(
            user_id=test_user3.id,
            org_id=single_org.id,
            role=OrgRoles.owner,
            user_email=test_user3.email,
            is_paid=True,
        )
        orm_session.add(owner)
        orm_session.flush()

        with (
            patch.object(billing_service, 'get_usage_for_period', return_value={"tokens": 100000}),
            patch.object(billing_service, 'calculate_usage_costs', return_value={"tokens": 2}),
            patch.object(billing_service, 'get_seat_price', return_value=4000),
        ):
            billing_period = await billing_service.create_billing_period_snapshot(
                orm_session,
                single_org,
                datetime(2024, 1, 1, tzinfo=timezone.utc),
                datetime(2024, 1, 31, tzinfo=timezone.utc),
            )

        # Verify single member billing
        assert billing_period.seat_count == 1
        assert billing_period.seat_cost == 4000
        assert billing_period.usage_costs == {"tokens": 2}
        assert billing_period.total_cost == 4002

    async def test_billing_during_org_deletion(
        self, mock_request, orm_session, test_pro_org, test_billing_members, billing_period_factory
    ):
        """Test billing data handling when organization is deleted."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        org_id = test_pro_org.id

        # Create billing data
        billing_period = billing_period_factory(
            org_id,
            total_cost=5000,
        )
        orm_session.add(billing_period)

        audit_log = BillingAuditLog(
            org_id=org_id, user_id=mock_request.state.session.user_id, action='test_action', details={}
        )
        orm_session.add(audit_log)
        orm_session.commit()

        # Delete organization
        orm_session.delete(test_pro_org)

        try:
            orm_session.commit()

            # Check if billing data was cascade deleted or preserved
            remaining_periods = orm_session.query(BillingPeriod).filter_by(org_id=org_id).all()
            remaining_logs = orm_session.query(BillingAuditLog).filter_by(org_id=org_id).all()

            # Behavior depends on database foreign key constraints
            # This test documents the expected behavior

        except Exception:
            # If foreign key constraints prevent deletion, that's also valid
            orm_session.rollback()

    async def test_billing_with_invalid_stripe_data(
        self, mock_request, orm_session, test_pro_org, test_billing_members
    ):
        """Test billing system handles invalid or corrupted Stripe data."""
        setup_mock_request_auth(mock_request, get_org_owner_id(test_pro_org))
        dashboard_view = BillingDashboardView(mock_request)

        # Test with invalid subscription ID
        test_pro_org.subscription_id = "sub_invalid_123"
        orm_session.flush()

        with patch('stripe.Subscription.retrieve') as mock_retrieve:
            mock_retrieve.side_effect = stripe.error.InvalidRequestError(
                "No such subscription", "subscription"
            )

            with (
                patch.object(billing_service, 'get_usage_for_period', return_value={}),
                patch.object(billing_service, 'calculate_usage_costs', return_value={}),
                patch.object(billing_service, 'get_seat_price', return_value=4000),
            ):
                # Should handle error gracefully
                result = await dashboard_view(org_id=str(test_pro_org.id), orm=orm_session)

        # Extract response data if wrapped in JSONResponse
        if hasattr(result, 'body'):
            import json

            result = json.loads(result.body.decode())

        # Verify it still returns data (fallback behavior)
        assert result['current_period'] is not None
        assert result['is_legacy_billing'] is False  # Default value

    async def test_billing_timezone_handling(
        self, orm_session, test_pro_org, test_billing_members, billing_period_factory
    ):
        """Test billing system correctly handles different timezones."""
        # Test with different timezone periods
        utc_start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
        utc_end = datetime(2024, 1, 31, 23, 59, 59, tzinfo=timezone.utc)

        # Create billing period with UTC times
        billing_period = billing_period_factory(
            test_pro_org.id,
            seat_cost=4000,
            total_cost=4000,
        )
        orm_session.add(billing_period)
        orm_session.commit()

        # Verify timezone-aware storage and retrieval
        retrieved_period = orm_session.query(BillingPeriod).filter_by(id=billing_period.id).first()

        assert retrieved_period.period_start.tzinfo is not None
        assert retrieved_period.period_end.tzinfo is not None