Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

25.2 KB
Raw Blame
import pytest
from unittest.mock import patch, MagicMock
from datetime import datetime, timedelta
from decimal import Decimal
import os

from agentops.opsboard.services.billing_service import BillingService
from agentops.opsboard.models import ProjectModel, BillingPeriod
from agentops.common.usage_tracking import UsageType


# Mock Stripe environment variables for testing
@pytest.fixture(autouse=True)
def mock_stripe_env_vars():
    with (
        patch.dict(
            os.environ,
            {
                'STRIPE_SECRET_KEY': 'sk_test_mock_key',
                'STRIPE_SUBSCRIPTION_PRICE_ID': 'price_test_subscription',
                'STRIPE_TOKEN_PRICE_ID': 'price_test_token',
                'STRIPE_SPAN_PRICE_ID': 'price_test_span',
            },
        ),
        patch('stripe.Account.retrieve'),
    ):
        yield


@pytest.fixture
def billing_service():
    """Create a billing service instance for testing."""
    # Mock the Stripe validation to avoid API calls during initialization
    with patch('stripe.Account.retrieve'):
        return BillingService()


@pytest.fixture
def mock_stripe_price():
    """Mock Stripe price object."""
    mock_price = MagicMock()
    mock_price.unit_amount = 2000  # $20.00 in cents
    mock_price.transform_quantity = None
    return mock_price


@pytest.fixture
def mock_clickhouse_client():
    """Mock ClickHouse client for usage queries."""
    mock_client = MagicMock()
    mock_result = MagicMock()
    mock_result.result_rows = [
        (1000, 82000)  # span_count, total_tokens (50000+25000+5000+2000)
    ]
    mock_client.query.return_value = mock_result
    return mock_client


class TestBillingService:
    """Test cases for BillingService class."""

    def test_init(self):
        """Test BillingService initialization."""
        service = BillingService()

        assert service._pricing_cache is None
        assert service._cache_timestamp is None
        assert service._cache_duration == 3600  # 1 hour
        assert service._usage_cache == {}
        assert service._usage_cache_ttl == 300  # 5 minutes

    def test_should_refresh_cache_when_no_cache(self, billing_service):
        """Test _should_refresh_cache returns True when no cache exists."""
        # Ensure no cache exists
        billing_service._pricing_cache = None
        billing_service._cache_timestamp = None

        assert billing_service._should_refresh_cache() is True

    def test_should_refresh_cache_when_cache_expired(self, billing_service):
        """Test _should_refresh_cache returns True when cache is expired."""
        # Set cache with expired timestamp
        billing_service._pricing_cache = {"test": "data"}
        billing_service._cache_timestamp = datetime.now() - timedelta(seconds=3700)  # Older than 1 hour

        assert billing_service._should_refresh_cache() is True

    def test_should_refresh_cache_when_cache_valid(self, billing_service):
        """Test _should_refresh_cache returns False when cache is still valid."""
        # Set cache with recent timestamp
        billing_service._pricing_cache = {"test": "data"}
        billing_service._cache_timestamp = datetime.now() - timedelta(seconds=1800)  # 30 minutes ago

        assert billing_service._should_refresh_cache() is False

    @patch('agentops.opsboard.services.billing_service.STRIPE_SPAN_PRICE_ID', 'price_span123')
    @patch('agentops.opsboard.services.billing_service.STRIPE_TOKEN_PRICE_ID', 'price_token123')
    @patch('stripe.Price.retrieve')
    def test_get_usage_pricing_with_stripe_success(self, mock_price_retrieve, billing_service):
        """Test get_usage_pricing successfully fetches from Stripe."""
        # Clear cache to ensure fresh call
        billing_service._pricing_cache = None
        billing_service._cache_timestamp = None

        # Mock Stripe price objects
        token_price = MagicMock()
        token_price.unit_amount = 2  # $0.02 in cents for 1000 tokens
        token_price.unit_amount_decimal = None
        token_price.custom_unit_amount = None
        token_price.tiers = None
        token_price.currency_options = None
        token_price.transform_quantity = MagicMock()
        token_price.transform_quantity.divide_by = 1000

        span_price = MagicMock()
        span_price.unit_amount = 100  # $1.00 in cents per span
        span_price.unit_amount_decimal = None
        span_price.custom_unit_amount = None
        span_price.tiers = None
        span_price.currency_options = None
        span_price.transform_quantity = None

        mock_price_retrieve.side_effect = [token_price, span_price]

        pricing = billing_service.get_usage_pricing()

        assert UsageType.TOKENS in pricing
        assert UsageType.SPANS in pricing

        # Check token pricing - unit_amount 2 becomes 2/100 = 0.02 dollars
        token_config = pricing[UsageType.TOKENS]
        assert token_config['price_per_unit'] == Decimal('0.02')
        assert token_config['unit_size'] == 1000
        assert token_config['display_unit'] == 'thousand tokens'
        assert token_config['stripe_price_id'] == 'price_token123'

        # Check span pricing
        span_config = pricing[UsageType.SPANS]
        assert span_config['price_per_unit'] == Decimal('1.00')
        assert span_config['unit_size'] == 1000
        assert span_config['display_unit'] == 'thousand spans'
        assert span_config['stripe_price_id'] == 'price_span123'

    @patch('stripe.Price.retrieve')
    def test_get_usage_pricing_with_stripe_error(self, mock_price_retrieve, billing_service):
        """Test get_usage_pricing falls back to defaults when Stripe fails."""
        import stripe

        mock_price_retrieve.side_effect = stripe.error.StripeError("API Error")

        pricing = billing_service.get_usage_pricing()

        # Should return default values
        assert UsageType.TOKENS in pricing
        assert UsageType.SPANS in pricing

        token_config = pricing[UsageType.TOKENS]
        assert token_config['price_per_unit'] == Decimal('0.0002')
        assert token_config['unit_size'] == 1000
        assert token_config['display_unit'] == 'thousand tokens'

        span_config = pricing[UsageType.SPANS]
        assert span_config['price_per_unit'] == Decimal('0.0001')
        assert span_config['unit_size'] == 1000
        assert span_config['display_unit'] == 'thousand spans'

    @patch('agentops.opsboard.services.billing_service.STRIPE_SPAN_PRICE_ID', 'price_span123')
    @patch('agentops.opsboard.services.billing_service.STRIPE_TOKEN_PRICE_ID', 'price_token123')
    def test_get_usage_pricing_with_env_vars(self, billing_service):
        """Test get_usage_pricing uses environment variable price IDs."""
        # Clear cache to ensure fresh call
        billing_service._pricing_cache = None
        billing_service._cache_timestamp = None

        with patch('stripe.Price.retrieve') as mock_retrieve:
            token_price = MagicMock()
            token_price.unit_amount = 5
            token_price.unit_amount_decimal = None
            token_price.custom_unit_amount = None
            token_price.tiers = None
            token_price.currency_options = None
            token_price.transform_quantity = None

            span_price = MagicMock()
            span_price.unit_amount = 150
            span_price.unit_amount_decimal = None
            span_price.custom_unit_amount = None
            span_price.tiers = None
            span_price.currency_options = None
            span_price.transform_quantity = None

            mock_retrieve.side_effect = [token_price, span_price]

            billing_service.get_usage_pricing()

            # Verify that Stripe was called with the environment variable price IDs
            assert mock_retrieve.call_count == 2
            mock_retrieve.assert_any_call('price_token123', expand=['currency_options', 'tiers'])
            mock_retrieve.assert_any_call('price_span123', expand=['currency_options', 'tiers'])

    def test_get_usage_pricing_returns_cached_values(self, billing_service):
        """Test get_usage_pricing returns cached values when cache is valid."""
        # Set up valid cache
        cached_pricing = {
            UsageType.TOKENS: {'price_per_unit': Decimal('0.01'), 'unit_size': 1000},
            UsageType.SPANS: {'price_per_unit': Decimal('0.002'), 'unit_size': 1},
        }
        billing_service._pricing_cache = cached_pricing
        billing_service._cache_timestamp = datetime.now()

        with patch('stripe.Price.retrieve') as mock_retrieve:
            pricing = billing_service.get_usage_pricing()

            # Should return cached values without calling Stripe
            assert pricing == cached_pricing
            mock_retrieve.assert_not_called()

    @patch('agentops.opsboard.services.billing_service.STRIPE_SUBSCRIPTION_PRICE_ID', 'price_sub123')
    @patch('stripe.Price.retrieve')
    def test_get_seat_price_from_stripe_success(self, mock_price_retrieve, billing_service):
        """Test get_seat_price successfully fetches from Stripe."""
        mock_price = MagicMock()
        mock_price.unit_amount = 5000  # $50.00 in cents
        mock_price.unit_amount_decimal = None
        mock_price.custom_unit_amount = None
        mock_price.tiers = None
        mock_price.currency_options = None
        mock_price.recurring = MagicMock()
        mock_price.recurring.usage_type = 'licensed'

        mock_price_retrieve.return_value = mock_price

        seat_price = billing_service.get_seat_price()

        assert seat_price == 5000
        mock_price_retrieve.assert_called_once_with('price_sub123', expand=['currency_options', 'tiers'])

    @patch('stripe.Price.retrieve')
    def test_get_seat_price_stripe_error_fallback(self, mock_price_retrieve, billing_service):
        """Test get_seat_price falls back to environment variable when Stripe fails."""
        import stripe

        mock_price_retrieve.side_effect = stripe.error.StripeError("API Error")

        with patch.dict(os.environ, {'STRIPE_SEAT_PRICE_CENTS': '6000'}):
            seat_price = billing_service.get_seat_price()

        assert seat_price == 6000

    @patch.dict('os.environ', {'STRIPE_SEAT_PRICE_CENTS': '5000'})
    def test_get_seat_price_from_env_var(self, billing_service):
        """Test get_seat_price uses environment variable when no Stripe config."""
        # No STRIPE_SUBSCRIPTION_PRICE_ID set
        with patch.dict(os.environ, {}, clear=True):
            os.environ['STRIPE_SEAT_PRICE_CENTS'] = '5000'
            seat_price = billing_service.get_seat_price()

        assert seat_price == 5000

    @patch('agentops.opsboard.services.billing_service.get_clickhouse')
    async def test_get_usage_for_period_success(
        self, mock_get_clickhouse, billing_service, orm_session, test_org, mock_clickhouse_client
    ):
        """Test get_usage_for_period successfully queries ClickHouse."""
        # Create test projects for the org
        project1 = ProjectModel(name="Test Project 1", org_id=test_org.id)
        project2 = ProjectModel(name="Test Project 2", org_id=test_org.id)
        orm_session.add_all([project1, project2])
        orm_session.flush()

        # Mock ClickHouse response
        mock_get_clickhouse.return_value = mock_clickhouse_client

        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        result = await billing_service.get_usage_for_period(
            orm_session, str(test_org.id), period_start, period_end
        )

        assert result == {'tokens': 82000, 'spans': 1000}  # 50000+25000+5000+2000 tokens, 1000 spans
        mock_clickhouse_client.query.assert_called_once()

    async def test_get_usage_for_period_no_projects(self, billing_service, orm_session, test_org):
        """Test get_usage_for_period returns empty when org has no projects."""
        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        result = await billing_service.get_usage_for_period(
            orm_session, str(test_org.id), period_start, period_end
        )

        assert result == {}

    @patch('agentops.opsboard.services.billing_service.get_clickhouse')
    async def test_get_usage_for_period_clickhouse_error(
        self, mock_get_clickhouse, billing_service, orm_session, test_org
    ):
        """Test get_usage_for_period handles ClickHouse errors gracefully."""
        # Create test project
        project = ProjectModel(name="Test Project", org_id=test_org.id)
        orm_session.add(project)
        orm_session.flush()

        # Mock ClickHouse to raise an error
        mock_client = MagicMock()
        mock_client.query.side_effect = Exception("ClickHouse connection failed")
        mock_get_clickhouse.return_value = mock_client

        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        result = await billing_service.get_usage_for_period(
            orm_session, str(test_org.id), period_start, period_end
        )

        assert result == {}

    async def test_get_usage_for_period_uses_cache(self, billing_service, orm_session, test_org):
        """Test get_usage_for_period returns cached data when available."""
        from datetime import timezone

        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        # Convert to UTC as the implementation does
        period_start_utc = period_start.replace(tzinfo=timezone.utc)
        period_end_utc = period_end.replace(tzinfo=timezone.utc)

        cache_key = f"{test_org.id}:{period_start_utc.isoformat()}:{period_end_utc.isoformat()}:all"

        # Set cache data
        cached_data = {'tokens': 5000, 'spans': 100}
        billing_service._usage_cache[cache_key] = (cached_data, datetime.now())

        result = await billing_service.get_usage_for_period(
            orm_session, str(test_org.id), period_start, period_end
        )

        assert result == cached_data

    async def test_get_usage_for_period_cache_expiry(
        self, billing_service, orm_session, test_org, mock_clickhouse_client
    ):
        """Test get_usage_for_period refreshes data when cache is expired."""
        from datetime import timezone

        period_start = datetime(2024, 1, 1)
        period_end = datetime(2024, 1, 31)

        # Convert to UTC as the implementation does
        period_start_utc = period_start.replace(tzinfo=timezone.utc)
        period_end_utc = period_end.replace(tzinfo=timezone.utc)

        cache_key = f"{test_org.id}:{period_start_utc.isoformat()}:{period_end_utc.isoformat()}:all"

        # Set expired cache data
        expired_data = {'tokens': 1000, 'spans': 50}
        expired_time = datetime.now() - timedelta(seconds=400)  # Older than 5 minutes
        billing_service._usage_cache[cache_key] = (expired_data, expired_time)

        with patch('agentops.opsboard.services.billing_service.get_clickhouse') as mock_get_clickhouse:
            mock_get_clickhouse.return_value = mock_clickhouse_client

            # Create test project
            project = ProjectModel(name="Test Project", org_id=test_org.id)
            orm_session.add(project)
            orm_session.flush()

            result = await billing_service.get_usage_for_period(
                orm_session, str(test_org.id), period_start, period_end
            )

            # Should get fresh data, not expired cache
            assert result == {'tokens': 82000, 'spans': 1000}

    async def test_calculate_usage_costs_tokens(self, billing_service):
        """Test calculate_usage_costs correctly calculates token costs."""
        # Mock pricing
        billing_service._pricing_cache = {
            UsageType.TOKENS: {'price_per_unit': Decimal('0.0002'), 'unit_size': 1000}
        }
        billing_service._cache_timestamp = datetime.now()

        usage_quantities = {'tokens': 5000000}  # 5M tokens

        result = await billing_service.calculate_usage_costs(usage_quantities)

        # 5M tokens / 1000 units * $0.0002 = $1.00 = 100 cents
        assert result == {'tokens': 100}

    async def test_calculate_usage_costs_spans(self, billing_service):
        """Test calculate_usage_costs correctly calculates span costs."""
        # Mock pricing
        billing_service._pricing_cache = {
            UsageType.SPANS: {'price_per_unit': Decimal('0.0001'), 'unit_size': 1000}
        }
        billing_service._cache_timestamp = datetime.now()

        usage_quantities = {'spans': 1000000}  # 1M spans

        result = await billing_service.calculate_usage_costs(usage_quantities)

        # 1M spans / 1000 units * $0.0001 = $0.10 = 10 cents
        assert result == {'spans': 10}

    async def test_calculate_usage_costs_mixed_usage(self, billing_service):
        """Test calculate_usage_costs handles multiple usage types."""
        # Mock pricing
        billing_service._pricing_cache = {
            UsageType.TOKENS: {'price_per_unit': Decimal('0.0002'), 'unit_size': 1000},
            UsageType.SPANS: {'price_per_unit': Decimal('0.0001'), 'unit_size': 1000},
        }
        billing_service._cache_timestamp = datetime.now()

        usage_quantities = {'tokens': 2000000, 'spans': 500000}

        result = await billing_service.calculate_usage_costs(usage_quantities)

        # Tokens: 2M / 1000 * $0.0002 = $0.40 = 40 cents
        # Spans: 500K / 1000 * $0.0001 = $0.05 = 5 cents
        assert result == {'tokens': 40, 'spans': 5}

    async def test_calculate_usage_costs_unknown_usage_type(self, billing_service):
        """Test calculate_usage_costs ignores unknown usage types."""
        # Mock pricing
        billing_service._pricing_cache = {
            UsageType.TOKENS: {'price_per_unit': Decimal('0.0002'), 'unit_size': 1000}
        }
        billing_service._cache_timestamp = datetime.now()

        usage_quantities = {'tokens': 1000000, 'unknown_type': 500}

        result = await billing_service.calculate_usage_costs(usage_quantities)

        # Should only calculate known types
        assert result == {'tokens': 20}
        assert 'unknown_type' not in result

    async def test_calculate_usage_costs_zero_quantities(self, billing_service):
        """Test calculate_usage_costs handles zero quantities."""
        # Mock pricing
        billing_service._pricing_cache = {
            UsageType.TOKENS: {'price_per_unit': Decimal('0.0002'), 'unit_size': 1000},
            UsageType.SPANS: {'price_per_unit': Decimal('0.0001'), 'unit_size': 1000},
        }
        billing_service._cache_timestamp = datetime.now()

        usage_quantities = {'tokens': 0, 'spans': 0}

        result = await billing_service.calculate_usage_costs(usage_quantities)

        # With minimum charge threshold, zero quantities return empty dict
        assert result == {}

    async def test_create_billing_period_snapshot_success(
        self, billing_service, orm_session, test_org, test_user, test_user2, test_user3
    ):
        """Test create_billing_period_snapshot creates a complete snapshot."""
        # Set up org with paid members using existing test user fixtures
        from agentops.opsboard.models import UserOrgModel, OrgRoles

        # Create 3 paid member relationships using existing test users
        test_users = [test_user, test_user2, test_user3]

        for user in test_users:
            user_org = UserOrgModel(
                user_id=user.id,
                org_id=test_org.id,
                role=OrgRoles.developer,
                user_email=user.email,
                is_paid=True,
            )
            orm_session.add(user_org)

        orm_session.flush()

        # Mock the get_usage_for_period method
        with (
            patch.object(billing_service, 'get_usage_for_period') as mock_get_usage,
            patch.object(billing_service, 'calculate_usage_costs') as mock_calc_costs,
            patch.object(billing_service, 'get_seat_price') as mock_seat_price,
        ):
            mock_get_usage.return_value = {'tokens': 1000000, 'spans': 500}
            mock_calc_costs.return_value = {'tokens': 20, 'spans': 50}
            mock_seat_price.return_value = 4000  # $40 per seat

            period_start = datetime(2024, 1, 1)
            period_end = datetime(2024, 1, 31)

            result = await billing_service.create_billing_period_snapshot(
                orm_session, test_org, period_start, period_end
            )

            assert isinstance(result, BillingPeriod)
            assert result.org_id == test_org.id
            assert result.period_start == period_start
            assert result.period_end == period_end
            assert result.seat_cost == 12000  # 3 seats * $40
            assert result.seat_count == 3
            assert result.usage_costs == {'tokens': 20, 'spans': 50}
            assert result.usage_quantities == {'tokens': 1000000, 'spans': 500}
            assert result.total_cost == 12070  # 12000 + 20 + 50
            assert result.status == 'pending'

    async def test_create_billing_period_snapshot_no_usage(
        self, billing_service, orm_session, test_org, test_user
    ):
        """Test create_billing_period_snapshot works with zero usage."""
        # Set up org with one paid member using existing test user fixture
        from agentops.opsboard.models import UserOrgModel, OrgRoles

        user_org = UserOrgModel(
            user_id=test_user.id,
            org_id=test_org.id,
            role=OrgRoles.owner,
            user_email=test_user.email,
            is_paid=True,
        )
        orm_session.add(user_org)
        orm_session.flush()

        with (
            patch.object(billing_service, 'get_usage_for_period') as mock_get_usage,
            patch.object(billing_service, 'calculate_usage_costs') as mock_calc_costs,
            patch.object(billing_service, 'get_seat_price') as mock_seat_price,
        ):
            mock_get_usage.return_value = {}
            mock_calc_costs.return_value = {}
            mock_seat_price.return_value = 4000

            period_start = datetime(2024, 1, 1)
            period_end = datetime(2024, 1, 31)

            result = await billing_service.create_billing_period_snapshot(
                orm_session, test_org, period_start, period_end
            )

            assert result.seat_cost == 4000
            assert result.usage_costs == {}
            assert result.total_cost == 4000

    async def test_create_billing_period_snapshot_multiple_seats(
        self, billing_service, orm_session, test_org, test_user, test_user2, test_user3
    ):
        """Test create_billing_period_snapshot calculates multiple seat costs correctly."""
        # Set up org with multiple paid members using existing test user fixtures
        from agentops.opsboard.models import UserOrgModel, OrgRoles

        # Create 3 paid member relationships using existing test users (can't reuse same user for same org)
        test_users = [test_user, test_user2, test_user3]

        for user in test_users:
            user_org = UserOrgModel(
                user_id=user.id,
                org_id=test_org.id,
                role=OrgRoles.developer,
                user_email=user.email,
                is_paid=True,
            )
            orm_session.add(user_org)

        orm_session.flush()

        with (
            patch.object(billing_service, 'get_usage_for_period') as mock_get_usage,
            patch.object(billing_service, 'calculate_usage_costs') as mock_calc_costs,
            patch.object(billing_service, 'get_seat_price') as mock_seat_price,
        ):
            mock_get_usage.return_value = {'tokens': 500000}
            mock_calc_costs.return_value = {'tokens': 10}
            mock_seat_price.return_value = 3500  # $35 per seat

            period_start = datetime(2024, 1, 1)
            period_end = datetime(2024, 1, 31)

            result = await billing_service.create_billing_period_snapshot(
                orm_session, test_org, period_start, period_end
            )

            assert result.seat_cost == 10500  # 3 seats * $35
            assert result.seat_count == 3
            assert result.total_cost == 10510  # 10500 + 10

    async def test_create_billing_period_snapshot_commits_to_db(
        self, billing_service, orm_session, test_org, test_user
    ):
        """Test create_billing_period_snapshot commits the billing period to database."""
        # Set up org with one paid member using existing test user fixture
        from agentops.opsboard.models import UserOrgModel, OrgRoles

        user_org = UserOrgModel(
            user_id=test_user.id,
            org_id=test_org.id,
            role=OrgRoles.owner,
            user_email=test_user.email,
            is_paid=True,
        )
        orm_session.add(user_org)
        orm_session.flush()

        with (
            patch.object(billing_service, 'get_usage_for_period') as mock_get_usage,
            patch.object(billing_service, 'calculate_usage_costs') as mock_calc_costs,
            patch.object(billing_service, 'get_seat_price') as mock_seat_price,
        ):
            mock_get_usage.return_value = {}
            mock_calc_costs.return_value = {}
            mock_seat_price.return_value = 4000

            period_start = datetime(2024, 1, 1)
            period_end = datetime(2024, 1, 31)

            result = await billing_service.create_billing_period_snapshot(
                orm_session, test_org, period_start, period_end
            )

            # Verify it was added to the session and committed
            billing_period_in_db = orm_session.query(BillingPeriod).filter_by(id=result.id).first()
            assert billing_period_in_db is not None
            assert billing_period_in_db.org_id == test_org.id