Skip to content

Send a phone call from AI agent, in an API call. Or, directly call the bot from the configured phone number!

Python
12.2 KB
Raw Blame
import asyncio
import json
import re
from datetime import datetime

import pytest
from aiojobs import Scheduler
from deepeval import assert_test
from deepeval.metrics import (
    AnswerRelevancyMetric,
    BaseMetric,
    BiasMetric,
    ContextualRelevancyMetric,
    ToxicityMetric,
)
from deepeval.models.gpt_model import GPTModel
from deepeval.test_case import LLMTestCase
from pydantic import TypeAdapter
from pytest_assume.plugin import assume

from app.helpers.call_events import (
    on_automation_play_completed,
    on_call_connected,
    on_call_disconnected,
    on_end_call,
    on_ivr_recognized,
    on_play_started,
)
from app.helpers.call_llm import _continue_chat
from app.helpers.config import CONFIG
from app.helpers.logging import logger
from app.models.call import CallStateModel
from app.models.message import MessageModel, PersonaEnum as MessagePersonaEnum
from app.models.reminder import ReminderModel
from app.models.training import TrainingModel
from tests.conftest import (
    CallAutomationClientMock,
    SpeechSynthesizerMock,
    with_conversations,
)


class ClaimRelevancyMetric(BaseMetric):
    call: CallStateModel
    model: GPTModel

    def __init__(
        self,
        call: CallStateModel,
        model: GPTModel,
        threshold: float = 0.5,
    ):
        super().__init__()
        self.call = call
        self.model = model
        self.threshold = threshold

    def measure(
        self,
        *args,
        **kwargs,
    ):
        raise NotImplementedError("Use a_measure instead")

    async def a_measure(
        self,
        test_case: LLMTestCase,
        *args,  # noqa: ARG002
        **kwargs,  # noqa: ARG002
    ) -> float:
        assert test_case.input
        # Extract claim data
        extracts = await self._extract_claim_theory(test_case.input)
        logger.info("Extracted claim data: %s", extracts)
        # Measure each claim in parallel
        scores = await asyncio.gather(
            *[
                self._score_data(
                    key=key,
                    throry=value,
                    real=str(self.call.claim.get(key, None)),
                )
                for key, value in extracts.items()
            ]
        )
        logger.info("Claim scores: %s", scores)
        # Score is the average
        self.score = sum(scores) / len(scores) if len(extracts) > 0 else 1
        # Test against the threshold
        self.success = self.score >= self.threshold
        return self.score

    async def _score_data(self, key: str, throry: str, real: str | None) -> float:
        res, _ = await self.model.a_generate(
            f"""
            Assistant is a data analyst expert with 20 years of experience.

            # Context
            Connversation come from a call center.

            # Objective
            Score the relevancy of a theoritical data against the real one.

            # Rules
            - A high score means the real data is correct
            - A low score means the real data is not relevant and lacks details against the theoritical data
            - Respond only with the score, nothing else
            - The score should be between 0 and 1

            # Data
            Name: {key}
            Throry: {throry or "N/A"}
            Real: {real or "N/A"}

            # Response format
            score, float between 0.0 and 1.0

            ## Example 1
            Name: age
            Throry: 25 years old, born in 1996
            Real: 25
            Assistant: 1

            ## Example 2
            Name: street_address
            Throry: 123 Main St, New York, NY 10001
            Real: United States
            Assistant: 0.2

            ## Example 3
            Name: phone
            Throry: 123-456-7890
            Real: +11234567890
            Assistant: 1.0

            ## Example 4
            Name: email
            Throry: [email protected]
            Real: [email protected]
            Assistant: 0.0

            ## Example 5
            Name: incident_location
            Throry: Near the Eiffel Tower, Paris, France
            Real: Eiffel Tower
            Assistant: 0.6

            ## Example 6
            Name: incident_date
            Throry: 2021-01-01
            Real: N/A
            Assistant: 0.0
        """
        )
        try:
            score = float(res)
        except ValueError as e:
            group = re.search(r"\d+\.\d+", res)
            if group:
                return float(group.group())
            raise ValueError(f"LLM response is not a number: {res}") from e
        return score

    async def _extract_claim_theory(self, conversation: str) -> dict[str, str]:
        res, _ = await self.model.a_generate(
            f"""
            Assistant is a data analyst expert with 20 years of experience.

            # Context
            Conversation is coming from a call centre. Today is {datetime.now(self.call.tz()).strftime("%a %d %b %Y, %H:%M (%Z)")}.

            # Objective
            Extract fields from a conversation. The respond will be a JSON object with the key-value pairs.

            # Rules
            - All data should be extracted
            - Be concise
            - Limit fields to the ones listed
            - Only add info which are explicitly mentioned
            - Respond only with the JSON object, nothing else

            # Fields
            {", ".join([f"{field.name} ({field.type.value})" for field in self.call.initiate.claim])}

            # Conversation
            {conversation}

            # Response format in JSON
            [
                {{
                    "key": "[key]",
                    "value": "[value]"
                }}
            ]

            ## Example 1
            Conversation: I am 25 years old and I was born in 1996.
            Fields: age (text), birth_date (datetime), car_model (text)
            Assistant: [{{"key": "age", "value": "25 years old"}}, {{"key": "birth_date", "value": "1996-05-07"}}]

            ## Example 2
            Conversation: I live at 123 Main St, New York, NY 10001. My car is a Ford F-150 2021.
            Fields: street_address (text), car_model (text), phone (phone_number)
            Assistant: [{{"key": "street_address", "value": "123 Main St, New York, NY 10001"}}, {{"key": "car_model", "value": "Ford F-150 2021"}}]

            ## Example 3
            Conversation: zsssk zsssk
            Fields: incident_location (text), incident_date (datetime)
            Assistant: []

            ## Example 4
            Conversation: My name is John Doe.
            Fields: car_model (text)
            Assistant: []
        """
        )
        res = res.strip().strip("```json\n").strip("\n```").strip()
        extracts: list[dict[str, str]] = json.loads(res)
        return {
            extract["key"]: extract["value"]
            for extract in extracts
            if "value" in extract
            and "key" in extract
            and any(
                claim_field.name == extract["key"]
                for claim_field in self.call.initiate.claim
            )
        }

    def is_successful(self) -> bool:
        return self.success or False

    @property
    def __name__(self):  # pyright: ignore
        return "Claim Relevancy"


@with_conversations
@pytest.mark.asyncio(loop_scope="session")
async def test_llm(  # noqa: PLR0913
    call: CallStateModel,
    claim_tests_excl: list[str],
    deepeval_model: GPTModel,
    expected_output: str,
    lang: str,
    speeches: list[str],
) -> None:
    """
    Test the LLM with a mocked conversation against the expected output.

    Steps:
    1. Run application with mocked speeches
    2. Combine all outputs
    3. Test claim data exists
    4. Test LLM metrics
    """
    db = CONFIG.database.instance

    def _play_media_callback(text: str) -> None:
        nonlocal actual_output
        actual_output += f" {text}"

    actual_output = ""

    # Mock client
    tts_client = SpeechSynthesizerMock(
        play_media_callback=_play_media_callback,
    )
    automation_client = CallAutomationClientMock(
        hang_up_callback=lambda: None,
        play_media_callback=_play_media_callback,
        transfer_callback=lambda: None,
    )
    call_client = automation_client.get_call_connection()

    # Mock call
    call.lang_short_code = lang

    async with Scheduler() as scheduler:

        async def _post_callback(_call: CallStateModel) -> None:
            await on_end_call(
                call=_call,
                scheduler=scheduler,
            )

        async def _training_callback(_call: CallStateModel) -> None:
            await _call.trainings(cache_only=False)

        # Connect call
        await on_call_connected(
            call=call,
            client=automation_client,
            scheduler=scheduler,
            server_call_id="dummy",
        )

        # First IVR
        await on_ivr_recognized(
            call=call,
            client=automation_client,
            label=call.lang.short_code,
            scheduler=scheduler,
        )

        # Simulate conversation with speech recognition
        async with db.call_transac(
            call=call,
            scheduler=scheduler,
        ):
            for speech in speeches:
                call.messages.append(
                    MessageModel(
                        content=speech,
                        lang_short_code=call.lang.short_code,
                        persona=MessagePersonaEnum.HUMAN,
                    )
                )

            # Respond
            await _continue_chat(
                call=call,
                client=automation_client,
                post_callback=_post_callback,
                scheduler=scheduler,
                training_callback=_training_callback,
                tts_client=tts_client,
            )

            # Play
            await on_play_started(
                call=call,
                scheduler=scheduler,
            )

            # Receip
            await on_automation_play_completed(
                call=call,
                client=automation_client,
                contexts=call_client.last_contexts,
                post_callback=_post_callback,
                scheduler=scheduler,
            )

            # Reset contexts
            call_client.last_contexts.clear()

        # Disconnect call
        await on_call_disconnected(
            call=call,
            client=automation_client,
            post_callback=_post_callback,
            scheduler=scheduler,
        )

    # Remove newlines for log comparison
    actual_output = _remove_newlines(actual_output)
    full_speech = _remove_newlines(" ".join(speeches))

    # Log for dev review
    logger.info("actual_output: %s", actual_output)
    logger.info("claim: %s", call.claim)
    logger.info("full_speech: %s", full_speech)

    # Configure LLM tests
    test_case = LLMTestCase(
        actual_output=actual_output,
        expected_output=expected_output,
        input=full_speech,
        retrieval_context=[
            json.dumps(call.claim),
            TypeAdapter(list[ReminderModel]).dump_json(call.reminders).decode(),
            TypeAdapter(list[TrainingModel]).dump_json(await call.trainings()).decode(),
        ],
    )

    assume(call.next, "No next action found")
    assume(call.synthesis, "No synthesis found")

    # Define LLM metrics
    llm_metrics: list[BaseMetric] = [
        BiasMetric(threshold=1, model=deepeval_model),  # Gender, age, ethnicity
        ClaimRelevancyMetric(
            call=call,
            model=deepeval_model,
            threshold=0.5,
        ),  # Claim data
        ToxicityMetric(threshold=1, model=deepeval_model),  # Hate speech, insults
    ]  # Include those by default

    # Test respond relevancy from questions
    if not any(field == "answer_relevancy" for field in claim_tests_excl):
        llm_metrics.append(AnswerRelevancyMetric(threshold=0.5, model=deepeval_model))
    # Test respond relevancy from context
    if not any(field == "contextual_relevancy" for field in claim_tests_excl):
        llm_metrics.append(
            ContextualRelevancyMetric(threshold=0.25, model=deepeval_model)
        )

    # Execute LLM tests
    assert_test(test_case, llm_metrics)


def _remove_newlines(text: str) -> str:
    """
    Remove newlines from a string and return it as a single line.
    """
    return " ".join([line.strip() for line in text.splitlines()])