Skip to content

A programming framework for agentic AI

15.0 KB
Raw Blame
// Copyright (c) Microsoft Corporation. All rights reserved.
// OpenAIChatRequestMessageConnector.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using OpenAI.Chat;

namespace AutoGen.OpenAI;

/// <summary>
/// This middleware converts the incoming <see cref="IMessage"/> to <see cref="IMessage{ChatMessage}" /> where T is <see cref="ChatMessage"/> before sending to agent. And converts the output <see cref="ChatCompletion"/> to <see cref="IMessage"/> after receiving from agent.
/// <para>Supported <see cref="IMessage"/> are</para>
/// <para>- <see cref="TextMessage"/></para> 
/// <para>- <see cref="ImageMessage"/></para> 
/// <para>- <see cref="MultiModalMessage"/></para>
/// <para>- <see cref="ToolCallMessage"/></para>
/// <para>- <see cref="ToolCallResultMessage"/></para>
/// <para>- <see cref="IMessage{ChatMessage}"/> where T is <see cref="ChatMessage"/></para>
/// <para>- <see cref="AggregateMessage{TMessage1, TMessage2}"/> where TMessage1 is <see cref="ToolCallMessage"/> and TMessage2 is <see cref="ToolCallResultMessage"/></para>
/// </summary>
public class OpenAIChatRequestMessageConnector : IMiddleware, IStreamingMiddleware
{
    private bool strictMode;

    /// <summary>
    /// Create a new instance of <see cref="OpenAIChatRequestMessageConnector"/>.
    /// </summary>
    /// <param name="strictMode">If true, <see cref="OpenAIChatRequestMessageConnector"/> will throw an <see cref="InvalidOperationException"/>
    /// When the message type is not supported. If false, it will ignore the unsupported message type.</param>
    public OpenAIChatRequestMessageConnector(bool strictMode = false)
    {
        this.strictMode = strictMode;
    }

    public string? Name => nameof(OpenAIChatRequestMessageConnector);

    public async Task<IMessage> InvokeAsync(MiddlewareContext context, IAgent agent, CancellationToken cancellationToken = default)
    {
        var chatMessages = ProcessIncomingMessages(agent, context.Messages);

        var reply = await agent.GenerateReplyAsync(chatMessages, context.Options, cancellationToken);

        return PostProcessMessage(reply);
    }

    public async IAsyncEnumerable<IMessage> InvokeAsync(
        MiddlewareContext context,
        IStreamingAgent agent,
        [EnumeratorCancellation] CancellationToken cancellationToken = default)
    {
        var chatMessages = ProcessIncomingMessages(agent, context.Messages);
        var streamingReply = agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken);
        var chunks = new List<StreamingChatCompletionUpdate>();

        // only streaming the text content
        await foreach (var reply in streamingReply)
        {
            if (reply is IMessage<StreamingChatCompletionUpdate> update)
            {
                if (update.Content.ContentUpdate.Count == 1 && update.Content.ContentUpdate[0].Kind == ChatMessageContentPartKind.Text)
                {
                    yield return new TextMessageUpdate(Role.Assistant, update.Content.ContentUpdate[0].Text, from: update.From);
                }

                chunks.Add(update.Content);
            }
            else
            {
                if (this.strictMode)
                {
                    throw new InvalidOperationException($"Invalid streaming message type {reply.GetType().Name}");
                }
                else
                {
                    yield return reply;
                }
            }
        }

        // process the tool call
        var streamingChatToolCallUpdates = chunks.Where(c => c.ToolCallUpdates.Count > 0)
                                                .SelectMany(c => c.ToolCallUpdates)
                                                .ToList();

        // collect all text parts
        var textParts = chunks.SelectMany(c => c.ContentUpdate)
            .Where(c => c.Kind == ChatMessageContentPartKind.Text)
            .Select(c => c.Text)
            .ToList();

        // combine the tool call and function call into one ToolCallMessages
        var text = string.Join(string.Empty, textParts);
        var toolCalls = new List<ToolCall>();
        var currentToolName = string.Empty;
        var currentToolArguments = string.Empty;
        var currentToolId = string.Empty;
        int? currentIndex = null;
        foreach (var toolCall in streamingChatToolCallUpdates)
        {
            if (currentIndex is null)
            {
                currentIndex = toolCall.Index;
            }

            if (toolCall.Index == currentIndex)
            {
                currentToolName += toolCall.FunctionName;
                currentToolArguments += toolCall.FunctionArgumentsUpdate;
                currentToolId += toolCall.ToolCallId;

                yield return new ToolCallMessageUpdate(currentToolName, currentToolArguments, from: agent.Name);
            }
            else
            {
                toolCalls.Add(new ToolCall(currentToolName, currentToolArguments) { ToolCallId = currentToolId });
                currentToolName = toolCall.FunctionName;
                currentToolArguments = toolCall.FunctionArgumentsUpdate.ToString();
                currentToolId = toolCall.ToolCallId;
                currentIndex = toolCall.Index;

                yield return new ToolCallMessageUpdate(currentToolName, currentToolArguments, from: agent.Name);
            }
        }

        if (string.IsNullOrEmpty(currentToolName) is false)
        {
            toolCalls.Add(new ToolCall(currentToolName, currentToolArguments) { ToolCallId = currentToolId });
        }

        if (toolCalls.Any())
        {
            yield return new ToolCallMessage(toolCalls, from: agent.Name)
            {
                Content = text,
            };
        }
    }

    public IMessage PostProcessMessage(IMessage message)
    {
        return message switch
        {
            IMessage<ChatCompletion> m => PostProcessChatCompletions(m),
            _ when strictMode is false => message,
            _ => throw new InvalidOperationException($"Invalid return message type {message.GetType().Name}"),
        };
    }

    private IMessage PostProcessChatCompletions(IMessage<ChatCompletion> message)
    {
        // throw exception if prompt filter results is not null
        if (message.Content.FinishReason == ChatFinishReason.ContentFilter)
        {
            throw new InvalidOperationException("The content is filtered because its potential risk. Please try another input.");
        }

        // throw exception is there is more than on choice
        if (message.Content.Content.Count > 1)
        {
            throw new InvalidOperationException("The content has more than one choice. Please try another input.");
        }

        return PostProcessChatResponseMessage(message.Content, message.From);
    }

    private IMessage PostProcessChatResponseMessage(ChatCompletion chatCompletion, string? from)
    {
        // throw exception if prompt filter results is not null
        if (chatCompletion.FinishReason == ChatFinishReason.ContentFilter)
        {
            throw new InvalidOperationException("The content is filtered because its potential risk. Please try another input.");
        }

        // throw exception is there is more than on choice
        if (chatCompletion.Content.Count > 1)
        {
            throw new InvalidOperationException("The content has more than one choice. Please try another input.");
        }
        var textContent = chatCompletion.Content is { Count: > 0 } ? chatCompletion.Content[0] : null;

        // if tool calls is not empty, return ToolCallMessage
        if (chatCompletion.ToolCalls is { Count: > 0 })
        {
            var toolCalls = chatCompletion.ToolCalls.Select(tc => new ToolCall(tc.FunctionName, tc.FunctionArguments.ToString()) { ToolCallId = tc.Id });
            return new ToolCallMessage(toolCalls, from)
            {
                Content = textContent?.Kind switch
                {
                    _ when textContent?.Kind == ChatMessageContentPartKind.Text => textContent.Text,
                    _ => null,
                },
            };
        }

        // if the content is text, return TextMessage
        if (textContent?.Kind == ChatMessageContentPartKind.Text)
        {
            return new TextMessage(Role.Assistant, textContent.Text, from);
        }

        throw new InvalidOperationException("Invalid ChatResponseMessage");
    }

    public IEnumerable<IMessage> ProcessIncomingMessages(IAgent agent, IEnumerable<IMessage> messages)
    {
        return messages.SelectMany<IMessage, IMessage>(m =>
        {
            if (m is IMessage<ChatMessage> crm)
            {
                return [crm];
            }
            else
            {
                var chatRequestMessages = m switch
                {
                    TextMessage textMessage => ProcessTextMessage(agent, textMessage),
                    ImageMessage imageMessage when (imageMessage.From is null || imageMessage.From != agent.Name) => ProcessImageMessage(agent, imageMessage),
                    MultiModalMessage multiModalMessage when (multiModalMessage.From is null || multiModalMessage.From != agent.Name) => ProcessMultiModalMessage(agent, multiModalMessage),
                    ToolCallMessage toolCallMessage when (toolCallMessage.From is null || toolCallMessage.From == agent.Name) => ProcessToolCallMessage(agent, toolCallMessage),
                    ToolCallResultMessage toolCallResultMessage => ProcessToolCallResultMessage(toolCallResultMessage),
                    AggregateMessage<ToolCallMessage, ToolCallResultMessage> aggregateMessage => ProcessFunctionCallMiddlewareMessage(agent, aggregateMessage),
                    _ when strictMode is false => [],
                    _ => throw new InvalidOperationException($"Invalid message type: {m.GetType().Name}"),
                };

                if (chatRequestMessages.Any())
                {
                    return chatRequestMessages.Select(cm => MessageEnvelope.Create(cm, m.From));
                }
                else
                {
                    return [m];
                }
            }
        });
    }

    private IEnumerable<ChatMessage> ProcessTextMessage(IAgent agent, TextMessage message)
    {
        if (message.Role == Role.System)
        {
            return [new SystemChatMessage(message.Content) { ParticipantName = message.From }];
        }

        if (agent.Name == message.From)
        {
            return [new AssistantChatMessage(message.Content) { ParticipantName = agent.Name }];
        }
        else
        {
            return message.From switch
            {
                null when message.Role == Role.User => [new UserChatMessage(message.Content)],
                null when message.Role == Role.Assistant => [new AssistantChatMessage(message.Content)],
                null => throw new InvalidOperationException("Invalid Role"),
                _ => [new UserChatMessage(message.Content) { ParticipantName = message.From }]
            };
        }
    }

    private IEnumerable<ChatMessage> ProcessImageMessage(IAgent agent, ImageMessage message)
    {
        if (agent.Name == message.From)
        {
            // image message from assistant is not supported
            throw new ArgumentException("ImageMessage is not supported when message.From is the same with agent");
        }

        var imageContentItem = this.CreateChatMessageImageContentItemFromImageMessage(message);
        return [new UserChatMessage([imageContentItem]) { ParticipantName = message.From }];
    }

    private IEnumerable<ChatMessage> ProcessMultiModalMessage(IAgent agent, MultiModalMessage message)
    {
        if (agent.Name == message.From)
        {
            // image message from assistant is not supported
            throw new ArgumentException("MultiModalMessage is not supported when message.From is the same with agent");
        }

        IEnumerable<ChatMessageContentPart> items = message.Content.Select<IMessage, ChatMessageContentPart>(ci => ci switch
        {
            TextMessage text => ChatMessageContentPart.CreateTextPart(text.Content),
            ImageMessage image => this.CreateChatMessageImageContentItemFromImageMessage(image),
            _ => throw new NotImplementedException(),
        });

        return [new UserChatMessage(items) { ParticipantName = message.From }];
    }

    private ChatMessageContentPart CreateChatMessageImageContentItemFromImageMessage(ImageMessage message)
    {
        return message.Data is null && message.Url is not null
            ? ChatMessageContentPart.CreateImagePart(new Uri(message.Url))
            : ChatMessageContentPart.CreateImagePart(message.Data, message.Data?.MediaType);
    }

    private IEnumerable<ChatMessage> ProcessToolCallMessage(IAgent agent, ToolCallMessage message)
    {
        if (message.From is not null && message.From != agent.Name)
        {
            throw new ArgumentException("ToolCallMessage is not supported when message.From is not the same with agent");
        }

        var toolCallParts = message.ToolCalls.Select((tc, i) => ChatToolCall.CreateFunctionToolCall(tc.ToolCallId ?? $"{tc.FunctionName}_{i}", tc.FunctionName, BinaryData.FromString(tc.FunctionArguments)));
        var textContent = message.GetContent() ?? null;

        // Don't set participant name for assistant when it is tool call
        // fix https://github.com/microsoft/autogen/issues/3437
        AssistantChatMessage chatRequestMessage;

        if (string.IsNullOrEmpty(textContent) is true)
        {
            chatRequestMessage = new AssistantChatMessage(toolCallParts);
        }
        else
        {
            chatRequestMessage = new AssistantChatMessage(textContent);

            foreach (var toolCallPart in toolCallParts)
            {
                chatRequestMessage.ToolCalls.Add(toolCallPart);
            }
        }

        return [chatRequestMessage];
    }

    private IEnumerable<ChatMessage> ProcessToolCallResultMessage(ToolCallResultMessage message)
    {
        return message.ToolCalls
            .Where(tc => tc.Result is not null)
            .Select((tc, i) => new ToolChatMessage(tc.ToolCallId ?? $"{tc.FunctionName}_{i}", tc.Result));
    }

    private IEnumerable<ChatMessage> ProcessFunctionCallMiddlewareMessage(IAgent agent, AggregateMessage<ToolCallMessage, ToolCallResultMessage> aggregateMessage)
    {
        if (aggregateMessage.From is not null && aggregateMessage.From != agent.Name)
        {
            // convert as user message
            var resultMessage = aggregateMessage.Message2;

            return resultMessage.ToolCalls.Select(tc => new UserChatMessage(tc.Result) { ParticipantName = aggregateMessage.From });
        }
        else
        {
            var toolCallMessage1 = aggregateMessage.Message1;
            var toolCallResultMessage = aggregateMessage.Message2;

            var assistantMessage = this.ProcessToolCallMessage(agent, toolCallMessage1);
            var toolCallResults = this.ProcessToolCallResultMessage(toolCallResultMessage);

            return assistantMessage.Concat(toolCallResults);
        }
    }
}