Skip to content

A programming framework for agentic AI

10.9 KB
Raw Blame
// Copyright (c) Microsoft Corporation. All rights reserved.
// AnthropicMessageConnector.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json.Nodes;
using System.Threading;
using System.Threading.Tasks;
using AutoGen.Anthropic.DTO;
using AutoGen.Core;

namespace AutoGen.Anthropic.Middleware;

public class AnthropicMessageConnector : IStreamingMiddleware
{
    public string? Name => nameof(AnthropicMessageConnector);

    public async Task<IMessage> InvokeAsync(MiddlewareContext context, IAgent agent, CancellationToken cancellationToken = default)
    {
        var messages = context.Messages;
        var chatMessages = await ProcessMessageAsync(messages, agent);
        var response = await agent.GenerateReplyAsync(chatMessages, context.Options, cancellationToken);

        return response is IMessage<ChatCompletionResponse> chatMessage
            ? PostProcessMessage(chatMessage.Content, agent)
            : response;
    }

    public async IAsyncEnumerable<IMessage> InvokeAsync(MiddlewareContext context, IStreamingAgent agent,
        [EnumeratorCancellation] CancellationToken cancellationToken = default)
    {
        var messages = context.Messages;
        var chatMessages = await ProcessMessageAsync(messages, agent);

        await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken))
        {
            if (reply is IMessage<ChatCompletionResponse> chatMessage)
            {
                var response = ProcessChatCompletionResponse(chatMessage, agent);
                if (response is not null)
                {
                    yield return response;
                }
            }
            else
            {
                yield return reply;
            }
        }
    }

    private IMessage? ProcessChatCompletionResponse(IMessage<ChatCompletionResponse> chatMessage,
        IStreamingAgent agent)
    {
        if (chatMessage.Content.Content is { Count: 1 } &&
            chatMessage.Content.Content[0] is ToolUseContent toolUseContent)
        {
            return new ToolCallMessage(
                toolUseContent.Name ??
                throw new InvalidOperationException($"Expected {nameof(toolUseContent.Name)} to be specified"),
                toolUseContent.Input?.ToString() ??
                throw new InvalidOperationException($"Expected {nameof(toolUseContent.Input)} to be specified"),
                from: agent.Name);
        }

        var delta = chatMessage.Content.Delta;
        return delta != null && !string.IsNullOrEmpty(delta.Text)
            ? new TextMessageUpdate(role: Role.Assistant, delta.Text, from: agent.Name)
            : null;
    }

    private async Task<IEnumerable<IMessage>> ProcessMessageAsync(IEnumerable<IMessage> messages, IAgent agent)
    {
        var processedMessages = new List<IMessage>();

        foreach (var message in messages)
        {
            var processedMessage = message switch
            {
                TextMessage textMessage => ProcessTextMessage(textMessage, agent),

                ImageMessage imageMessage =>
                    (MessageEnvelope<ChatMessage>[])[new MessageEnvelope<ChatMessage>(new ChatMessage("user",
                            new ContentBase[] { new ImageContent { Source = await ProcessImageSourceAsync(imageMessage) } }
                                .ToList()),
                        from: agent.Name)],

                MultiModalMessage multiModalMessage => await ProcessMultiModalMessageAsync(multiModalMessage, agent),

                ToolCallMessage toolCallMessage => ProcessToolCallMessage(toolCallMessage, agent),
                ToolCallResultMessage toolCallResultMessage => ProcessToolCallResultMessage(toolCallResultMessage),
                AggregateMessage<ToolCallMessage, ToolCallResultMessage> toolCallAggregateMessage => ProcessToolCallAggregateMessage(toolCallAggregateMessage, agent),
                _ => [message],
            };

            processedMessages.AddRange(processedMessage);
        }

        return processedMessages;
    }

    private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from)
    {
        if (response.Content is null)
        {
            throw new ArgumentNullException(nameof(response.Content));
        }

        // When expecting a tool call, sometimes the response will contain two messages, one chat and one tool.
        // The first message is typically a TextContent, of the LLM explaining what it is trying to do.
        // The second message contains the tool call.
        if (response.Content.Count > 1)
        {
            if (response.Content.Count == 2 && response.Content[0] is TextContent &&
                response.Content[1] is ToolUseContent toolUseContent)
            {
                return new ToolCallMessage(toolUseContent.Name ?? string.Empty,
                    toolUseContent.Input?.ToJsonString() ?? string.Empty,
                    from: from.Name);
            }

            throw new NotSupportedException($"Expected {nameof(response.Content)} to have one output");
        }

        var content = response.Content[0];
        switch (content)
        {
            case TextContent textContent:
                return new TextMessage(Role.Assistant, textContent.Text ?? string.Empty, from: from.Name);

            case ToolUseContent toolUseContent:
                return new ToolCallMessage(toolUseContent.Name ?? string.Empty,
                    toolUseContent.Input?.ToJsonString() ?? string.Empty,
                    from: from.Name);

            case ImageContent:
                throw new InvalidOperationException(
                    "Claude is an image understanding model only. It can interpret and analyze images, but it cannot generate, produce, edit, manipulate or create images");
            default:
                throw new ArgumentOutOfRangeException(nameof(content));
        }
    }

    private IEnumerable<IMessage<ChatMessage>> ProcessTextMessage(TextMessage textMessage, IAgent agent)
    {
        ChatMessage messages;

        if (textMessage.From == agent.Name)
        {
            messages = new ChatMessage(
                "assistant", textMessage.Content);
        }
        else if (textMessage.From is null)
        {
            if (textMessage.Role == Role.User)
            {
                messages = new ChatMessage(
                    "user", textMessage.Content);
            }
            else if (textMessage.Role == Role.Assistant)
            {
                messages = new ChatMessage(
                    "assistant", textMessage.Content);
            }
            else if (textMessage.Role == Role.System)
            {
                messages = new ChatMessage(
                    "system", textMessage.Content);
            }
            else
            {
                throw new NotSupportedException($"Role {textMessage.Role} is not supported");
            }
        }
        else
        {
            // if from is not null, then the message is from user
            messages = new ChatMessage(
                "user", textMessage.Content);
        }

        return [new MessageEnvelope<ChatMessage>(messages, from: textMessage.From)];
    }

    private async Task<IEnumerable<IMessage>> ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent)
    {
        var content = new List<ContentBase>();
        foreach (var message in multiModalMessage.Content)
        {
            switch (message)
            {
                case TextMessage textMessage when textMessage.GetContent() is not null:
                    content.Add(new TextContent { Text = textMessage.GetContent() });
                    break;
                case ImageMessage imageMessage:
                    content.Add(new ImageContent() { Source = await ProcessImageSourceAsync(imageMessage) });
                    break;
            }
        }

        return [MessageEnvelope.Create(new ChatMessage("user", content), agent.Name)];
    }

    private async Task<ImageSource> ProcessImageSourceAsync(ImageMessage imageMessage)
    {
        if (imageMessage.Data != null)
        {
            return new ImageSource
            {
                MediaType = imageMessage.Data.MediaType,
                Data = Convert.ToBase64String(imageMessage.Data.ToArray())
            };
        }

        if (imageMessage.Url is null)
        {
            throw new InvalidOperationException("Invalid ImageMessage, the data or url must be provided");
        }

        var uri = new Uri(imageMessage.Url);
        using var client = new HttpClient();
        var response = client.GetAsync(uri).Result;
        if (!response.IsSuccessStatusCode)
        {
            throw new HttpRequestException($"Failed to download the image from {uri}");
        }

        return new ImageSource
        {
            MediaType = "image/jpeg",
            Data = Convert.ToBase64String(await response.Content.ReadAsByteArrayAsync())
        };
    }

    private IEnumerable<IMessage> ProcessToolCallMessage(ToolCallMessage toolCallMessage, IAgent _)
    {
        var chatMessage = new ChatMessage("assistant", new List<ContentBase>());
        foreach (var toolCall in toolCallMessage.ToolCalls)
        {
            chatMessage.AddContent(new ToolUseContent
            {
                Id = toolCall.ToolCallId,
                Name = toolCall.FunctionName,
                Input = JsonNode.Parse(toolCall.FunctionArguments)
            });
        }

        return [MessageEnvelope.Create(chatMessage, toolCallMessage.From)];
    }

    private IEnumerable<IMessage> ProcessToolCallResultMessage(ToolCallResultMessage toolCallResultMessage)
    {
        var chatMessage = new ChatMessage("user", new List<ContentBase>());
        foreach (var toolCall in toolCallResultMessage.ToolCalls)
        {
            chatMessage.AddContent(new ToolResultContent
            {
                Id = toolCall.ToolCallId ?? string.Empty,
                Content = toolCall.Result,
            });
        }

        return [MessageEnvelope.Create(chatMessage, toolCallResultMessage.From)];
    }

    private IEnumerable<IMessage> ProcessToolCallAggregateMessage(AggregateMessage<ToolCallMessage, ToolCallResultMessage> aggregateMessage, IAgent agent)
    {
        if (aggregateMessage.From is { } from && from != agent.Name)
        {
            var contents = aggregateMessage.Message2.ToolCalls.Select(t => t.Result);
            var messages = contents.Select(c =>
                new ChatMessage("assistant", c ?? throw new ArgumentNullException(nameof(c))));

            return messages.Select(m => new MessageEnvelope<ChatMessage>(m, from: from));
        }

        var toolCallMessage = ProcessToolCallMessage(aggregateMessage.Message1, agent);
        var toolCallResult = ProcessToolCallResultMessage(aggregateMessage.Message2);

        return toolCallMessage.Concat(toolCallResult);
    }
}