Skip to content

A programming framework for agentic AI

25.4 KB
Raw Blame
// Copyright (c) Microsoft Corporation. All rights reserved.
// Messages.cs

using System.Collections;
using System.Diagnostics;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using Microsoft.AutoGen.AgentChat.GroupChat;
using Microsoft.Extensions.AI;

namespace Microsoft.AutoGen.AgentChat.Abstractions;

/// <summary>
/// The base class for all messages that can be sent between agents.
/// </summary>
/// <remarks>
/// This functions as a combination of both <c>BaseMessage</c> and <c>AgentMessage</c> on the Python side.
/// </remarks>
public abstract class AgentMessage
{
    /// <summary>
    /// The name of the agent that sent this message.
    /// </summary>
    public required string Source { get; set; }

    /// <summary>
    /// The <see cref="IChatClient"/> usage incurred when producing this message.
    /// </summary>
    public RequestUsage? ModelUsage { get; set; }

    // IMPORTANT NOTE: Unlike the ITypeMarshal<AgentMessage, WireProtocol.AgentMessage> implementation in ProtobufTypeMarshal,
    // the .ToWire() call on this is intended to be used for directly converting a concrete message type to its leaf representation.
    // In the context of Protobuf these may not be the same due to discriminated union types being real types, as opposed to
    // a runtime union restriction.
    //public IMessage ToWire()
    //{
    //    return this switch
    //    {
    //        ChatMessage chatMessage => ProtobufTypeMarshal.Convert<ChatMessage, WireProtocol.ChatMessage>(chatMessage),
    //        AgentEvent agentEvent => ProtobufTypeMarshal.Convert<AgentEvent, WireProtocol.AgentEvent>(agentEvent),
    //        _ => throw new InvalidOperationException($"Unknown type {this.GetType().Name}"),
    //    };
    //}
}

/// <summary>
/// Events emitted by agents and teams when they work, not used for agent-to-agent communication.
/// </summary>
public abstract class AgentEvent : AgentMessage
{
    public Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage()
        => ToCompletionClientMessage(role: ChatRole.Assistant);

    /// <summary>
    /// Converts the <see cref="AgentEvent"/> to a <see cref="Microsoft.Extensions.AI.ChatMessage"/>.
    /// </summary>
    /// <remarks>
    /// This should usually be <see cref="ChatRole.Assistant"/>
    /// </remarks>
    /// <param name="role">The role of the agent that is sending the message.</param>
    /// <returns>
    /// A <see cref="Microsoft.Extensions.AI.ChatMessage"/> that represents the <see cref="AgentEvent"/>.
    /// </returns>
    public abstract Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role);
}

/// <summary>
/// Messages for agent-to-agent communication.
/// </summary>
public abstract class ChatMessage : AgentMessage
{
    /// <summary>
    /// Converts the <see cref="ChatMessage"/> to a <see cref="Microsoft.Extensions.AI.ChatMessage"/>.
    /// </summary>
    /// <param name="role">The role of the agent that is sending the message.</param>
    /// <returns>
    /// A <see cref="Microsoft.Extensions.AI.ChatMessage"/> that represents the <see cref="ChatMessage"/>.
    /// </returns>
    public abstract Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role);
}

// Leaf Classes

/// <summary>
/// A text message.
/// </summary>
public class TextMessage : ChatMessage
{
    /// <summary>
    /// The content of the message.
    /// </summary>
    public required string Content { get; set; }

    /// <inheritdoc cref="ChatMessage.ToCompletionClientMessage(ChatRole)" />/>
    public override Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        return new Microsoft.Extensions.AI.ChatMessage(role, this.Content) { AuthorName = this.Source };
    }
}

/// <summary>
/// The data inside of a multi-modal message. Can be either a <c>string</c> or an Image.
/// </summary>
/// <remarks>
/// This presents an API surface around the types that are supported by AgentChat, rather
/// than allowing any <see cref="Microsoft.Extensions.AI.AIContent"/>.
/// </remarks>
public struct MultiModalData
{
    /// <summary>
    /// Supported <c>Type</c>s of <see cref="Microsoft.Extensions.AI.AIContent"/>.
    /// </summary>
    public enum Type
    {
        String, Image
    }

    /// <summary>
    /// Checks the type of the <see cref="AIContent"/> and wraps it in a <see cref="MultiModalData"/> instance if
    /// it is a supported type.
    /// </summary>
    /// <param name="item">The <see cref="AIContent"/> to wrap.</param>
    /// <returns>A <see cref="MultiModalData"/> instance wrapping the <paramref name="item"/>.</returns>
    /// <exception cref="ArgumentException">
    /// Thrown if the <paramref name="item"/> is not a <see cref="TextContent"/> or <see cref="DataContent"/>.
    /// </exception>
    public static MultiModalData CheckTypeAndCreate(AIContent item)
    {
        if (item is TextContent text)
        {
            return new MultiModalData(text);
        }
        else if (item is DataContent image)
        {
            return new MultiModalData(image);
        }
        else
        {
            throw new ArgumentException("Only TextContent and ImageContent are allowed in MultiModalMessage");
        }
    }

    /// <summary>
    /// Initializes a new instance of the <see cref="MultiModalData"/> with a <see cref="string"/>.
    /// </summary>
    /// <param name="text">The text to wrap.</param>
    public MultiModalData(string text)
    {
        ContentType = Type.String;
        AIContent = new TextContent(text);
    }

    /// <summary>
    /// Initializes a new instance of the <see cref="MultiModalData"/> with a <see cref="TextContent"/>.
    /// </summary>
    /// <param name="textContent">The <see cref="TextContent"/> to wrap.</param>
    public MultiModalData(TextContent textContent)
    {
        ContentType = Type.String;
        AIContent = textContent;
    }

    /// <summary>
    /// Initializes a new instance of the <see cref="MultiModalData"/> with an <see cref="DataContent"/>.
    /// </summary>
    /// <param name="image">The image to wrap.</param>
    public MultiModalData(DataContent image)
    {
        ContentType = Type.Image;
        AIContent = image;
    }

    /// <summary>
    /// Gets the <see cref="AIContent"/> wrapped by this instance.
    /// </summary>
    public Type ContentType { get; }

    /// <summary>
    /// Gets the <see cref="AIContent"/> wrapped by this instance.
    /// </summary>
    public AIContent AIContent { get; }
}

/// <summary>
/// A multi-modal message.
/// </summary>
public class MultiModalMessage : ChatMessage, IList<AIContent>
{
    /// <inheritdoc cref="IList{T}.this" />"
    public AIContent this[int index]
    {
        get => this.Content[index].AIContent;
        set => this.Content[index] = MultiModalData.CheckTypeAndCreate(value);
    }

    /// <summary>
    /// The contents of the message.
    /// </summary>
    public List<MultiModalData> Content { get; private set; } = new List<MultiModalData>();

    /// <inheritdoc cref="ICollection{AIContent}.Count" />
    public int Count => this.Content.Count;

    /// <inheritdoc cref="ICollection{AIContent}.IsReadOnly" />
    public bool IsReadOnly => false;

    /// <summary>
    /// Adds a range of <see cref="MultiModalData"/> to the message. The type does not need
    /// to be checked because it was already validated when the <see cref="MultiModalData"/>
    /// was created.
    /// </summary>
    /// <param name="items">The items to add.</param>
    internal void AddRangeUnchecked(IEnumerable<MultiModalData> items)
    {
        this.Content.AddRange(items);
    }

    /// <summary>
    /// Checks and adds a range of <see cref="AIContent"/> to the message.
    /// </summary>
    /// <param name="items">The items to add.</param>
    public void AddRange(IEnumerable<AIContent> items)
    {
        foreach (AIContent item in items)
        {
            this.Content.Add(MultiModalData.CheckTypeAndCreate(item));
        }
    }

    /// <summary>
    /// Adds a range of <see cref="string"/> to the message.
    /// </summary>
    /// <param name="textItems">The items to add.</param>
    public void AddRange(IEnumerable<TextContent> textItems)
    {
        foreach (TextContent item in textItems)
        {
            this.Add(item);
        }
    }

    /// <summary>
    /// Adds a range of <see cref="string"/> to the message.
    /// </summary>
    /// <param name="textItems">The items to add.</param>
    public void AddRange(IEnumerable<string> textItems)
    {
        foreach (string item in textItems)
        {
            this.Add(item);
        }
    }

    /// <summary>
    /// Adds a range of <see cref="DataContent"/> to the message.
    /// </summary>
    /// <param name="images">The items to add.</param>
    public void AddRange(IEnumerable<DataContent> images)
    {
        foreach (DataContent image in images)
        {
            this.Add(image);
        }
    }

    /// <summary>
    /// Checks and adds an <see cref="AIContent"/> to the message.
    /// </summary>
    /// <param name="item">The item to add.</param>
    public void Add(AIContent item)
    {
        this.Content.Add(MultiModalData.CheckTypeAndCreate(item));
    }

    /// <summary>
    /// Adds a <see cref="string"/> to the message.
    /// </summary>
    /// <param name="text">The text to add.</param>
    public void Add(string text)
    {
        this.Content.Add(new(text));
    }

    /// <summary>
    /// Adds a <see cref="TextContent"/> to the message.
    /// </summary>
    /// <param name="image">The image to add.</param>
    public void Add(DataContent image)
    {
        this.Content.Add(new(image));
    }

    /// <summary>
    /// Adds a <see cref="TextContent"/> to the message.
    /// </summary>
    /// <param name="text">The <see cref="TextContent"/> to add.</param>
    public void Add(TextContent text)
    {
        this.Content.Add(new(text));
    }

    /// <inheritdoc cref="ICollection{AIContent}.Clear" />
    public void Clear()
    {
        this.Content.Clear();
    }

    /// <inheritdoc cref="ICollection{AIContent}.Contains" />
    public bool Contains(AIContent item)
    {
        return this.Content.Any(x => x.AIContent == item);
    }

    /// <inheritdoc cref="ICollection{AIContent}.CopyTo" />
    public void CopyTo(AIContent[] array, int arrayIndex)
    {
        if (array == null)
        {
            throw new ArgumentNullException(nameof(array));
        }

        if (arrayIndex < 0 || arrayIndex >= array.Length)
        {
            throw new ArgumentOutOfRangeException(nameof(arrayIndex));
        }

        if (array.Length - arrayIndex < this.Content.Count)
        {
            throw new ArgumentException("The number of elements in the source is greater than the available space from arrayIndex to the end of the destination array.");
        }

        for (var i = 0; i < this.Content.Count; i++)
        {
            array[arrayIndex + i] = this.Content[i].AIContent;
        }
    }

    /// <inheritdoc cref="IEnumerable{AIContent}.GetEnumerator" />
    public IEnumerator<AIContent> GetEnumerator()
    {
        return this.Content.Select(x => x.AIContent).GetEnumerator();
    }

    /// <inheritdoc cref="IList{AIContent}.IndexOf" />
    public int IndexOf(AIContent item)
    {
        return this.Content.FindIndex(x => x.AIContent == item);
    }

    /// <inheritdoc cref="IList{String}.IndexOf(String)"/>
    public int IndexOf(string text)
    {
        return this.Content.FindIndex(x => x.ContentType == MultiModalData.Type.String && ((TextContent)x.AIContent).Text == text);
    }

    /// <inheritdoc cref="IList{AIContent}.Insert" />/>
    public void Insert(int index, AIContent item)
    {
        this.Content.Insert(index, MultiModalData.CheckTypeAndCreate(item));
    }

    /// <inheritdoc cref="IList{String}.Insert(int, String)"/>
    public void Insert(int index, string text)
    {
        this.Content.Insert(index, new(text));
    }

    /// <inheritdoc cref="IList{TextContent}.Insert(int, TextContent)"/>
    public void Insert(int index, TextContent text)
    {
        this.Content.Insert(index, new(text));
    }

    /// <inheritdoc cref="IList{ImageContent}.Insert(int, ImageContent)"/>
    public void Insert(int index, DataContent image)
    {
        this.Content.Insert(index, new(image));
    }

    /// <inheritdoc cref="ICollection{AIContent}.Remove" />
    public bool Remove(AIContent item)
    {
        int targetIndex = Content.FindIndex(x => x.AIContent == item);
        if (targetIndex == -1)
        {
            return false;
        }

        this.Content.RemoveAt(targetIndex);
        return true;
    }

    /// <inheritdoc cref="IList{AIContent}.RemoveAt" />
    public void RemoveAt(int index)
    {
        this.Content.RemoveAt(index);
    }

    /// <inheritdoc cref="IEnumerable.GetEnumerator" />
    IEnumerator IEnumerable.GetEnumerator()
    {
        return GetEnumerator();
    }

    /// <inheritdoc cref="ChatMessage.ToCompletionClientMessage(ChatRole)" />/>
    public override Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        StringBuilder contentBuilder = new StringBuilder();
        foreach (MultiModalData item in this.Content)
        {
            if (item.ContentType == MultiModalData.Type.String)
            {
                contentBuilder.AppendLine(item.AIContent.RawRepresentation as string ?? "");
            }
            else if (item.ContentType == MultiModalData.Type.Image)
            {
                contentBuilder.AppendLine("[Image]");
            }
        }

        return new Microsoft.Extensions.AI.ChatMessage(role, contentBuilder.ToString()) { AuthorName = this.Source };
    }
}

/// <summary>
/// A message requesting stop of a conversation.
/// </summary>
public class StopMessage : ChatMessage
{
    public required string Content { get; set; }

    public override Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        Debug.Assert(role == ChatRole.Assistant, "StopMessage can only come from agents in the Assistant Role");
        return new Microsoft.Extensions.AI.ChatMessage(ChatRole.Assistant, this.Content) { AuthorName = this.Source };
    }
}

/// <summary>
/// A message requesting handoff of a conversation to another agent.
/// </summary>
public class HandoffMessage : ChatMessage
{
    /// <summary>
    /// The name of the target agent to handoff to.
    /// </summary>
    public required string Target { get; set; }

    /// <summary>
    /// The handoff message to the target agent.
    /// </summary>
    public required string Context { get; set; }

    /// <inheritdoc cref="ChatMessage.ToCompletionClientMessage(ChatRole)" />/>
    public override Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        Debug.Assert(role == ChatRole.Assistant, "HandoffMessage can only come from agents in the Assistant Role");
        return new Microsoft.Extensions.AI.ChatMessage(ChatRole.Assistant, this.Context) { AuthorName = this.Source };
    }
}

/// <summary>
/// A request to call a function.
/// </summary>
public class FunctionCall
{
    // TODO: Should this be part of the Autogen "Core" (and what does that even mean on the .NET side?)
    // It is unfortuante that we have to duplicate this type, but in order to be compatible with Python, it is necessary for
    // us to be able to process incoming FunctionCalls with parameters in the form of a JSON string. This means that without
    // knowing the target function, and unless the types are specified inline in the JSON, we cannot deserialize them in a
    // generic manner (or we need to have a central registry of function calls, which is undesirable).
    // The solution, for now, is to keep the representation as JSON and provide a helper that binds the JSON to a candidate
    // schema.

    /// <summary>
    /// An identifier representing this specific request. Responses will include this identifier.
    /// </summary>
    public required string Id { get; set; }

    /// <summary>
    /// The arguments to pass to the function in JSON format.
    /// </summary>
    public string? Arguments { get; set; }

    /// <summary>
    /// The name of the function to call.
    /// </summary>
    public required string Name { get; set; }
}

/// <summary>
/// The result of a function call.
/// </summary>
public class FunctionExecutionResult
{
    /// <summary>
    /// The identifier of the request that this result is for.
    /// </summary>
    public required string Id { get; set; }

    /// <summary>
    /// The name of the function that was called.
    /// </summary>
    public required string Name { get; set; }

    /// <summary>
    /// The result of calling the function.
    /// </summary>
    public required string Content { get; set; }
}

/// <summary>
/// An event signaling a request to use tools.
/// </summary>
public class ToolCallRequestEvent : AgentEvent
{
    /// <summary>
    /// The tool calls.
    /// </summary>
    public List<FunctionCall> Content { get; private set; } = new List<FunctionCall>();

    /// <inheritdoc cref="AgentEvent.ToCompletionClientMessage(ChatRole)" />/>
    public override Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        Debug.Assert(role == ChatRole.Assistant, "ToolCallMessage can only come from agents in the Assistant Role");
        return new Microsoft.Extensions.AI.ChatMessage(ChatRole.Assistant, (IList<AIContent>)this.Content) { AuthorName = this.Source };
    }
}

/// <summary>
/// An event signaling the execution of tool calls.
/// </summary>
public class ToolCallExecutionEvent : AgentEvent
{
    /// <summary>
    /// The tool call results.
    /// </summary>
    public List<FunctionExecutionResult> Content { get; private set; } = new List<FunctionExecutionResult>();

    /// <inheritdoc cref="AgentEvent.ToCompletionClientMessage(ChatRole)" />/>
    public override Microsoft.Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        Debug.Assert(role == ChatRole.Tool, "ToolCallResultMessage can only come from agents in the Tool Role");
        return new Microsoft.Extensions.AI.ChatMessage(ChatRole.Tool, (IList<AIContent>)this.Content) { AuthorName = this.Source };
    }
}

/// <summary>
/// A message summarizing the results of tool calls.
/// </summary>
public class ToolCallSummaryMessage : ChatMessage
{
    /// <summary>
    /// Summary of the tool call results.
    /// </summary>
    public required string Content { get; set; }

    public override Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        Debug.Assert(role == ChatRole.Assistant, "ToolCallSummaryMessage can only come from agents in the Assistant Role");
        return new Microsoft.Extensions.AI.ChatMessage(ChatRole.Assistant, this.Content) { AuthorName = this.Source };
    }
}

/// <summary>
/// An event signaling that the user proxy has requested user input. Published prior to invoking the
/// input callback.
/// </summary>
public class UserInputRequestedEvent : AgentEvent
{
    /// <summary>
    /// Identifier for the user input request.
    /// </summary>
    public required string RequestId { get; set; }

    /// <inheritdoc cref="AgentEvent.ToCompletionClientMessage(ChatRole)" />/>
    public override Extensions.AI.ChatMessage ToCompletionClientMessage(ChatRole role)
    {
        throw new Exception("UserInputRequestedEvent should not be sent to the completion client");
    }
}

public static class CompletionChatMessageExtensions
{
    /// <summary>
    /// Flattens a <see cref="Microsoft.Extensions.AI.ChatMessage"/> into a single <see cref="Microsoft.Extensions.AI.ChatMessage"/>
    /// containing all of the content in the original message as a single string.
    /// </summary>
    /// <remarks>
    /// </remarks>
    /// <param name="msg">
    /// The <see cref="Microsoft.Extensions.AI.ChatMessage"/> to flatten.
    /// </param>
    /// <returns>
    /// A new <see cref="Microsoft.Extensions.AI.ChatMessage"/> that is a flattened version of the input.
    /// </returns>
    public static Microsoft.Extensions.AI.ChatMessage Flatten(this Microsoft.Extensions.AI.ChatMessage msg)
    {
        if (msg.Contents.Count == 1 && msg.Contents[0] is TextContent)
        {
            return msg;
        }

        StringBuilder contentBuilder = new StringBuilder();
        foreach (AIContent content in msg.Contents)
        {
            if (content is TextContent textContent)
            {
                contentBuilder.AppendLine(textContent.Text);
            }
            else if (content is DataContent)
            {
                contentBuilder.AppendLine("[Image]");
            }
            else
            {
                contentBuilder.AppendLine($"[{content.GetType().Name}]");
            }
        }

        return new Microsoft.Extensions.AI.ChatMessage(msg.Role, contentBuilder.ToString())
        {
            AuthorName = msg.AuthorName,
            AdditionalProperties = msg.AdditionalProperties
        };
    }
}

public static class MessageSerializationHelpers
{
    internal sealed class TypeNode(Type type)
    {
        public Type Type { get; } = type;
        public TypeNode? Parent { get; set; }
        public TypeNode Root => this.Parent?.Root ?? this;
        public List<TypeNode> Children { get; } = new List<TypeNode>();

        public IEnumerable<Type> ChildrenTransitiveClosure
        {
            get
            {
                return this.Children.Select(c => c.Type)
                                    .Concat(Children.SelectMany(c => c.ChildrenTransitiveClosure));
            }
        }
    }

    internal sealed class TypeTree
    {
        private static IEnumerable<Type> GetDerivedTypes(Type type)
        {
            // Across all assemblies
            foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
            {
                // Get all types in the assembly
                foreach (var derivedType in assembly.GetTypes().Where(t => type.IsAssignableFrom(t) && t != type))
                {
                    yield return derivedType;
                }
            }
        }

        private TypeNode EnsureTypeNode(Type type)
        {
            if (!this.TypeNodes.TryGetValue(type, out TypeNode? currentNode))
            {
                currentNode = this.TypeNodes[type] = new TypeNode(type);
            }

            return currentNode;
        }

        private void EnsureType(Type type)
        {
            TypeNode? currentNode = this.EnsureTypeNode(type);

            while (currentNode != null &&
                   currentNode.Parent == null &&
                   !this.RootTypes.Contains(currentNode.Type))
            {
                Type parentType = currentNode.Type.BaseType
                                  ?? throw new InvalidOperationException("We should never have a non-Root, underived base");

                TypeNode parentNode = this.EnsureTypeNode(parentType);
                currentNode.Parent = parentNode;
                parentNode.Children.Add(currentNode);

                currentNode = parentNode;
            }
        }

        public HashSet<Type> RootTypes { get; }
        public Dictionary<Type, TypeNode> TypeNodes { get; } = new Dictionary<Type, TypeNode>();

        public TypeTree(params Type[] rootTypes)
        {
            this.RootTypes = new HashSet<Type>();
            foreach (var rootType in rootTypes)
            {
                // Check that there are no other types that this type derives from in the root types
                // or vice versa
                if (this.RootTypes.Any(t => t.IsAssignableFrom(rootType) || rootType.IsAssignableFrom(t)))
                {
                    throw new ArgumentException($"Root types cannot be derived from each other: {rootType.Name}");
                }

                this.RootTypes.Add(rootType);

                this.EnsureType(rootType);

                foreach (var derivedType in GetDerivedTypes(rootType))
                {
                    this.EnsureType(derivedType);
                }
            }
        }
    }

    internal static readonly TypeTree MessageTypeTree = new(typeof(AgentMessage), typeof(GroupChatEventBase));

    internal sealed class MessagesTypeInfoResolver : DefaultJsonTypeInfoResolver
    {
        public override JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options)
        {
            JsonTypeInfo baseTypeInfo = base.GetTypeInfo(type, options);

            if (MessageTypeTree.TypeNodes.TryGetValue(type, out TypeNode? typeNode) &&
                typeNode.Children.Any()) // Only add polymorphism info if there are derived children
            {
                if (baseTypeInfo.PolymorphismOptions == null)
                {
                    baseTypeInfo.PolymorphismOptions = new JsonPolymorphismOptions();
                }

                baseTypeInfo.PolymorphismOptions.IgnoreUnrecognizedTypeDiscriminators = true;
                baseTypeInfo.PolymorphismOptions.UnknownDerivedTypeHandling = JsonUnknownDerivedTypeHandling.FailSerialization;

                foreach (Type childType in typeNode.ChildrenTransitiveClosure)
                {
                    if (childType.IsAbstract || childType.IsInterface || childType.IsGenericTypeDefinition)
                    {
                        // Can only deserialize concrete, complete types.
                        continue;
                    }

                    baseTypeInfo.PolymorphismOptions.DerivedTypes.Add(new JsonDerivedType(childType, childType.FullName ?? childType.Name));
                }
            }

            return baseTypeInfo;
        }
    }
}