A programming framework for agentic AI
Squash changes (#2849)
David Luongd578d0dfd949
diff --git a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs b/dotnet/src/AutoGen.Anthropic/AnthropicClient.csindex 8ea0bef86..90bd33683 100644--- a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs+++ b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs@@ -23,7 +23,8 @@ public sealed class AnthropicClient : IDisposableprivate static readonly JsonSerializerOptions JsonSerializerOptions = new(){- DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull+ DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,+ Converters = { new ContentBaseConverter() }};private static readonly JsonSerializerOptions JsonDeserializerOptions = new()diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.csindex f8967bc7e..0c1749eaa 100644--- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs+++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs@@ -49,9 +49,15 @@ public class ChatMessagepublic string Role { get; set; }[JsonPropertyName("content")]- public string Content { get; set; }+ public List<ContentBase> Content { get; set; }public ChatMessage(string role, string content)+ {+ Role = role;+ Content = new List<ContentBase>() { new TextContent { Text = content } };+ }++ public ChatMessage(string role, List<ContentBase> content){Role = role;Content = content;diff --git a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.csindex bfe791909..bb2f5820f 100644--- a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs+++ b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs@@ -4,6 +4,7 @@using System;using System.Collections.Generic;using System.Linq;+using System.Net.Http;using System.Runtime.CompilerServices;using System.Threading;using System.Threading.Tasks;@@ -19,7 +20,7 @@ public class AnthropicMessageConnector : IStreamingMiddlewarepublic async Task<IMessage> InvokeAsync(MiddlewareContext context, IAgent agent, CancellationToken cancellationToken = default){var messages = context.Messages;- var chatMessages = ProcessMessage(messages, agent);+ var chatMessages = await ProcessMessageAsync(messages, agent);var response = await agent.GenerateReplyAsync(chatMessages, context.Options, cancellationToken);return response is IMessage<ChatCompletionResponse> chatMessage@@ -31,7 +32,7 @@ public class AnthropicMessageConnector : IStreamingMiddleware[EnumeratorCancellation] CancellationToken cancellationToken = default){var messages = context.Messages;- var chatMessages = ProcessMessage(messages, agent);+ var chatMessages = await ProcessMessageAsync(messages, agent);await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)){@@ -53,60 +54,78 @@ public class AnthropicMessageConnector : IStreamingMiddlewareprivate IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage<ChatCompletionResponse> chatMessage,IStreamingAgent agent){- Delta? delta = chatMessage.Content.Delta;+ var delta = chatMessage.Content.Delta;return delta != null && !string.IsNullOrEmpty(delta.Text)? new TextMessageUpdate(role: Role.Assistant, delta.Text, from: agent.Name): null;}- private IEnumerable<IMessage> ProcessMessage(IEnumerable<IMessage> messages, IAgent agent)+ private async Task<IEnumerable<IMessage>> ProcessMessageAsync(IEnumerable<IMessage> messages, IAgent agent){- return messages.SelectMany<IMessage, IMessage>(m =>+ var processedMessages = new List<IMessage>();++ foreach (var message in messages){- return m switch+ var processedMessage = message switch{TextMessage textMessage => ProcessTextMessage(textMessage, agent),- _ => [m],++ ImageMessage imageMessage =>+ new MessageEnvelope<ChatMessage>(new ChatMessage("user",+ new ContentBase[] { new ImageContent { Source = await ProcessImageSourceAsync(imageMessage) } }+ .ToList()),+ from: agent.Name),++ MultiModalMessage multiModalMessage => await ProcessMultiModalMessageAsync(multiModalMessage, agent),+ _ => message,};- });++ processedMessages.Add(processedMessage);+ }++ return processedMessages;}private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from){if (response.Content is null)+ {throw new ArgumentNullException(nameof(response.Content));+ }if (response.Content.Count != 1)+ {throw new NotSupportedException($"{nameof(response.Content)} != 1");+ }return new TextMessage(Role.Assistant, ((TextContent)response.Content[0]).Text ?? string.Empty, from: from.Name);}- private IEnumerable<IMessage<ChatMessage>> ProcessTextMessage(TextMessage textMessage, IAgent agent)+ private IMessage<ChatMessage> ProcessTextMessage(TextMessage textMessage, IAgent agent){- IEnumerable<ChatMessage> messages;+ ChatMessage messages;if (textMessage.From == agent.Name){- messages = [new ChatMessage(- "assistant", textMessage.Content)];+ messages = new ChatMessage(+ "assistant", textMessage.Content);}else if (textMessage.From is null){if (textMessage.Role == Role.User){- messages = [new ChatMessage(- "user", textMessage.Content)];+ messages = new ChatMessage(+ "user", textMessage.Content);}else if (textMessage.Role == Role.Assistant){- messages = [new ChatMessage(- "assistant", textMessage.Content)];+ messages = new ChatMessage(+ "assistant", textMessage.Content);}else if (textMessage.Role == Role.System){- messages = [new ChatMessage(- "system", textMessage.Content)];+ messages = new ChatMessage(+ "system", textMessage.Content);}else{@@ -116,10 +135,61 @@ public class AnthropicMessageConnector : IStreamingMiddlewareelse{// if from is not null, then the message is from user- messages = [new ChatMessage(- "user", textMessage.Content)];+ messages = new ChatMessage(+ "user", textMessage.Content);}- return messages.Select(m => new MessageEnvelope<ChatMessage>(m, from: textMessage.From));+ return new MessageEnvelope<ChatMessage>(messages, from: textMessage.From);+ }++ private async Task<IMessage> ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent)+ {+ var content = new List<ContentBase>();+ foreach (var message in multiModalMessage.Content)+ {+ switch (message)+ {+ case TextMessage textMessage when textMessage.GetContent() is not null:+ content.Add(new TextContent { Text = textMessage.GetContent() });+ break;+ case ImageMessage imageMessage:+ content.Add(new ImageContent() { Source = await ProcessImageSourceAsync(imageMessage) });+ break;+ }+ }++ var chatMessage = new ChatMessage("user", content);+ return MessageEnvelope.Create(chatMessage, agent.Name);+ }++ private async Task<ImageSource> ProcessImageSourceAsync(ImageMessage imageMessage)+ {+ if (imageMessage.Data != null)+ {+ return new ImageSource+ {+ MediaType = imageMessage.Data.MediaType,+ Data = Convert.ToBase64String(imageMessage.Data.ToArray())+ };+ }++ if (imageMessage.Url is null)+ {+ throw new InvalidOperationException("Invalid ImageMessage, the data or url must be provided");+ }++ var uri = new Uri(imageMessage.Url);+ using var client = new HttpClient();+ var response = client.GetAsync(uri).Result;+ if (!response.IsSuccessStatusCode)+ {+ throw new HttpRequestException($"Failed to download the image from {uri}");+ }++ return new ImageSource+ {+ MediaType = "image/jpeg",+ Data = Convert.ToBase64String(await response.Content.ReadAsByteArrayAsync())+ };}}diff --git a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.csindex 251d3c110..52967d6ff 100644--- a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs+++ b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs@@ -49,7 +49,6 @@ public class MiddlewareStreamingAgent : IMiddlewareStreamAgentpublic IAsyncEnumerable<IStreamingMessage> GenerateStreamingReplyAsync(IEnumerable<IMessage> messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default){-return _agent.GenerateStreamingReplyAsync(messages, options, cancellationToken);}diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.csindex ba31f2297..d29025b44 100644--- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs+++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs@@ -1,31 +1,108 @@// Copyright (c) Microsoft Corporation. All rights reserved.// AnthropicClientAgentTest.cs+using AutoGen.Anthropic.DTO;using AutoGen.Anthropic.Extensions;using AutoGen.Anthropic.Utils;+using AutoGen.Core;using AutoGen.Tests;-using Xunit.Abstractions;+using FluentAssertions;-namespace AutoGen.Anthropic;+namespace AutoGen.Anthropic.Tests;public class AnthropicClientAgentTest{- private readonly ITestOutputHelper _output;-- public AnthropicClientAgentTest(ITestOutputHelper output) => _output = output;-[ApiKeyFact("ANTHROPIC_API_KEY")]public async Task AnthropicAgentChatCompletionTestAsync(){var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);+ var agent = new AnthropicClientAgent(+ client,+ name: "AnthropicAgent",+ AnthropicConstants.Claude3Haiku,+ systemMessage: "You are a helpful AI assistant that convert user message to upper case")+ .RegisterMessageConnector();++ var uppCaseMessage = new TextMessage(Role.User, "abcdefg");++ var reply = await agent.SendAsync(chatHistory: new[] { uppCaseMessage });++ reply.GetContent().Should().Contain("ABCDEFG");+ reply.From.Should().Be(agent.Name);+ }++ [ApiKeyFact("ANTHROPIC_API_KEY")]+ public async Task AnthropicAgentTestProcessImageAsync()+ {+ var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);var agent = new AnthropicClientAgent(client,name: "AnthropicAgent",AnthropicConstants.Claude3Haiku).RegisterMessageConnector();- var singleAgentTest = new SingleAgentTest(_output);- await singleAgentTest.UpperCaseTestAsync(agent);- await singleAgentTest.UpperCaseStreamingTestAsync(agent);+ var base64Image = await AnthropicTestUtils.Base64FromImageAsync("square.png");+ var imageMessage = new ChatMessage("user",+ [new ImageContent { Source = new ImageSource { MediaType = "image/png", Data = base64Image } }]);++ var messages = new IMessage[] { MessageEnvelope.Create(imageMessage) };++ // test streaming+ foreach (var message in messages)+ {+ var reply = agent.GenerateStreamingReplyAsync([message]);++ await foreach (var streamingMessage in reply)+ {+ streamingMessage.Should().BeOfType<TextMessageUpdate>();+ streamingMessage.As<TextMessageUpdate>().From.Should().Be(agent.Name);+ }+ }+ }++ [ApiKeyFact("ANTHROPIC_API_KEY")]+ public async Task AnthropicAgentTestMultiModalAsync()+ {+ var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);+ var agent = new AnthropicClientAgent(+ client,+ name: "AnthropicAgent",+ AnthropicConstants.Claude3Haiku)+ .RegisterMessageConnector();++ var image = Path.Combine("images", "square.png");+ var binaryData = BinaryData.FromBytes(await File.ReadAllBytesAsync(image), "image/png");+ var imageMessage = new ImageMessage(Role.User, binaryData);+ var textMessage = new TextMessage(Role.User, "What's in this image?");+ var multiModalMessage = new MultiModalMessage(Role.User, [textMessage, imageMessage]);++ var reply = await agent.SendAsync(multiModalMessage);+ reply.Should().BeOfType<TextMessage>();+ reply.GetRole().Should().Be(Role.Assistant);+ reply.GetContent().Should().NotBeNullOrEmpty();+ reply.From.Should().Be(agent.Name);+ }++ [ApiKeyFact("ANTHROPIC_API_KEY")]+ public async Task AnthropicAgentTestImageMessageAsync()+ {+ var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);+ var agent = new AnthropicClientAgent(+ client,+ name: "AnthropicAgent",+ AnthropicConstants.Claude3Haiku,+ systemMessage: "You are a helpful AI assistant that is capable of determining what an image is. Tell me a brief description of the image."+ )+ .RegisterMessageConnector();++ var image = Path.Combine("images", "square.png");+ var binaryData = BinaryData.FromBytes(await File.ReadAllBytesAsync(image), "image/png");+ var imageMessage = new ImageMessage(Role.User, binaryData);++ var reply = await agent.SendAsync(imageMessage);+ reply.Should().BeOfType<TextMessage>();+ reply.GetRole().Should().Be(Role.Assistant);+ reply.GetContent().Should().NotBeNullOrEmpty();+ reply.From.Should().Be(agent.Name);}}diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.csindex af050eef9..a0b1f60cf 100644--- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs+++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs@@ -7,7 +7,7 @@ using AutoGen.Tests;using FluentAssertions;using Xunit;-namespace AutoGen.Anthropic;+namespace AutoGen.Anthropic.Tests;public class AnthropicClientTests{@@ -73,6 +73,41 @@ public class AnthropicClientTestsresults.First().streamingMessage!.Role.Should().Be("assistant");}+ [ApiKeyFact("ANTHROPIC_API_KEY")]+ public async Task AnthropicClientImageChatCompletionTestAsync()+ {+ var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);++ var request = new ChatCompletionRequest();+ request.Model = AnthropicConstants.Claude3Haiku;+ request.Stream = false;+ request.MaxTokens = 100;+ request.SystemMessage = "You are a LLM that is suppose to describe the content of the image. Give me a description of the provided image.";++ var base64Image = await AnthropicTestUtils.Base64FromImageAsync("square.png");+ var messages = new List<ChatMessage>+ {+ new("user",+ [+ new ImageContent { Source = new ImageSource {MediaType = "image/png", Data = base64Image} }+ ])+ };++ request.Messages = messages;++ var response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None);++ Assert.NotNull(response);+ Assert.NotNull(response.Content);+ Assert.NotEmpty(response.Content);+ response.Content.Count.Should().Be(1);+ response.Content.First().Should().BeOfType<TextContent>();+ var textContent = (TextContent)response.Content.First();+ Assert.Equal("text", textContent.Type);+ Assert.NotNull(response.Usage);+ response.Usage.OutputTokens.Should().BeGreaterThan(0);+ }+private sealed class Person{[JsonPropertyName("name")]diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.csindex a5b80eee3..de630da6d 100644--- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs+++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs@@ -1,10 +1,16 @@// Copyright (c) Microsoft Corporation. All rights reserved.// AnthropicTestUtils.cs-namespace AutoGen.Anthropic;+namespace AutoGen.Anthropic.Tests;public static class AnthropicTestUtils{public static string ApiKey => Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ??throw new Exception("Please set ANTHROPIC_API_KEY environment variable.");++ public static async Task<string> Base64FromImageAsync(string imageName)+ {+ return Convert.ToBase64String(+ await File.ReadAllBytesAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "images", imageName)));+ }}diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csprojindex 8ce405a07..0f22d9fe6 100644--- a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj+++ b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj@@ -13,4 +13,10 @@<ProjectReference Include="..\..\src\AutoGen.Anthropic\AutoGen.Anthropic.csproj" /><ProjectReference Include="..\AutoGen.Tests\AutoGen.Tests.csproj" /></ItemGroup>++ <ItemGroup>+ <None Update="images\square.png">+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>+ </None>+ </ItemGroup></Project>diff --git a/dotnet/test/AutoGen.Anthropic.Tests/images/.gitattributes b/dotnet/test/AutoGen.Anthropic.Tests/images/.gitattributesnew file mode 100644index 000000000..56e7c34d4--- /dev/null+++ b/dotnet/test/AutoGen.Anthropic.Tests/images/.gitattributes@@ -0,0 +1 @@+square.png filter=lfs diff=lfs merge=lfs -textdiff --git a/dotnet/test/AutoGen.Anthropic.Tests/images/square.png b/dotnet/test/AutoGen.Anthropic.Tests/images/square.pngnew file mode 100644index 000000000..5c2b3ed82--- /dev/null+++ b/dotnet/test/AutoGen.Anthropic.Tests/images/square.png@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:8341030e5b93aab2c55dcd40ffa26ced8e42cc15736a8348176ffd155ad2d937+size 8167