A programming framework for agentic AI
update openai model support (#1082)
Chi Wange1da7f7d68a5
diff --git a/flaml/autogen/agent/assistant_agent.py b/flaml/autogen/agent/assistant_agent.pyindex 09e0ae076..0381cfdb9 100644--- a/flaml/autogen/agent/assistant_agent.py+++ b/flaml/autogen/agent/assistant_agent.py@@ -39,6 +39,7 @@ class AssistantAgent(Agent):self._conversations[sender.name] = [{"content": self._system_message, "role": "system"}]super().receive(message, sender)responses = oai.ChatCompletion.create(messages=self._conversations[sender.name], **self._config)+ # TODO: handle function_callresponse = oai.ChatCompletion.extract_text(responses)[0]self._send(response, sender)diff --git a/flaml/autogen/oai/completion.py b/flaml/autogen/oai/completion.pyindex e7b4a5031..d43755982 100644--- a/flaml/autogen/oai/completion.py+++ b/flaml/autogen/oai/completion.py@@ -45,12 +45,16 @@ class Completion(openai_Completion):# set of models that support chat completionchat_models = {"gpt-3.5-turbo",- "gpt-3.5-turbo-0301",+ "gpt-3.5-turbo-0301", # deprecate in Sep+ "gpt-3.5-turbo-0613",+ "gpt-3.5-turbo-16k","gpt-35-turbo","gpt-4","gpt-4-32k",- "gpt-4-32k-0314",- "gpt-4-0314",+ "gpt-4-32k-0314", # deprecate in Sep+ "gpt-4-0314", # deprecate in Sep+ "gpt-4-0613",+ "gpt-4-32k-0613",}# price per 1k tokens@@ -62,13 +66,17 @@ class Completion(openai_Completion):"code-davinci-002": 0.1,"text-davinci-002": 0.02,"text-davinci-003": 0.02,- "gpt-3.5-turbo": 0.002,- "gpt-3.5-turbo-0301": 0.002,+ "gpt-3.5-turbo": (0.0015, 0.002),+ "gpt-3.5-turbo-0301": (0.0015, 0.002), # deprecate in Sep+ "gpt-3.5-turbo-0613": (0.0015, 0.002),+ "gpt-3.5-turbo-16k": (0.003, 0.004),"gpt-35-turbo": 0.002,"gpt-4": (0.03, 0.06),- "gpt-4-0314": (0.03, 0.06),"gpt-4-32k": (0.06, 0.12),- "gpt-4-32k-0314": (0.06, 0.12),+ "gpt-4-0314": (0.03, 0.06), # deprecate in Sep+ "gpt-4-32k-0314": (0.06, 0.12), # deprecate in Sep+ "gpt-4-0613": (0.03, 0.06),+ "gpt-4-32k-0613": (0.06, 0.12),}default_search_space = {@@ -386,7 +394,7 @@ class Completion(openai_Completion):result["cost"] = costreturn result# evaluate the quality of the responses- responses = cls.extract_text(response)+ responses = cls.extract_text_or_function_call(response)usage = response["usage"]n_input_tokens = usage["prompt_tokens"]n_output_tokens = usage.get("completion_tokens", 0)@@ -898,7 +906,7 @@ class Completion(openai_Completion):response = cls.create(data_i, use_cache, **config)cost += response["cost"]# evaluate the quality of the responses- responses = cls.extract_text(response)+ responses = cls.extract_text_or_function_call(response)if eval_func is not None:metrics = eval_func(responses, **data_i)elif hasattr(cls, "_eval_func"):@@ -991,6 +999,24 @@ class Completion(openai_Completion):return [choice["text"] for choice in choices]return [choice["message"].get("content", "") for choice in choices]+ @classmethod+ def extract_text_or_function_call(cls, response: dict) -> List[str]:+ """Extract the text or function calls from a completion or chat response.++ Args:+ response (dict): The response from OpenAI API.++ Returns:+ A list of text or function calls in the responses.+ """+ choices = response["choices"]+ if "text" in choices[0]:+ return [choice["text"] for choice in choices]+ return [+ choice["message"] if "function_call" in choice["message"] else choice["message"].get("content", "")+ for choice in choices+ ]+@classmethod@propertydef logged_history(cls) -> Dict:diff --git a/setup.py b/setup.pyindex 33c860477..c29272b38 100644--- a/setup.py+++ b/setup.py@@ -127,8 +127,8 @@ setuptools.setup("pytorch-forecasting>=0.9.0",],"benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],- "openai": ["openai==0.27.4", "diskcache"],- "autogen": ["openai==0.27.4", "diskcache", "docker"],+ "openai": ["openai==0.27.8", "diskcache"],+ "autogen": ["openai==0.27.8", "diskcache", "docker"],"synapse": ["joblibspark>=0.5.0","optuna==2.8.0",diff --git a/test/autogen/test_assistant_agent.py b/test/autogen/test_assistant_agent.pyindex c5230930b..46bacb884 100644--- a/test/autogen/test_assistant_agent.py+++ b/test/autogen/test_assistant_agent.py@@ -11,10 +11,10 @@ def test_gpt35(human_input_mode="NEVER", max_consecutive_auto_reply=5):import openaiexcept ImportError:return- config_list = oai.config_list_from_models(key_file_path=KEY_LOC, model_list=["gpt-3.5-turbo"])+ config_list = oai.config_list_from_models(key_file_path=KEY_LOC, model_list=["gpt-3.5-turbo-0613"])assistant = AssistantAgent("coding_agent",- request_timeout=600,+ # request_timeout=600,seed=40,max_tokens=1024,config_list=config_list,diff --git a/test/autogen/test_function_call.py b/test/autogen/test_function_call.pynew file mode 100644index 000000000..9a86ae941--- /dev/null+++ b/test/autogen/test_function_call.py@@ -0,0 +1,63 @@+try:+ import openai+except ImportError:+ openai = None+import pytest+import json+from flaml import oai+from flaml.autogen.math_utils import eval_math_responses++KEY_LOC = "test/autogen"++[email protected](openai is None, reason="openai not installed")+def test_eval_math_responses():+ config_list = oai.config_list_openai_aoai(KEY_LOC, exclude="aoai")+ functions = [+ {+ "name": "eval_math_responses",+ "description": "Select a response for a math problem using voting, and check if the response is correct if the solution is provided",+ "parameters": {+ "type": "object",+ "properties": {+ "responses": {+ "type": "string",+ "description": "The responses in a list",+ },+ "solution": {+ "type": "string",+ "description": "The canonical solution",+ },+ },+ "required": ["responses"],+ },+ },+ ]+ response = oai.ChatCompletion.create(+ model="gpt-3.5-turbo-0613",+ config_list=config_list,+ messages=[+ {+ "role": "user",+ "content": 'evaluate the math responses ["1", "5/2", "5/2"] against the true answer \\frac{5}{2}',+ },+ ],+ functions=functions,+ )+ print(response)+ responses = oai.ChatCompletion.extract_text_or_function_call(response)+ print(responses[0])+ function_call = responses[0]["function_call"]+ name, arguments = function_call["name"], json.loads(function_call["arguments"])+ assert name == "eval_math_responses"+ print(arguments["responses"])+ if isinstance(arguments["responses"], str):+ arguments["responses"] = json.loads(arguments["responses"])+ arguments["responses"] = [f"\\boxed{{{x}}}" for x in arguments["responses"]]+ print(arguments["responses"])+ arguments["solution"] = f"\\boxed{{{arguments['solution']}}}"+ print(eval_math_responses(**arguments))+++if __name__ == "__main__":+ test_eval_math_responses()diff --git a/website/docs/Use-Cases/Auto-Generation.md b/website/docs/Use-Cases/Auto-Generation.mdindex 4838668de..9f00f1e13 100644--- a/website/docs/Use-Cases/Auto-Generation.md+++ b/website/docs/Use-Cases/Auto-Generation.md@@ -371,14 +371,14 @@ Set `compact=False` in `start_logging()` to switch.},}```-It can be seen that the individual API call history contain redundant information of the conversation. For a long conversation the degree of redundancy is high.+It can be seen that the individual API call history contains redundant information of the conversation. For a long conversation the degree of redundancy is high.The compact history is more efficient and the individual API call history contains more details.### Other Utilities- a [`cost`](../reference/autogen/oai/completion#cost) function to calculate the cost of an API call.- a [`test`](../reference/autogen/oai/completion#test) function to conveniently evaluate the configuration over test data.-- a [`extract_text`](../reference/autogen/oai/completion#extract_text) function to extract the text from a completion or chat response.+- an [`extract_text_or_function_call`](../reference/autogen/oai/completion#extract_text_or_function_call) function to extract the text or function call from a completion or chat response.## Agents (Experimental)