A programming framework for agentic AI
Skip tests that depend on OpenAI via `--skip-openai` (#1097)
Maxim Saplinc80df8acab49
diff --git a/.github/workflows/build.yml b/.github/workflows/build.ymlindex 8d4b84a30..2f1af9ab7 100644--- a/.github/workflows/build.yml+++ b/.github/workflows/build.yml@@ -41,17 +41,15 @@ jobs:pip install -e .python -c "import autogen"pip install -e. pytest mock- pip uninstall -y openai- name: Test with pytestif: matrix.python-version != '3.10'run: |- pytest test+ pytest test --skip-openai- name: Coverageif: matrix.python-version == '3.10'run: |pip install -e .[test]- pip uninstall -y openai- coverage run -a -m pytest test --ignore=test/agentchat/contrib+ coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openaicoverage xml- name: Upload coverage to Codecovif: matrix.python-version == '3.10'diff --git a/.gitignore b/.gitignoreindex 70bb7c9bf..03c569500 100644--- a/.gitignore+++ b/.gitignore@@ -171,3 +171,7 @@ test/my_tmp/*# Storage for the AgentEval outputtest/test_files/agenteval-in-out/out/++# Files created by tests+*tmp_code_*+test/agentchat/test_agent_scripts/*diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.pyindex 709ca31f2..7380d6e62 100644--- a/test/agentchat/contrib/test_agent_builder.py+++ b/test/agentchat/contrib/test_agent_builder.py@@ -5,6 +5,7 @@ import sysfrom packaging.requirements import Requirementfrom autogen.agentchat.contrib.agent_builder import AgentBuilderfrom autogen import UserProxyAgent+from conftest import skip_openaisys.path.append(os.path.join(os.path.dirname(__file__), ".."))from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402@@ -20,15 +21,15 @@ try:from openai.types.completion import Completionfrom openai.types.completion_usage import CompletionUsageimport diskcache-- OPENAI_INSTALLED = Trueexcept ImportError:- OPENAI_INSTALLED = False+ skip = True+else:+ skip = False or skip_openai@pytest.mark.skipif(- not OPENAI_INSTALLED,- reason="do not run when dependency is not installed",+ skip,+ reason="openai not installed OR requested to skip",)def test_build():builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")@@ -57,8 +58,8 @@ def test_build():@pytest.mark.skipif(- not OPENAI_INSTALLED,- reason="do not run when dependency is not installed",+ skip,+ reason="openai not installed OR requested to skip",)def test_save():builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")@@ -93,8 +94,8 @@ def test_save():@pytest.mark.skipif(- not OPENAI_INSTALLED,- reason="do not run when dependency is not installed",+ skip,+ reason="openai not installed OR requested to skip",)def test_load():builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")@@ -128,8 +129,8 @@ def test_load():@pytest.mark.skipif(- not OPENAI_INSTALLED,- reason="do not run when dependency is not installed",+ skip,+ reason="openai not installed OR requested to skip",)def test_clear_agent():builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")diff --git a/test/agentchat/contrib/test_compressible_agent.py b/test/agentchat/contrib/test_compressible_agent.pyindex 089db650e..47a11ef16 100644--- a/test/agentchat/contrib/test_compressible_agent.py+++ b/test/agentchat/contrib/test_compressible_agent.py@@ -2,6 +2,7 @@ import pytestimport sysimport autogenimport os+from conftest import skip_openaifrom autogen.agentchat.contrib.compressible_agent import CompressibleAgenthere = os.path.abspath(os.path.dirname(__file__))@@ -19,15 +20,15 @@ config_list = autogen.config_list_from_json(try:import openai-- OPENAI_INSTALLED = Trueexcept ImportError:- OPENAI_INSTALLED = False+ skip = True+else:+ skip = False or skip_openai@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_mode_compress():conversations = {}@@ -65,8 +66,8 @@ def test_mode_compress():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_mode_customized():try:@@ -135,8 +136,8 @@ def test_mode_customized():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_compress_message():assistant = CompressibleAgent(@@ -169,6 +170,10 @@ def test_compress_message():assert is_success, "Compression failed."+ skip,+ reason="do not run if dependency is not installed OR requested to skip",+)def test_mode_terminate():assistant = CompressibleAgent(name="assistant",diff --git a/test/agentchat/contrib/test_gpt_assistant.py b/test/agentchat/contrib/test_gpt_assistant.pyindex f7226081e..fe92501c4 100644--- a/test/agentchat/contrib/test_gpt_assistant.py+++ b/test/agentchat/contrib/test_gpt_assistant.py@@ -3,6 +3,7 @@ import osimport sysimport autogenfrom autogen import OpenAIWrapper+from conftest import skip_openaisys.path.append(os.path.join(os.path.dirname(__file__), ".."))from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402@@ -11,10 +12,10 @@ try:import openaifrom autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgentfrom autogen.oai.openai_utils import retrieve_assistants_by_name-- skip_test = Falseexcept ImportError:- skip_test = True+ skip = True+else:+ skip = False or skip_openaiconfig_list = autogen.config_list_from_json(OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"api_type": ["openai"]}@@ -26,8 +27,8 @@ def ask_ossinsight(question):@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_gpt_assistant_chat():ossinsight_api_schema = {@@ -73,8 +74,8 @@ def test_gpt_assistant_chat():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_get_assistant_instructions():"""@@ -97,8 +98,8 @@ def test_get_assistant_instructions():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_gpt_assistant_instructions_overwrite():"""@@ -142,8 +143,8 @@ def test_gpt_assistant_instructions_overwrite():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_gpt_assistant_existing_no_instructions():"""@@ -178,8 +179,8 @@ def test_gpt_assistant_existing_no_instructions():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_get_assistant_files():"""@@ -212,8 +213,8 @@ def test_get_assistant_files():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_assistant_retrieval():"""@@ -283,8 +284,8 @@ def test_assistant_retrieval():@pytest.mark.skipif(- sys.platform in ["darwin", "win32"] or skip_test,- reason="do not run on MacOS or windows or dependency is not installed",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",)def test_assistant_mismatch_retrieval():"""Test function to check if the GPTAssistantAgent can filter out the mismatch assistant"""diff --git a/test/agentchat/contrib/test_teachable_agent.py b/test/agentchat/contrib/test_teachable_agent.pyindex 9bb607a59..b8156296b 100644--- a/test/agentchat/contrib/test_teachable_agent.py+++ b/test/agentchat/contrib/test_teachable_agent.py@@ -2,6 +2,7 @@ import pytestimport osimport sysfrom autogen import ConversableAgent, config_list_from_json+from conftest import skip_openaisys.path.append(os.path.join(os.path.dirname(__file__), ".."))from test_assistant_agent import OAI_CONFIG_LIST, KEY_LOC # noqa: E402@@ -12,7 +13,7 @@ try:except ImportError:skip = Trueelse:- skip = False+ skip = False or skip_openaitry:from termcolor import coloreddiff --git a/test/agentchat/test_assistant_agent.py b/test/agentchat/test_assistant_agent.pyindex e0b85583b..b05a1b77b 100644--- a/test/agentchat/test_assistant_agent.py+++ b/test/agentchat/test_assistant_agent.py@@ -2,23 +2,26 @@ import osimport sysimport pytestimport autogen+from conftest import skip_openaifrom autogen.agentchat import AssistantAgent, UserProxyAgent+try:+ from openai import OpenAI+except ImportError:+ skip = True+else:+ skip = False or skip_openai+KEY_LOC = "notebook"OAI_CONFIG_LIST = "OAI_CONFIG_LIST"here = os.path.abspath(os.path.dirname(__file__))@pytest.mark.skipif(- sys.platform in ["darwin", "win32"],- reason="do not run on MacOS or windows",+ sys.platform in ["darwin", "win32"] or skip,+ reason="do not run on MacOS or windows OR openai not installed OR requested to skip",)def test_ai_user_proxy_agent():- try:- import openai- except ImportError:- return-conversations = {}# autogen.ChatCompletion.start_logging(conversations)@@ -57,11 +60,8 @@ def test_ai_user_proxy_agent():print(conversations)[email protected](skip, reason="openai not installed OR requested to skip")def test_gpt35(human_input_mode="NEVER", max_consecutive_auto_reply=5):- try:- import openai- except ImportError:- returnconfig_list = autogen.config_list_from_json(OAI_CONFIG_LIST,file_location=KEY_LOC,@@ -115,12 +115,8 @@ If "Thank you" or "You\'re welcome" are said in the conversation, then say TERMIassert not isinstance(user.use_docker, bool) # None or str[email protected](skip, reason="openai not installed OR requested to skip")def test_create_execute_script(human_input_mode="NEVER", max_consecutive_auto_reply=10):- try:- import openai- except ImportError:- return-config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, file_location=KEY_LOC)conversations = {}# autogen.ChatCompletion.start_logging(conversations)@@ -160,12 +156,8 @@ print('Hello world!')# autogen.ChatCompletion.stop_logging()[email protected](skip, reason="openai not installed OR requested to skip")def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10):- try:- import openai- except ImportError:- return-config_list = autogen.config_list_from_json(OAI_CONFIG_LIST,file_location=KEY_LOC,diff --git a/test/agentchat/test_async.py b/test/agentchat/test_async.pyindex b5f2c8fee..d31f50c0c 100644--- a/test/agentchat/test_async.py+++ b/test/agentchat/test_async.py@@ -1,8 +1,16 @@import pytestimport asyncioimport autogen+from conftest import skip_openaifrom test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST+try:+ from openai import OpenAI+except ImportError:+ skip = True+else:+ skip = False or skip_openai+def get_market_news(ind, ind_upper):data = {@@ -45,13 +53,9 @@ def get_market_news(ind, ind_upper):return feeds_summary[email protected](skip, reason="openai not installed OR requested to skip")@pytest.mark.asyncioasync def test_async_groupchat():- try:- import openai- except ImportError:- return-config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC)llm_config = {@@ -91,12 +95,9 @@ async def test_async_groupchat():assert len(user_proxy.chat_messages) > 0[email protected](skip, reason="openai not installed OR requested to skip")@pytest.mark.asyncioasync def test_stream():- try:- import openai- except ImportError:- returnconfig_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC)data = asyncio.Future()diff --git a/test/agentchat/test_async_get_human_input.py b/test/agentchat/test_async_get_human_input.pyindex 2daca19f2..f9de9f1c2 100644--- a/test/agentchat/test_async_get_human_input.py+++ b/test/agentchat/test_async_get_human_input.py@@ -1,15 +1,20 @@import asyncioimport autogenimport pytest+from conftest import skip_openaifrom test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST+try:+ from openai import OpenAI+except ImportError:+ skip = True+else:+ skip = False or skip_openai+[email protected](skip, reason="openai not installed OR requested to skip")@pytest.mark.asyncioasync def test_async_get_human_input():- try:- import openai- except ImportError:- returnconfig_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC)# create an AssistantAgent instance named "assistant"diff --git a/test/agentchat/test_conversable_agent.py b/test/agentchat/test_conversable_agent.pyindex 3a23fd8fd..87868b05d 100644--- a/test/agentchat/test_conversable_agent.py+++ b/test/agentchat/test_conversable_agent.py@@ -2,6 +2,7 @@ import copyfrom typing import Any, Callable, Dict, Literalimport pytest+from unittest.mock import patchfrom pydantic import BaseModel, Fieldfrom typing_extensions import Annotated@@ -421,7 +422,7 @@ def test__wrap_function_sync():else:raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}")- agent = ConversableAgent(name="agent", llm_config={})+ agent = ConversableAgent(name="agent", llm_config=False)@agent._wrap_functiondef currency_calculator(@@ -457,7 +458,7 @@ async def test__wrap_function_async():else:raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}")- agent = ConversableAgent(name="agent", llm_config={})+ agent = ConversableAgent(name="agent", llm_config=False)@agent._wrap_functionasync def currency_calculator(diff --git a/test/agentchat/test_function_call.py b/test/agentchat/test_function_call.pyindex cfe606b08..54f1f6dff 100644--- a/test/agentchat/test_function_call.py+++ b/test/agentchat/test_function_call.py@@ -1,17 +1,21 @@-try:- from openai import OpenAI-except ImportError:- OpenAI = Noneimport pytestimport asyncioimport jsonimport autogen+from conftest import skip_openaifrom autogen.math_utils import eval_math_responsesfrom test_assistant_agent import KEY_LOC, OAI_CONFIG_LISTimport sys+try:+ from openai import OpenAI+except ImportError:+ skip = True+else:+ skip = False or skip_openai+[email protected](OpenAI is None, reason="openai>=1 not installed")[email protected](skip, reason="openai not installed OR requested to skip")def test_eval_math_responses():config_list = autogen.config_list_from_models(KEY_LOC, exclude="aoai", model_list=["gpt-4-0613", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k"]@@ -190,8 +194,8 @@ async def test_a_execute_function():@pytest.mark.skipif(- not OpenAI or not sys.version.startswith("3.10"),- reason="do not run if openai is not installed or py!=3.10",+ skip or not sys.version.startswith("3.10"),+ reason="do not run if openai is not installed OR reeusted to skip OR py!=3.10",)def test_update_function():config_list_gpt4 = autogen.config_list_from_json(diff --git a/test/agentchat/test_function_call_groupchat.py b/test/agentchat/test_function_call_groupchat.pyindex d8fdf30d3..02fe14a3e 100644--- a/test/agentchat/test_function_call_groupchat.py+++ b/test/agentchat/test_function_call_groupchat.py@@ -1,6 +1,7 @@import autogenimport pytestimport sys+from conftest import skip_openaifrom test_assistant_agent import KEY_LOC, OAI_CONFIG_LISTtry:@@ -8,7 +9,7 @@ try:except ImportError:skip = Trueelse:- skip = False+ skip = False or skip_openai@pytest.mark.skipif(diff --git a/test/agentchat/test_math_user_proxy_agent.py b/test/agentchat/test_math_user_proxy_agent.pyindex bb825ddfb..787a63f04 100644--- a/test/agentchat/test_math_user_proxy_agent.py+++ b/test/agentchat/test_math_user_proxy_agent.py@@ -6,6 +6,7 @@ from autogen.agentchat.contrib.math_user_proxy_agent import (_remove_print,_add_print_to_last_line,)+from conftest import skip_openaifrom test_assistant_agent import KEY_LOC, OAI_CONFIG_LISTtry:@@ -13,7 +14,7 @@ try:except ImportError:skip = Trueelse:- skip = False+ skip = False or skip_openai@pytest.mark.skipif(diff --git a/test/conftest.py b/test/conftest.pynew file mode 100644index 000000000..8cf1762b4--- /dev/null+++ b/test/conftest.py@@ -0,0 +1,16 @@+import pytest++skip_openai = False+++# Registers command-line option '--skip-openai' via pytest hook.+# When this flag is set, it indicates that tests requiring OpenAI should be skipped.+def pytest_addoption(parser):+ parser.addoption("--skip-openai", action="store_true", help="Skip all tests that require openai")+++# pytest hook implementation extracting the '--skip-openai' command line arg and exposing it globally[email protected](tryfirst=True)+def pytest_configure(config):+ global skip_openai+ skip_openai = config.getoption("--skip-openai", False)diff --git a/test/oai/test_client.py b/test/oai/test_client.pyindex 4ac9374c3..749c272ea 100644--- a/test/oai/test_client.py+++ b/test/oai/test_client.py@@ -1,18 +1,19 @@import pytestfrom autogen import OpenAIWrapper, config_list_from_json, config_list_openai_aoai+from conftest import skip_openaiTOOL_ENABLED = Falsetry:from openai import OpenAI- from openai.types.chat.chat_completion import ChatCompletionMessage-except ImportError:- skip = True-else:- skip = Falseimport openaiif openai.__version__ >= "1.1.0":TOOL_ENABLED = True+ from openai.types.chat.chat_completion import ChatCompletionMessage+except ImportError:+ skip = True+else:+ skip = False or skip_openaiKEY_LOC = "notebook"OAI_CONFIG_LIST = "OAI_CONFIG_LIST"@@ -35,7 +36,7 @@ def test_aoai_chat_completion():print(client.extract_text_or_completion_object(response))[email protected](skip and not TOOL_ENABLED, reason="openai>=1.1.0 not installed")[email protected](skip or not TOOL_ENABLED, reason="openai>=1.1.0 not installed")def test_oai_tool_calling_extraction():config_list = config_list_from_json(env_or_file=OAI_CONFIG_LIST,diff --git a/test/oai/test_client_stream.py b/test/oai/test_client_stream.pyindex 613770964..c0ff76071 100644--- a/test/oai/test_client_stream.py+++ b/test/oai/test_client_stream.py@@ -1,12 +1,13 @@import pytestfrom autogen import OpenAIWrapper, config_list_from_json, config_list_openai_aoai+from conftest import skip_openaitry:from openai import OpenAIexcept ImportError:skip = Trueelse:- skip = False+ skip = False or skip_openaiKEY_LOC = "notebook"OAI_CONFIG_LIST = "OAI_CONFIG_LIST"diff --git a/test/test_notebook.py b/test/test_notebook.pyindex fc10f4afd..3abdbc230 100644--- a/test/test_notebook.py+++ b/test/test_notebook.py@@ -1,13 +1,14 @@import sysimport osimport pytest+from conftest import skip_openaitry:import openai-- skip = Falseexcept ImportError:skip = True+else:+ skip = False or skip_openaihere = os.path.abspath(os.path.dirname(__file__))diff --git a/test/test_retrieve_utils.py b/test/test_retrieve_utils.pyindex 013228e47..ba7485f5f 100644--- a/test/test_retrieve_utils.py+++ b/test/test_retrieve_utils.py@@ -1,6 +1,8 @@"""Unit test for retrieve_utils.py"""+import pytest+try:import chromadbfrom autogen.retrieve_utils import (@@ -18,8 +20,6 @@ except ImportError:else:skip = Falseimport os-import sys-import pytesttry:from unstructured.partition.auto import partitiondiff --git a/website/docs/Contribute.md b/website/docs/Contribute.mdindex 4ed0ed755..3b13d2f8e 100644--- a/website/docs/Contribute.md+++ b/website/docs/Contribute.md@@ -120,22 +120,20 @@ Tests are automatically run via GitHub actions. There are two workflows:1. [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml)1. [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml)-The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in one python version only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml).+The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml).#### Run non-OpenAI testsTo run the subset of the tests not depending on `openai` (and not calling LLMs)):-- Install `pytest`-- Remove `openai` library, this is required to skip the tests which check for `openai` presence ([`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19))-- Run the tests from the `test` folder. Make sure they all pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass-- Reinstall `openai`-+- Install pytest:```pip install pytest-pip uninstall -y openai-pytest test-pip install openai```+- Run the tests from the `test` folder using the `--skip-openai` flag.+```+pytest test --skip-openai+```+- Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass### Coverage