Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
Creates testing subpackage
Stefan Krawczyk7623005c9935
diff --git a/burr/cli/__main__.py b/burr/cli/__main__.pyindex 1759e093..5a148c7b 100644--- a/burr/cli/__main__.py+++ b/burr/cli/__main__.py@@ -200,7 +200,9 @@ def test_case():PYTEST_TEMPLATE = """import pytest-# TODO: import the action you're testing+from burr.testing import pytest_generate_tests # noqa: F401+# TODO: import action you're testing, i.e. import {ACTION_NAME}.+@pytest.mark.file_name("{FILE_NAME}")def test_{ACTION_NAME}(input_state, expected_state):\"\"\"Function for testing the action\"\"\"@@ -230,7 +232,7 @@ def create_test_case():"""Create a test case from a persisted state.- Prints a pytest test case to the console assuming you have a `pytest_generate_tests` function set up.+ Prints a pytest test case to the console.See examples/test-case-creation/test_application.py for details."""@@ -266,9 +268,7 @@ def create_test_case(else:logger.info(json.dumps(tc_json, indent=2))# print out python test to add- print(- "\nAdd the following to your test file assuming you've got `pytest_generate_tests` set up:\n"- )+ print("\nAdd the following to your test file:\n")print(PYTEST_TEMPLATE.format(FILE_NAME=target_file_name, ACTION_NAME=data["position"]))diff --git a/burr/testing/__init__.py b/burr/testing/__init__.pynew file mode 100644index 00000000..8ec0854b--- /dev/null+++ b/burr/testing/__init__.py@@ -0,0 +1,50 @@+"""+Module to help with testing.+"""++import json+++# one way to parameterize tests is to store the serialized state in a json file+# and then load it for the test.+def load_test_cases(file_name: str) -> tuple:+ """Load test cases from a json file."""+ with open(file_name, "r") as f:+ json_test_cases = json.load(f)+ test_cases = [(tc.get("input_state"), tc.get("expected_state")) for tc in json_test_cases]+ test_ids = [+ f"{tc.get('action', 'ACTION_MISSING')}-{tc.get('name', 'NAME_MISSING')}"+ for tc in json_test_cases+ ]+ return test_cases, test_ids+++# Define the pytest_generate_tests hook to generate test cases dynamically based on the+# contents of a file+def pytest_generate_tests(metafunc):+ """This function dynamically generates test cases based on the contents of a file.++ Use this with PyTest.+ """++ # for each function that has the 'input_state' and 'expected_state' arguments:+ if "input_state" in metafunc.fixturenames and "expected_state" in metafunc.fixturenames:+ # gets all filenames from the 'file_name' marker+ file_names = [+ file_arg+ for m in metafunc.definition.own_markers+ if m.name == "file_name"+ for file_arg in m.args+ ]+ all_test_cases = []+ all_test_ids = []+ for file_name in file_names:+ test_cases, test_case_ids = load_test_cases(file_name)+ all_test_cases.extend(test_cases)+ all_test_ids.extend(test_case_ids)+ if not all_test_cases:+ raise ValueError(+ f"No test cases could be created for test {metafunc.definition.originalname}."+ )+ # Generate test cases based on the test_data list+ metafunc.parametrize("input_state,expected_state", all_test_cases, ids=all_test_ids)diff --git a/examples/test-case-creation/notebook.ipynb b/examples/test-case-creation/notebook.ipynbindex 9416eb62..6bdaac53 100644--- a/examples/test-case-creation/notebook.ipynb+++ b/examples/test-case-creation/notebook.ipynb@@ -29,7 +29,7 @@],"source": ["%%sh \n",- "burr-test-case create --help\n"+ "burr-test-case create --help"],"metadata": {"collapsed": false,@@ -64,10 +64,12 @@"\n","Writing data to file /tmp/test-case.json\n","\n",- "Add the following to your test file assuming you've got `pytest_generate_tests` set up:\n",+ "Add the following to your test file:\n","\n","import pytest\n",- "# TODO: import the action you're testing\n",+ "from burr.testing import pytest_generate_tests # noqa: F401\n",+ "# TODO: import action you're testing, i.e. import plan_cot.\n",+ "\n","@pytest.mark.file_name(\"/tmp/test-case.json\")\n","def test_plan_cot(input_state, expected_state):\n"," \"\"\"Function for testing the action\"\"\"\n",@@ -80,17 +82,17 @@],"source": ["%%sh\n",- "burr-test-case create --project-name \"oa:determine_params\" --partition-key \"\" --app-id \"3350dd68-bcfd-48a1-85c5-dd72186382ef\" --sequence-id 0 --target-file-name /tmp/test-case.json\n"+ "burr-test-case create --project-name \"oa:determine_params\" --partition-key \"\" --app-id \"3350dd68-bcfd-48a1-85c5-dd72186382ef\" --sequence-id 0 --target-file-name /tmp/test-case.json"],"metadata": {"collapsed": false,"ExecuteTime": {- "end_time": "2024-04-01T05:33:01.015032Z",- "start_time": "2024-04-01T05:33:00.146352Z"+ "end_time": "2024-04-01T06:16:19.392833Z",+ "start_time": "2024-04-01T06:16:18.000721Z"}},"id": "586bcb0bdb5dde8b",- "execution_count": 11+ "execution_count": 12},{"cell_type": "markdown",diff --git a/examples/test-case-creation/test_application.py b/examples/test-case-creation/test_application.pyindex 98c32d39..d875e333 100644--- a/examples/test-case-creation/test_application.py+++ b/examples/test-case-creation/test_application.py@@ -3,14 +3,16 @@ This is an example test module showing how you can test an action in your applic1. There is a very straightforward unit testing style approach.2. There is a more complex approach that uses pytest_generate_tests to dynamically generate test cases.+To use this approach you need to do `from burr.testing import pytest_generate_tests # noqa: F401 `"""-import json-import pytestfrom application import prompt_for_morefrom burr.core import state+# the following is required to run file based tests+from burr.testing import pytest_generate_tests # noqa: F401+def test_prompt_for_more():"""A basic test showing how to 'unit' test an action."""@@ -45,48 +47,6 @@ def test_prompt_for_more():}-# one way to parameterize tests is to store the serialized state in a json file-# and then load it for the test.-def load_test_cases(file_name: str) -> tuple:- """Load test cases from a json file."""- with open(file_name, "r") as f:- json_test_cases = json.load(f)- test_cases = [(tc.get("input_state"), tc.get("expected_state")) for tc in json_test_cases]- test_ids = [- f"{tc.get('action', 'ACTION_MISSING')}-{tc.get('name', 'NAME_MISSING')}"- for tc in json_test_cases- ]- return test_cases, test_ids---# Define the pytest_generate_tests hook to generate test cases dynamically based on the-# contents of a file-def pytest_generate_tests(metafunc):- """This function dynamically generates test cases based on the contents of a file."""-- # for each function that has the 'input_state' and 'expected_state' arguments:- if "input_state" in metafunc.fixturenames and "expected_state" in metafunc.fixturenames:- # gets all filenames from the 'file_name' marker- file_names = [- file_arg- for m in metafunc.definition.own_markers- if m.name == "file_name"- for file_arg in m.args- ]- all_test_cases = []- all_test_ids = []- for file_name in file_names:- test_cases, test_case_ids = load_test_cases(file_name)- all_test_cases.extend(test_cases)- all_test_ids.extend(test_case_ids)- if not all_test_cases:- raise ValueError(- f"No test cases could be created for test {metafunc.definition.originalname}."- )- # Generate test cases based on the test_data list- metafunc.parametrize("input_state,expected_state", all_test_cases, ids=all_test_ids)--@pytest.mark.file_name("prompt_for_more.json")def test_prompt_for_more_from_file(input_state, expected_state):"""Function for testing the action"""