A programming framework for agentic AI
Updated agbench to find agnext via environment variable to facilitate running from magentic. (#348)
afourney2e7d984b050a
diff --git a/python/tools/agbench/src/agbench/run_cmd.py b/python/tools/agbench/src/agbench/run_cmd.pyindex f44e2170b..374b2c1d4 100644--- a/python/tools/agbench/src/agbench/run_cmd.py+++ b/python/tools/agbench/src/agbench/run_cmd.py@@ -61,7 +61,7 @@ def run_scenarios(subsample: Union[None, int, float] = None,) -> None:"""- Run a set autogenbench scenarios a given number of times.+ Run a set agbench scenarios a given number of times.Args:scenario (path): The file or folder containing the scenario JSONL instances. If given a folder, then@@ -306,13 +306,13 @@ def run_scenario_natively(work_dir: str, env: Mapping[str, str], timeout: int =f.write(f"""#echo RUN.SH STARTING !#!#-export AUTOGEN_TESTBED_SETTING="Native"-echo "autogenbench version: {__version__}" > timestamp.txt+export AGNEXT_TESTBED_SETTING="Native"+echo "agbench version: {__version__}" > timestamp.txt# Create and activate the virtual environment# This is called in a subprocess, and will not impact the parent-{sys.executable} -m venv .autogenbench_venv-. .autogenbench_venv/bin/activate+{sys.executable} -m venv .agbench_venv+. .agbench_venv/bin/activate# Run the global init script if it existsif [ -f global_init.sh ] ; then@@ -356,8 +356,8 @@ fi# We don't need to deactivate the venv because it's# contained in the subprocess; but we should clean it up-if [ -d .autogenbench_venv ] ; then- rm -Rf .autogenbench_venv+if [ -d .agbench_venv ] ; then+ rm -Rf .agbench_venvfiecho RUN.SH COMPLETE !#!#@@ -426,10 +426,10 @@ def run_scenario_in_docker(f.write(f"""#echo RUN.SH STARTING !#!#-export AUTOGEN_TESTBED_SETTING="Docker"+export AGNEXT_TESTBED_SETTING="Docker"umask 000-echo "autogenbench version: {__version__}" > timestamp.txt+echo "agbench version: {__version__}" > timestamp.txt# Run the global init script if it existsif [ -f global_init.sh ] ; then@@ -478,15 +478,20 @@ echo RUN.SH COMPLETE !#!## Figure out what folders to mountvolumes = {str(pathlib.Path(work_dir).absolute()): {"bind": "/workspace", "mode": "rw"}}- # Add the autogen repo if we can find it- autogen_repo_base = os.environ.get("AUTOGENBENCH_REPO_BASE")- if autogen_repo_base is None:- autogen_repo_base = find_autogen_repo(os.getcwd())- elif not os.path.isdir(autogen_repo_base):- raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), autogen_repo_base)+ # Add the agnext repo if we can find it+ agnext_repo_base = os.environ.get("AGNEXT_REPO_BASE")+ if agnext_repo_base is None:+ agnext_repo_base = find_agnext_repo(os.getcwd())+ elif not os.path.isdir(agnext_repo_base):+ raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), agnext_repo_base)- if autogen_repo_base is not None:- volumes[str(pathlib.Path(autogen_repo_base).absolute())] = {"bind": "/agnext", "mode": "rw"}+ if agnext_repo_base is None:+ raise ValueError(+ "Could not find AGNext repo base. Please set the environment variable AGNEXT_REPO_BASE to the correct value."+ )++ agnext_repo_base = os.path.join(agnext_repo_base, "python")+ volumes[str(pathlib.Path(agnext_repo_base).absolute())] = {"bind": "/agnext", "mode": "rw"}print("Mounting:")for k in volumes:@@ -582,11 +587,11 @@ def build_default_docker_image(docker_client: docker.DockerClient, image_tag: stsys.stdout.write(segment["stream"])-def find_autogen_repo(path: str) -> Optional[str]:+def find_agnext_repo(path: str) -> Optional[str]:"""- Utility for identifying if the path is a subdirectory of the autogen repo.+ Utility for identifying if the path is a subdirectory of the agnext repo.- Returns: the path to the root of the autogen repo if one is found, otherwise None+ Returns: the path to the root of the agnext repo if one is found, otherwise None"""# Normalize the path (we expect a directory)@@ -597,7 +602,7 @@ def find_autogen_repo(path: str) -> Optional[str]:while True:test_path = os.path.join(path, "python", "src", "agnext") # We found agnextif os.path.isdir(test_path):- return os.path.join(path, "python")+ return path# Stop if we hit the rootparent_dir = os.path.abspath(os.path.join(path, os.pardir))@@ -672,7 +677,7 @@ def run_cli(args: Sequence[str]) -> None:)# Does an environment variable override the prompt?- allow_native = os.environ.get("AUTOGENBENCH_ALLOW_NATIVE")+ allow_native = os.environ.get("AGBENCH_ALLOW_NATIVE")if allow_native is None or allow_native == "":choice = input('Are you absolutely sure you want to continue with native execution? Type "Yes" exactly, and in full, to proceed: '@@ -680,9 +685,9 @@ def run_cli(args: Sequence[str]) -> None:if choice.strip().lower() != "yes":sys.exit("Received '" + choice + "'. Exiting.")elif allow_native.strip().lower() != "yes":- sys.exit(f"Exiting because AUTOGENBENCH_ALLOW_NATIVE is '{allow_native}'\n")+ sys.exit(f"Exiting because AGBENCH_ALLOW_NATIVE is '{allow_native}'\n")else:- sys.stderr.write(f"Continuing because AUTOGENBENCH_ALLOW_NATIVE is '{allow_native}'\n")+ sys.stderr.write(f"Continuing because AGBENCH_ALLOW_NATIVE is '{allow_native}'\n")time.sleep(0.75) # Pause very briefly so the message isn't lost in the noise# Parse the subsample