Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

ci(backend): shard tests with fail-closed reports (#14279)

Nicholas Tindlebb4b2e488abb
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.json
new file mode 100644
index 0000000000..43f6f4bdca
--- /dev/null
+++ b/.github/scripts/backend-allowed-skips.json
@@ -0,0 +1,138 @@
+[
+ "backend.api.features.orgs.regression_test.TestPR11RealtimeTenancy.test_subscribe_to_session_validates_org_membership",
+ "backend.api.features.orgs.regression_test.TestPR15MarketplaceOrg.test_slug_unique_per_org",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_agent_graphs_have_organization_id",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_executions_have_organization_id",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_all_store_listings_have_owning_org_id",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_get_execution_requires_org_membership",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_get_graph_requires_org_membership",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_api_keys_scoped_by_org_cutover",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_executions_scoped_by_team",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_graphs_excludes_other_org_graphs",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_list_graphs_returns_only_org_graphs",
+ "backend.api.features.orgs.regression_test.TestPR18Cutover.test_read_by_user_id_fallback_removed",
+ "backend.blocks.airtable._api_test.test_create_and_update_field",
+ "backend.blocks.airtable._api_test.test_create_update_table",
+ "backend.blocks.airtable._api_test.test_invalid_field_type",
+ "backend.blocks.airtable._api_test.test_record_management",
+ "backend.blocks.airtable._api_test.test_webhook_management",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_call_tool_list_api_endpoints",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_call_tool_search",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_initialize",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_list_tools",
+ "backend.blocks.mcp.test_e2e.TestRealMCPServer.test_sse_response_handling",
+ "backend.blocks.test.test_block.test_available_blocks[HumanInTheLoopBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[CountdownTimerBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[ExecuteCodeBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[GetWeatherInformationBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[GithubListStargazersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[Slant3DSlicerBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCloseTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistCreateTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteCommentBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteSectionBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistDeleteTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetCommentBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetCommentsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetSectionBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetSharedLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistGetTasksBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistListCollaboratorsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistListLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistListSectionsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistRemoveSharedLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistRenameSharedLabelsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistReopenTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateCommentBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateLabelBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateProjectBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TodoistUpdateTaskBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterAddListMemberBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterBookmarkTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterCreateListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterDeleteListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterDeleteTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterFollowListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterFollowUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetBlockedUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetBookmarkedTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetFollowersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetFollowingBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetHomeTimelineBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetLikedTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetLikingUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListMembersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListMembershipsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetListTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetMutedUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetOwnedListsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetPinnedListsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetQuoteTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetRetweetersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceBuyersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceByIdBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpacesBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetSpaceTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserMentionsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUsersBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterGetUserTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterHideReplyBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterLikeTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterMuteUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterPinListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterPostTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveBookmarkTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveListMemberBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRemoveRetweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterRetweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterSearchRecentTweetsBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterSearchSpacesBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnfollowListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnfollowUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnhideReplyBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnlikeTweetBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnmuteUserBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUnpinListBlock]",
+ "backend.blocks.test.test_block.test_block_ids_valid[TwitterUpdateListBlock]",
+ "backend.copilot.baseline.service_test.test_baseline_multi_turn",
+ "backend.copilot.graphiti.client_test.TestHyphenInGroupIdRegression.test_build_fulltext_query_escapes_hyphens_in_group_id",
+ "backend.copilot.sdk.cli_openrouter_compat_test.test_bare_cli_does_not_send_openrouter_incompatible_features",
+ "backend.copilot.sdk.tool_adapter_test.TestBug1DuplicateExecution.test_old_code_duplicates_on_arg_mismatch",
+ "backend.copilot.sdk.tool_adapter_test.TestBug2FIFODesync.test_old_code_fifo_desync_on_denial",
+ "backend.copilot.sdk.tool_adapter_test.TestBug3CancelRace.test_old_code_cancel_arrives_too_late",
+ "backend.copilot.service_test.test_sdk_resume_multi_turn",
+ "backend.copilot.test_copilot_e2e.test_session_state_persistence",
+ "backend.copilot.tools.agent_browser_integration_test.test_chromium_executable_env_is_set",
+ "backend.copilot.tools.agent_browser_integration_test.test_close_session",
+ "backend.copilot.tools.agent_browser_integration_test.test_concurrent_independent_sessions",
+ "backend.copilot.tools.agent_browser_integration_test.test_fill_form_field",
+ "backend.copilot.tools.agent_browser_integration_test.test_get_title_after_navigate",
+ "backend.copilot.tools.agent_browser_integration_test.test_get_url_after_navigate",
+ "backend.copilot.tools.agent_browser_integration_test.test_navigate_returns_success",
+ "backend.copilot.tools.agent_browser_integration_test.test_screenshot_produces_valid_png",
+ "backend.copilot.tools.agent_browser_integration_test.test_scroll_down",
+ "backend.copilot.tools.agent_browser_integration_test.test_snapshot_returns_interactive_elements",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_fill_and_click",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_missing_action",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_missing_target",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_act_scroll",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://[::1]/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://0.0.0.0/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://10.0.0.1/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://127.0.0.1/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_blocked_url[http://169.254.169.254/]",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_missing_url",
+ "backend.copilot.tools.agent_browser_integration_test.test_tool_navigate_returns_response",
+ "test.test_data_creator.test_main_function_runs_without_errors"
+]
diff --git a/.github/scripts/backend_test_shard.py b/.github/scripts/backend_test_shard.py
new file mode 100644
index 0000000000..34890d4eb5
--- /dev/null
+++ b/.github/scripts/backend_test_shard.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import sys
+
+
+SHARDS = {
+ "data": ["backend/data"],
+ "copilot": ["backend/copilot"],
+ "util-executor": ["backend/util", "backend/executor"],
+}
+
+
+def shard_args(shard: str) -> list[str]:
+ if shard in SHARDS:
+ return ["-c", "pyproject.toml", *SHARDS[shard]]
+ if shard != "remainder":
+ raise ValueError(f"Unknown backend test shard: {shard}")
+ return [
+ "-c",
+ "pyproject.toml",
+ ".",
+ "../../.github/scripts",
+ *(f"--ignore={path}" for paths in SHARDS.values() for path in paths),
+ ]
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("shard", choices=[*SHARDS, "remainder"])
+ parser.add_argument("pytest_args", nargs=argparse.REMAINDER)
+ args = parser.parse_args()
+ pytest_args = args.pytest_args
+ if pytest_args[:1] == ["--"]:
+ pytest_args = pytest_args[1:]
+ if args.shard == "data":
+ os.environ["E2E_REDIS_CLUSTER_RESTART"] = "1"
+ return subprocess.run(
+ ["poetry", "run", "pytest", *shard_args(args.shard), *pytest_args],
+ check=False,
+ ).returncode
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/docker-pull-with-retry.sh b/.github/scripts/docker-pull-with-retry.sh
new file mode 100644
index 0000000000..791c0a30a7
--- /dev/null
+++ b/.github/scripts/docker-pull-with-retry.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ "$#" -ne 1 || -z "$1" ]]; then
+ echo "usage: $0 IMAGE" >&2
+ exit 2
+fi
+
+readonly image="$1"
+readonly max_attempts=3
+
+for ((attempt = 1; attempt <= max_attempts; attempt++)); do
+ echo "Pulling ${image} (attempt ${attempt}/${max_attempts})"
+ if docker pull "$image"; then
+ exit 0
+ fi
+
+ if [[ "$attempt" -eq "$max_attempts" ]]; then
+ echo "Failed to pull ${image} after ${max_attempts} attempts" >&2
+ exit 1
+ fi
+
+ delay_seconds=$((attempt * 2))
+ echo "Retrying ${image} in ${delay_seconds}s" >&2
+ sleep "$delay_seconds"
+done
+
+exit 1
diff --git a/.github/scripts/test_backend_test_shard.py b/.github/scripts/test_backend_test_shard.py
new file mode 100644
index 0000000000..cf65340af1
--- /dev/null
+++ b/.github/scripts/test_backend_test_shard.py
@@ -0,0 +1,103 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+from xml.etree import ElementTree
+
+import backend_test_shard
+from backend_test_shard import SHARDS, shard_args
+
+
+class BackendShardTests(unittest.TestCase):
+ def test_all_test_locations_are_assigned_once_with_backend_config(self):
+ paths = [
+ "backend/data/sample_unit_test.py",
+ "backend/copilot/sample_unit_test.py",
+ "backend/util/sample_unit_test.py",
+ "backend/executor/sample_unit_test.py",
+ "backend/blocks/sample_unit_test.py",
+ "backend/new_folder/sample_unit_test.py",
+ "backend/new_unit_test.py",
+ "new_top_level/sample_unit_test.py",
+ "root_unit_test.py",
+ ]
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ backend = root / "autogpt_platform/backend"
+ backend.mkdir(parents=True)
+ (backend / "pyproject.toml").write_text(
+ '[tool.pytest.ini_options]\npython_files = ["*_unit_test.py"]\n'
+ )
+ helpers = root / ".github/scripts"
+ helpers.mkdir(parents=True)
+ (helpers / "helper_unit_test.py").write_text("def test_helper(): pass\n")
+ for index, path in enumerate(paths):
+ target = backend / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(f"def test_case_{index}(): pass\n")
+ names = []
+ for shard in ("data", "copilot", "util-executor", "remainder"):
+ report = root / f"{shard}.xml"
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "pytest",
+ "--import-mode=importlib",
+ *shard_args(shard),
+ f"--junitxml={report}",
+ "-q",
+ ],
+ cwd=backend,
+ env={**os.environ, "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"},
+ capture_output=True,
+ text=True,
+ )
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ names.extend(
+ case.get("name")
+ for case in ElementTree.parse(report).iter("testcase")
+ )
+ self.assertCountEqual(
+ names, [f"test_case_{i}" for i in range(len(paths))] + ["test_helper"]
+ )
+
+ def test_unknown_shard_is_rejected(self):
+ with self.assertRaises(ValueError):
+ shard_args("typo")
+
+ def test_workflow_matrix_matches_script_shards(self):
+ workflow = (
+ Path(__file__).resolve().parents[1] / "workflows/platform-backend-ci.yml"
+ )
+ line = next(
+ line
+ for line in workflow.read_text().splitlines()
+ if line.strip().startswith("test-shard: [")
+ )
+ matrix = [
+ shard.strip() for shard in line.split("[", 1)[1].rstrip("] ").split(",")
+ ]
+ self.assertCountEqual(matrix, [*SHARDS, "remainder"])
+
+ def test_main_propagates_pytest_status_and_scopes_restart_test_to_data(self):
+ for shard, restart in (("data", "1"), ("copilot", None)):
+ with (
+ self.subTest(shard=shard),
+ patch.object(
+ backend_test_shard.subprocess,
+ "run",
+ return_value=subprocess.CompletedProcess([], 3),
+ ) as run,
+ patch.dict(os.environ),
+ patch.object(sys, "argv", ["backend_test_shard.py", shard, "--", "-q"]),
+ ):
+ os.environ.pop("E2E_REDIS_CLUSTER_RESTART", None)
+ self.assertEqual(backend_test_shard.main(), 3)
+ self.assertEqual(os.environ.get("E2E_REDIS_CLUSTER_RESTART"), restart)
+ run.assert_called_once_with(
+ ["poetry", "run", "pytest", *shard_args(shard), "-q"], check=False
+ )
diff --git a/.github/scripts/test_docker_pull_retry.py b/.github/scripts/test_docker_pull_retry.py
new file mode 100644
index 0000000000..d183a58188
--- /dev/null
+++ b/.github/scripts/test_docker_pull_retry.py
@@ -0,0 +1,104 @@
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).with_name("docker-pull-with-retry.sh")
+BASH = (
+ str(Path("C:/Program Files/Git/bin/bash.exe"))
+ if sys.platform == "win32"
+ else shutil.which("bash")
+)
+
+
+class DockerPullRetryTests(unittest.TestCase):
+ def run_pull(self, failures, args):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ docker = root / "docker"
+ docker.write_text(
+ "#!/usr/bin/env bash\n"
+ '[[ "$#" == 2 && "$1" == pull ]] || exit 90\n'
+ 'count=0; [[ ! -f "$COUNTER" ]] || read -r count < "$COUNTER"\n'
+ 'count=$((count + 1)); echo "$count" > "$COUNTER"\n'
+ 'printf "%s" "$2" > "$ARGUMENT"\n'
+ "((count > FAILURES))\n",
+ newline="\n",
+ )
+ sleep = root / "sleep"
+ sleep.write_text(
+ '#!/usr/bin/env bash\necho "$1" >> "$DELAYS"\n', newline="\n"
+ )
+ script = root / "pull.sh"
+ script.write_text(SCRIPT.read_text(), newline="\n")
+ docker.chmod(0o755)
+ sleep.chmod(0o755)
+ result = subprocess.run(
+ [
+ BASH,
+ "-c",
+ 'export PATH="$(cd "$STUB_BIN" && pwd):$PATH"; exec bash "$@"',
+ "bash",
+ str(script),
+ *args,
+ ],
+ env={
+ **os.environ,
+ "PATH": directory + os.pathsep + os.environ["PATH"],
+ "STUB_BIN": directory,
+ "COUNTER": str(root / "counter"),
+ "ARGUMENT": str(root / "argument"),
+ "DELAYS": str(root / "delays"),
+ "FAILURES": str(failures),
+ },
+ capture_output=True,
+ text=True,
+ )
+ count = (
+ int((root / "counter").read_text())
+ if (root / "counter").exists()
+ else 0
+ )
+ delays = (
+ (root / "delays").read_text().splitlines()
+ if (root / "delays").exists()
+ else []
+ )
+ argument = (
+ (root / "argument").read_text()
+ if (root / "argument").exists()
+ else None
+ )
+ return result, count, delays, argument
+
+ def test_success_after_zero_one_or_two_failures(self):
+ for failures in (0, 1, 2):
+ with self.subTest(failures=failures):
+ result, count, delays, argument = self.run_pull(failures, ["image:tag"])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(count, failures + 1)
+ self.assertEqual(delays, ["2", "4"][:failures])
+ self.assertEqual(argument, "image:tag")
+
+ def test_terminal_failure_is_not_reported_as_success(self):
+ result, count, delays, _ = self.run_pull(3, ["image:tag"])
+ self.assertEqual(result.returncode, 1, result.stderr)
+ self.assertEqual(count, 3)
+ self.assertEqual(delays, ["2", "4"])
+
+ def test_invalid_arguments_never_call_docker(self):
+ for args in ([], [""], ["one", "two"]):
+ with self.subTest(args=args):
+ result, count, _, _ = self.run_pull(0, args)
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(count, 0)
+
+ def test_image_remains_one_literal_argument(self):
+ image = "image with spaces; no-command"
+ result, _, _, argument = self.run_pull(0, [image])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(argument, image)
diff --git a/.github/scripts/test_validate_junit.py b/.github/scripts/test_validate_junit.py
new file mode 100644
index 0000000000..dc66e2daf2
--- /dev/null
+++ b/.github/scripts/test_validate_junit.py
@@ -0,0 +1,212 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+from xml.etree import ElementTree
+
+from validate_junit import main, summarize_junit
+
+
+def report_xml(
+ *, failures: int = 0, errors: int = 0, skipped: int = 0
+) -> ElementTree.Element:
+ tests = 1 + failures + errors + skipped
+ root = ElementTree.Element(
+ "testsuites",
+ tests=str(tests),
+ failures=str(failures),
+ errors=str(errors),
+ skipped=str(skipped),
+ )
+ suite = ElementTree.SubElement(root, "testsuite", name="suite")
+ ElementTree.SubElement(suite, "testcase", name="passes")
+ for index in range(failures):
+ case = ElementTree.SubElement(suite, "testcase", name=f"failure-{index}")
+ ElementTree.SubElement(case, "failure")
+ for index in range(errors):
+ case = ElementTree.SubElement(suite, "testcase", name=f"error-{index}")
+ ElementTree.SubElement(case, "error")
+ for index in range(skipped):
+ case = ElementTree.SubElement(suite, "testcase", name=f"skipped-{index}")
+ ElementTree.SubElement(case, "skipped")
+ return root
+
+
+class ValidateJUnitTests(unittest.TestCase):
+ def test_failed_synthetic_write_returns_failure_without_losing_cause(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "missing.xml"
+ with (
+ patch(
+ "validate_junit.write_synthetic_error",
+ side_effect=OSError("read only"),
+ ),
+ patch("sys.stderr") as stderr,
+ ):
+ self.assertEqual(main(["--synthesize-invalid", str(report)]), 1)
+ output = "".join(call.args[0] for call in stderr.write.call_args_list)
+ self.assertIn("report is missing or empty", output)
+ self.assertIn("could not write synthetic error report: read only", output)
+
+ def test_skips_without_classname_cannot_match_fully_qualified_allowlist(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "skipped.xml"
+ allowlist = Path(directory) / "allowed.json"
+ ElementTree.ElementTree(report_xml(skipped=1)).write(report)
+ allowlist.write_text(json.dumps(["backend.example.skipped-0"]))
+ self.assertEqual(
+ main(["--allow-skips-from", str(allowlist), str(report)]), 1
+ )
+
+ def test_accepts_passing_report_with_accounted_skips(self):
+ summary = summarize_junit(report_xml(skipped=2))
+
+ self.assertEqual(summary.tests, 3)
+ self.assertEqual(summary.passed, 1)
+ self.assertEqual(summary.skipped, 2)
+
+ def test_rejects_failure_and_error_cases(self):
+ for root in (report_xml(failures=1), report_xml(errors=1)):
+ with (
+ self.subTest(root=root.attrib),
+ self.assertRaisesRegex(ValueError, "failures and .* errors"),
+ ):
+ summarize_junit(root)
+
+ def test_rejects_declared_count_mismatch(self):
+ root = report_xml()
+ root.set("failures", "1")
+
+ with self.assertRaisesRegex(ValueError, "root failures count"):
+ summarize_junit(root)
+
+ def test_rejects_nested_suite_count_mismatch_without_root_counts(self):
+ root = report_xml()
+ root.attrib.clear()
+ suite = next(root.iter("testsuite"))
+ suite.set("tests", "2")
+
+ with self.assertRaisesRegex(ValueError, "testsuite 'suite' tests count"):
+ summarize_junit(root)
+
+ def test_invalid_nested_counts_identify_the_affected_suite(self):
+ for key in ("tests", "failures", "errors", "skipped"):
+ for value, reason in (("invalid", "not an integer"), ("-1", "negative")):
+ with self.subTest(key=key, value=value):
+ root = report_xml()
+ suite = next(root.iter("testsuite"))
+ suite.set(key, value)
+ with self.assertRaisesRegex(
+ ValueError, f"testsuite 'suite' {key} count is {reason}"
+ ):
+ summarize_junit(root)
+
+ def test_accepts_skipped_only_report(self):
+ root = report_xml(skipped=1)
+ passing_case = next(root.iter("testcase"))
+ passing_case.append(ElementTree.Element("skipped"))
+ root.set("skipped", "2")
+
+ summary = summarize_junit(root)
+
+ self.assertEqual(summary.tests, 2)
+ self.assertEqual(summary.passed, 0)
+
+ def test_rejects_report_with_no_test_cases(self):
+ with self.assertRaisesRegex(ValueError, "no test cases"):
+ summarize_junit(
+ ElementTree.fromstring(
+ '<testsuites tests="0"><testsuite name="s" tests="0"/></testsuites>'
+ )
+ )
+
+ def test_require_no_skips_rejects_a_skipped_case(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "skipped.xml"
+ ElementTree.ElementTree(report_xml(skipped=1)).write(report)
+
+ self.assertEqual(main([str(report)]), 0)
+ self.assertEqual(main(["--require-no-skips", str(report)]), 1)
+
+ def test_skip_allowlist_rejects_new_skips_and_accepts_existing_ids(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "skipped.xml"
+ allowlist = Path(directory) / "allowed.json"
+ root = report_xml(skipped=1)
+ for case in root.iter("testcase"):
+ case.set("classname", "backend.example")
+ ElementTree.ElementTree(root).write(report)
+ allowlist.write_text(json.dumps(["backend.example.skipped-0"]))
+ args = ["--allow-skips-from", str(allowlist), str(report)]
+ self.assertEqual(main(args), 0)
+ allowlist.write_text(json.dumps(["backend.example.other"]))
+ with patch("sys.stderr") as stderr:
+ self.assertEqual(main(args), 1)
+ output = "".join(call.args[0] for call in stderr.write.call_args_list)
+ self.assertIn(str(allowlist), output)
+ self.assertIn("do not allowlist a regression", output)
+
+ def test_skip_allowlist_must_be_a_valid_list_of_exact_ids(self):
+ self._check_invalid_allowlists()
+
+ def test_allowlisted_all_skipped_report_still_fails(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "skipped.xml"
+ allowlist = Path(directory) / "allowed.json"
+ report.write_text(
+ '<testsuite><testcase classname="backend.example" name="only"><skipped/></testcase></testsuite>'
+ )
+ allowlist.write_text(json.dumps(["backend.example.only"]))
+ self.assertEqual(
+ main(["--allow-skips-from", str(allowlist), str(report)]), 1
+ )
+
+ def _check_invalid_allowlists(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "passed.xml"
+ allowlist = Path(directory) / "allowed.json"
+ ElementTree.ElementTree(report_xml()).write(report)
+ args = ["--allow-skips-from", str(allowlist), str(report)]
+ for content in (
+ "{",
+ "{}",
+ "[1]",
+ '[""]',
+ '["a", "a"]',
+ '[".skipped-0"]',
+ '["backend.example."]',
+ '["missing-separator"]',
+ ):
+ with self.subTest(content=content):
+ allowlist.write_text(content)
+ self.assertEqual(main(args), 1)
+ allowlist.unlink()
+ self.assertEqual(main(args), 1)
+
+ def test_synthesizes_machine_readable_error_for_missing_report(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "missing.xml"
+
+ status = main(["--synthesize-invalid", str(report)])
+
+ self.assertEqual(status, 1)
+ root = ElementTree.parse(report).getroot()
+ self.assertEqual(root.get("tests"), "1")
+ self.assertEqual(root.get("errors"), "1")
+ self.assertEqual(len(list(root.iter("error"))), 1)
+
+ def test_synthesizes_machine_readable_error_for_malformed_report(self):
+ with tempfile.TemporaryDirectory() as directory:
+ report = Path(directory) / "malformed.xml"
+ report.write_text("<testsuites>", encoding="utf-8")
+
+ status = main(["--synthesize-invalid", str(report)])
+
+ self.assertEqual(status, 1)
+ root = ElementTree.parse(report).getroot()
+ self.assertEqual(root.get("errors"), "1")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/validate_junit.py b/.github/scripts/validate_junit.py
new file mode 100644
index 0000000000..6971292161
--- /dev/null
+++ b/.github/scripts/validate_junit.py
@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from xml.etree import ElementTree
+
+
+@dataclass(frozen=True)
+class JUnitSummary:
+ tests: int
+ failures: int
+ errors: int
+ skipped: int
+
+ @property
+ def passed(self) -> int:
+ return self.tests - self.failures - self.errors - self.skipped
+
+
+def _declared_count(element: ElementTree.Element, key: str, label: str) -> int | None:
+ raw = element.get(key)
+ if raw is None:
+ return None
+ try:
+ value = int(raw)
+ except ValueError as exc:
+ raise ValueError(f"{label} {key} count is not an integer: {raw!r}") from exc
+ if value < 0:
+ raise ValueError(f"{label} {key} count is negative: {value}")
+ return value
+
+
+def _summarize_cases(element: ElementTree.Element) -> JUnitSummary:
+ cases = list(element.iter("testcase"))
+ return JUnitSummary(
+ tests=len(cases),
+ failures=sum(case.find("failure") is not None for case in cases),
+ errors=sum(case.find("error") is not None for case in cases),
+ skipped=sum(case.find("skipped") is not None for case in cases),
+ )
+
+
+def _validate_declared_counts(
+ element: ElementTree.Element, summary: JUnitSummary, label: str
+) -> None:
+ for key in ("tests", "failures", "errors", "skipped"):
+ declared = _declared_count(element, key, label)
+ actual = getattr(summary, key)
+ if declared is not None and declared != actual:
+ raise ValueError(
+ f"{label} {key} count is {declared}, but test cases contain {actual}"
+ )
+
+
+def summarize_junit(root: ElementTree.Element) -> JUnitSummary:
+ if root.tag not in {"testsuite", "testsuites"}:
+ raise ValueError(f"unexpected root element: {root.tag!r}")
+
+ summary = _summarize_cases(root)
+ _validate_declared_counts(root, summary, "root")
+ for suite in root.iter("testsuite"):
+ if suite is root:
+ continue
+ suite_summary = _summarize_cases(suite)
+ _validate_declared_counts(
+ suite, suite_summary, f"testsuite {suite.get('name', '<unnamed>')!r}"
+ )
+
+ if summary.tests == 0:
+ raise ValueError("report contains no test cases")
+ if summary.failures or summary.errors:
+ raise ValueError(
+ f"report contains {summary.failures} failures and {summary.errors} errors"
+ )
+ return summary
+
+
+def write_synthetic_error(path: Path, reason: str) -> None:
+ root = ElementTree.Element(
+ "testsuites", tests="1", failures="0", errors="1", skipped="0"
+ )
+ suite = ElementTree.SubElement(
+ root,
+ "testsuite",
+ name="ci-report-finalization",
+ tests="1",
+ failures="0",
+ errors="1",
+ skipped="0",
+ )
+ case = ElementTree.SubElement(
+ suite,
+ "testcase",
+ classname="ci.report",
+ name=f"{path.name} was not produced",
+ )
+ error = ElementTree.SubElement(case, "error", message=reason)
+ error.text = reason
+ ElementTree.indent(root)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ ElementTree.ElementTree(root).write(path, encoding="utf-8", xml_declaration=True)
+
+
+def validate_path(
+ path: Path, synthesize_invalid: bool, allowed_skips: set[str] | None = None
+) -> JUnitSummary:
+ try:
+ if not path.is_file() or path.stat().st_size == 0:
+ raise ValueError("report is missing or empty")
+ root = ElementTree.parse(path).getroot()
+ except (OSError, ElementTree.ParseError, ValueError) as exc:
+ if synthesize_invalid:
+ try:
+ write_synthetic_error(path, str(exc))
+ except OSError as write_exc:
+ raise ValueError(
+ f"{exc}; could not write synthetic error report: {write_exc}"
+ ) from write_exc
+ raise ValueError(str(exc)) from exc
+ summary = summarize_junit(root)
+ if allowed_skips is not None:
+ unexpected = [
+ f"{case.get('classname', '')}.{case.get('name', '')}"
+ for case in root.iter("testcase")
+ if case.find("skipped") is not None
+ and f"{case.get('classname', '')}.{case.get('name', '')}"
+ not in allowed_skips
+ ]
+ if unexpected:
+ raise ValueError(f"unapproved skipped test IDs: {', '.join(unexpected)}")
+ if summary.passed == 0:
+ raise ValueError("report contains no executed passing tests")
+ return summary
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--synthesize-invalid",
+ action="store_true",
+ help="replace missing, empty, or malformed reports with a one-error JUnit report",
+ )
+ skip_policy = parser.add_mutually_exclusive_group()
+ skip_policy.add_argument(
+ "--require-no-skips",
+ action="store_true",
+ help="reject reports containing skipped test cases",
+ )
+ skip_policy.add_argument(
+ "--allow-skips-from",
+ type=Path,
+ help="reject skips not listed as exact classname.name IDs in this JSON file",
+ )
+ parser.add_argument("reports", nargs="+", type=Path)
+ return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = parse_args(sys.argv[1:] if argv is None else argv)
+ problems: list[str] = []
+ allowed_skips = None
+ if args.allow_skips_from:
+ try:
+ entries = json.loads(args.allow_skips_from.read_text(encoding="utf-8"))
+ if (
+ not isinstance(entries, list)
+ or any(
+ not isinstance(entry, str)
+ or "." not in entry
+ or not entry.split(".", 1)[0].strip()
+ or not entry.rsplit(".", 1)[-1].strip()
+ for entry in entries
+ )
+ or len(set(entries)) != len(entries)
+ ):
+ raise ValueError(
+ "skip allowlist must contain unique nonempty classname.name IDs"
+ )
+ allowed_skips = set(entries)
+ except (OSError, ValueError) as exc:
+ print(f"Invalid skip allowlist: {exc}", file=sys.stderr)
+ return 1
+ for report in args.reports:
+ try:
+ summary = validate_path(report, args.synthesize_invalid, allowed_skips)
+ except ValueError as exc:
+ problems.append(f"{report}: {exc}")
+ if args.allow_skips_from and "unapproved skipped test IDs" in str(exc):
+ problems.append(
+ f"Review the skip policy in {args.allow_skips_from} before "
+ "approving an intentional new skip; do not allowlist a regression."
+ )
+ continue
+ if args.require_no_skips and summary.skipped:
+ problems.append(
+ f"{report}: report contains {summary.skipped} skipped tests"
+ )
+ continue
+ print(
+ f"{report}: {summary.tests} tests, {summary.passed} passed, "
+ f"{summary.skipped} skipped"
+ )
+
+ if problems:
+ for problem in problems:
+ print(problem, file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.yml
index 8c6386b53c..734a30ea04 100644
--- a/.github/workflows/platform-backend-ci.yml
+++ b/.github/workflows/platform-backend-ci.yml
@@ -6,6 +6,12 @@ on:
paths:
- ".github/workflows/platform-backend-ci.yml"
- ".github/workflows/scripts/get_package_version_from_lockfile.py"
+ - ".github/scripts/docker-pull-with-retry.sh"
+ - ".github/scripts/validate_junit.py"
+ - ".github/scripts/test_validate_junit.py"
+ - ".github/scripts/backend_test_shard.py"
+ - ".github/scripts/test_backend_test_shard.py"
+ - ".github/scripts/backend-allowed-skips.json"
- "autogpt_platform/backend/**"
- "autogpt_platform/autogpt_libs/**"
pull_request:
@@ -13,6 +19,12 @@ on:
paths:
- ".github/workflows/platform-backend-ci.yml"
- ".github/workflows/scripts/get_package_version_from_lockfile.py"
+ - ".github/scripts/docker-pull-with-retry.sh"
+ - ".github/scripts/validate_junit.py"
+ - ".github/scripts/test_validate_junit.py"
+ - ".github/scripts/backend_test_shard.py"
+ - ".github/scripts/test_backend_test_shard.py"
+ - ".github/scripts/backend-allowed-skips.json"
- "autogpt_platform/backend/**"
- "autogpt_platform/autogpt_libs/**"
merge_group:
@@ -21,18 +33,17 @@ on:
# automatic push/pull_request runs are skipped) but you still want to run the
# suite and produce a fresh coverage upload for that branch's HEAD commit.
# gh workflow run platform-backend-ci.yml --ref <branch>
- # Pass pr_number to attach the coverage upload to an open PR (refreshes that
- # PR's codecov/project/platform-backend status):
+ # Pass pr_number to attach the upload to that branch's open PR:
# gh workflow run platform-backend-ci.yml --ref <branch> -f pr_number=<PR#>
workflow_dispatch:
inputs:
pr_number:
- description: "Open PR number to attach this coverage upload to (refreshes that PR's codecov status). Leave blank for a plain branch run."
+ description: "Open PR number to attach coverage to. Leave blank for a branch run."
required: false
type: string
concurrency:
- group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
+ group: ${{ format('backend-ci-{0}', github.event_name == 'workflow_dispatch' && github.run_id || (github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha)) }}
cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
defaults:
@@ -68,6 +79,17 @@ jobs:
echo "Using Poetry version ${HEAD_POETRY_VERSION}"
curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
+ - name: Check poetry.lock
+ run: |
+ poetry lock
+
+ if ! git diff --quiet --ignore-matching-lines="^# " poetry.lock; then
+ echo "Error: poetry.lock not up to date."
+ echo
+ git diff poetry.lock
+ exit 1
+ fi
+
- name: Install Python dependencies
run: poetry install
@@ -116,7 +138,22 @@ jobs:
run: poetry install
- name: Run pytest
- run: poetry run pytest -q
+ run: poetry run pytest -q --junitxml=junit-autogpt-libs.xml
+
+ - name: Validate test report
+ if: ${{ always() }}
+ run: >-
+ python ../../.github/scripts/validate_junit.py --synthesize-invalid
+ --require-no-skips
+ junit-autogpt-libs.xml
+
+ - name: Upload test report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: autogpt-libs-test-report
+ path: autogpt_platform/autogpt_libs/junit-autogpt-libs.xml
+ if-no-files-found: error
env:
CI: true
@@ -167,20 +204,19 @@ jobs:
PLAIN_OUTPUT: True
test:
+ name: test (${{ matrix.python-version }}, ${{ matrix.test-shard }})
permissions:
contents: read
- # The suite is ~10.6k tests run serially (no pytest-xdist): the pytest step
- # alone measures p50 ~13min / max ~18min, on top of 2-7min of container,
- # checkout and dependency setup. A 20min cap sat *below* the job's real p95,
- # so healthy runs were killed mid-suite and reported as `cancelled`. On
- # merge_group that is fatal: `Check PR Status` treats a non-success check as
- # a failure and GitHub ejects the PR from the merge queue. This is a guard
- # against a hung job, not a performance budget - keep it well above p99.
+ # Each Python leg preserves the complete suite while its four disjoint path
+ # groups run on separate standard runners. The cap is a hang guard, not a
+ # performance budget: on merge_group a cancelled job reads as a failed
+ # check and ejects the PR from the queue, so keep it well above p99.
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
+ test-shard: [data, copilot, util-executor, remainder]
runs-on: ubuntu-latest
services:
@@ -198,7 +234,7 @@ jobs:
RABBITMQ_DEFAULT_USER: ${{ env.RABBITMQ_DEFAULT_USER }}
RABBITMQ_DEFAULT_PASS: ${{ env.RABBITMQ_DEFAULT_PASS }}
options: >-
- --health-cmd "rabbitmq-diagnostics -q ping"
+ --health-cmd "rabbitmq-diagnostics -q check_running"
--health-interval 30s
--health-timeout 10s
--health-retries 5
@@ -255,6 +291,7 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6
with:
+ ref: ${{ github.sha }}
# Full ref history is needed so the "Install Poetry" step below can
# read poetry.lock off the base branch. `filter: blob:none` keeps
# every ref reachable while skipping the blob download for all of
@@ -266,75 +303,184 @@ jobs:
filter: blob:none
submodules: true
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
-
- - name: Start Redis Cluster (3 shards)
+ - name: Configure test shard
+ shell: bash
+ env:
+ MATRIX_TEST_SHARD: ${{ matrix.test-shard }}
run: |
- # 3-master Redis Cluster matching the local compose stack
- # (autogpt_platform/docker-compose.platform.yml) and prod. Each
- # shard runs in its own container on a dedicated bridge network,
- # announces its compose-style hostname for intra-network clients,
- # and publishes 1700N on the GHA host so tests can reach every
- # shard via localhost. The backend's ``_address_remap`` rewrites
- # every CLUSTER SLOTS reply to localhost:<announced-port>, which
- # picks the right published port per shard.
- #
- # Not reusing docker-compose.platform.yml directly because compose
- # validates the full file even when only some services are ``up``
- # — pulling it in would needlessly couple CI to the full
- # local-dev stack.
- docker network create redis-cluster-ci
- for i in 0 1 2; do
- port=$((17000 + i))
- bus=$((27000 + i))
- docker run -d --name redis-$i --network redis-cluster-ci \
- --network-alias redis-$i \
- -p $port:$port \
- redis:7 \
- redis-server --port $port \
- --cluster-enabled yes \
- --cluster-config-file nodes.conf \
- --cluster-node-timeout 5000 \
- --cluster-require-full-coverage no \
- --cluster-announce-hostname redis-$i \
- --cluster-announce-port $port \
- --cluster-announce-bus-port $bus \
- --cluster-preferred-endpoint-type hostname
+ # The trial tests' safety guard only accepts localhost:5432/postgres
+ # under GITHUB_ACTIONS, and the per-shard names are cosmetic anyway —
+ # each shard already has its own runner and its own Postgres container.
+ case "$MATRIX_TEST_SHARD" in
+ data)
+ database_name="postgres"
+ ;;
+ copilot|remainder)
+ database_name="ci_${MATRIX_TEST_SHARD}"
+ ;;
+ util-executor)
+ database_name="ci_util_executor"
+ ;;
+ *)
+ echo "Unknown backend test shard: $MATRIX_TEST_SHARD" >&2
+ exit 1
+ ;;
+ esac
+
+ {
+ echo "TEST_SHARD=$MATRIX_TEST_SHARD"
+ echo "DATABASE_NAME=${database_name}"
+ echo "RABBITMQ_VHOST=ci-${MATRIX_TEST_SHARD}"
+ echo "REDIS_PORT=17000"
+ } >> "$GITHUB_ENV"
+
+ - name: Start isolated stateful services
+ id: stateful-services
+ working-directory: autogpt_platform
+ background: true
+ env:
+ CLAMAV_CONTAINER: ${{ job.services.clamav.id }}
+ RABBITMQ_CONTAINER: ${{ job.services.rabbitmq.id }}
+ run: |
+ postgres_image=pgvector/pgvector:pg15
+ bash "$GITHUB_WORKSPACE/.github/scripts/docker-pull-with-retry.sh" "$postgres_image"
+ docker run --pull=never -d --name postgres-test \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=postgres \
+ -p 5432:5432 \
+ -v "$(pwd)/db/init/00-init.sql:/docker-entrypoint-initdb.d/00-init.sql:ro" \
+ "$postgres_image"
+
+ postgres_ready=false
+ for _ in $(seq 1 30); do
+ if docker exec postgres-test pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1; then
+ postgres_ready=true
+ break
+ fi
+ sleep 1
done
- # Wait for each shard to accept commands.
- for i in 0 1 2; do
- port=$((17000 + i))
+ if [[ "$postgres_ready" != "true" ]]; then
+ echo "Postgres failed to start" >&2
+ docker logs postgres-test >&2
+ exit 1
+ fi
+
+ # The container's entrypoint already created and initialised `postgres`.
+ if [[ "$DATABASE_NAME" != "postgres" ]]; then
+ docker exec postgres-test createdb -h 127.0.0.1 -U postgres "$DATABASE_NAME"
+ docker exec postgres-test psql -h 127.0.0.1 -U postgres \
+ -v ON_ERROR_STOP=1 -d "$DATABASE_NAME" \
+ -f /docker-entrypoint-initdb.d/00-init.sql
+ fi
+
+ if [[ -z "$RABBITMQ_CONTAINER" ]]; then
+ echo "RabbitMQ service container was not found" >&2
+ exit 1
+ fi
+ docker exec "$RABBITMQ_CONTAINER" rabbitmqctl add_vhost "$RABBITMQ_VHOST"
+ docker exec "$RABBITMQ_CONTAINER" rabbitmqctl set_permissions \
+ -p "$RABBITMQ_VHOST" "$RABBITMQ_DEFAULT_USER" ".*" ".*" ".*"
+
+ start_redis_cluster() {
+ local name="$1"
+ local base_port="$2"
+ local network="redis-cluster-ci-${name}"
+ docker network create "$network"
+
+ for offset in 0 1 2; do
+ local port=$((base_port + offset))
+ local bus=$((base_port + 10000 + offset))
+ local container="redis-${name}-${offset}"
+ docker run --pull=never -d --name "$container" --network "$network" \
+ --network-alias "$container" \
+ -p "$port:$port" \
+ "$redis_image" \
+ redis-server --port "$port" \
+ --cluster-enabled yes \
+ --cluster-config-file nodes.conf \
+ --cluster-node-timeout 5000 \
+ --cluster-require-full-coverage no \
+ --cluster-announce-hostname "$container" \
+ --cluster-announce-port "$port" \
+ --cluster-announce-bus-port "$bus" \
+ --cluster-preferred-endpoint-type hostname
+ done
+
+ for offset in 0 1 2; do
+ local port=$((base_port + offset))
+ local container="redis-${name}-${offset}"
+ local ready=false
+ for _ in $(seq 1 30); do
+ if docker exec "$container" redis-cli -p "$port" ping 2>/dev/null | grep -q '^PONG$'; then
+ ready=true
+ break
+ fi
+ sleep 1
+ done
+ if [[ "$ready" != "true" ]]; then
+ echo "$container failed to accept commands" >&2
+ docker logs "$container" >&2
+ return 1
+ fi
+ done
+
+ docker run --pull=never --rm --network "$network" "$redis_image" \
+ redis-cli --cluster create \
+ "redis-${name}-0:${base_port}" \
+ "redis-${name}-1:$((base_port + 1))" \
+ "redis-${name}-2:$((base_port + 2))" \
+ --cluster-replicas 0 --cluster-yes
+
for _ in $(seq 1 30); do
- docker exec redis-$i redis-cli -p $port ping 2>/dev/null | grep -q PONG && break
+ local info
+ if info=$(docker exec "redis-${name}-0" redis-cli -p "$base_port" cluster info); then
+ local state
+ state=$(printf '%s\n' "$info" | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]')
+ if [[ "$state" == "ok" ]]; then
+ echo "Redis cluster ${name} ready on ${base_port}-$((base_port + 2))"
+ return 0
+ fi
+ fi
sleep 1
done
- done
- # Form the cluster from an init container on the same network so
- # --cluster-preferred-endpoint-type hostname resolves redis-0/1/2.
- docker run --rm --network redis-cluster-ci redis:7 \
- redis-cli --cluster create \
- redis-0:17000 redis-1:17001 redis-2:17002 \
- --cluster-replicas 0 --cluster-yes
- # Confirm convergence.
- for _ in $(seq 1 30); do
- state=$(docker exec redis-0 redis-cli -p 17000 cluster info | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]')
- if [ "$state" = "ok" ]; then
- echo "Redis Cluster ready (3 shards, state=ok)"
- docker exec redis-0 redis-cli -p 17000 cluster nodes
- exit 0
+
+ echo "Redis cluster ${name} failed to converge" >&2
+ docker exec "redis-${name}-0" redis-cli -p "$base_port" cluster info >&2
+ return 1
+ }
+
+ redis_image=redis:7
+ bash "$GITHUB_WORKSPACE/.github/scripts/docker-pull-with-retry.sh" "$redis_image"
+ start_redis_cluster "$TEST_SHARD" "$REDIS_PORT"
+
+ if [[ -z "$CLAMAV_CONTAINER" ]]; then
+ echo "ClamAV service container was not found" >&2
+ exit 1
+ fi
+ clamav_ready=false
+ for _ in $(seq 1 60); do
+ if nc -z localhost 3310; then
+ clamav_ready=true
+ break
fi
- sleep 1
+ sleep 5
done
- echo "Redis Cluster failed to reach ok state" >&2
- docker exec redis-0 redis-cli -p 17000 cluster info >&2 || true
- exit 1
+ if [[ "$clamav_ready" != "true" ]]; then
+ echo "ClamAV failed to start" >&2
+ docker logs "$CLAMAV_CONTAINER" >&2
+ exit 1
+ fi
+ if ! timeout 10 bash -c 'echo "PING" | nc localhost 3310'; then
+ echo "ClamAV is not responding to PING" >&2
+ docker logs "$CLAMAV_CONTAINER" >&2
+ exit 1
+ fi
- - id: get_date
- name: Get date
- run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
- name: Set up Python dependency cache
uses: actions/cache@v5
@@ -350,7 +496,11 @@ jobs:
if [ -n "$BASE_REF" ]; then
BASE_BRANCH=${BASE_REF/refs\/heads\//}
- BASE_POETRY_VERSION=$((git show "origin/$BASE_BRANCH":./poetry.lock; true) | python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry -)
+ if ! BASE_LOCK=$(git show "origin/$BASE_BRANCH":./poetry.lock); then
+ echo "Could not read poetry.lock from ${BASE_REF}" >&2
+ exit 1
+ fi
+ BASE_POETRY_VERSION=$(printf '%s\n' "$BASE_LOCK" | python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry -)
echo "Found Poetry version ${BASE_POETRY_VERSION} in backend/poetry.lock on ${BASE_REF}"
POETRY_VERSION=$(printf '%s\n' "$HEAD_POETRY_VERSION" "$BASE_POETRY_VERSION" | sort -V | tail -n1)
else
@@ -363,109 +513,90 @@ jobs:
if [ "${{ runner.os }}" = "macOS" ]; then
PATH="$HOME/.local/bin:$PATH"
- echo "$HOME/.local/bin" >> $GITHUB_PATH
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
fi
env:
BASE_REF: ${{ github.base_ref || github.event.merge_group.base_ref }}
- - name: Check poetry.lock
- run: |
- poetry lock
-
- if ! git diff --quiet --ignore-matching-lines="^# " poetry.lock; then
- echo "Error: poetry.lock not up to date."
- echo
- git diff poetry.lock
- exit 1
- fi
-
- name: Install Python dependencies
run: poetry install
- name: Generate Prisma Client
run: poetry run prisma generate && poetry run gen-prisma-stub
- - name: Start Postgres
- working-directory: autogpt_platform
- run: |
- # Plain pgvector Postgres; the init script creates the legacy
- # `auth` schema shim that historical migrations reference.
- docker run -d --name postgres-test \
- -e POSTGRES_USER=postgres \
- -e POSTGRES_PASSWORD=postgres \
- -e POSTGRES_DB=postgres \
- -p 5432:5432 \
- -v "$(pwd)/db/init/00-init.sql:/docker-entrypoint-initdb.d/00-init.sql:ro" \
- pgvector/pgvector:pg15
- for _ in $(seq 1 30); do
- if docker exec postgres-test pg_isready -U postgres >/dev/null 2>&1; then
- echo "Postgres ready"
- exit 0
- fi
- sleep 1
- done
- echo "Postgres failed to start" >&2
- docker logs postgres-test >&2
- exit 1
+ - name: Wait for isolated stateful services
+ wait: stateful-services
- - name: Wait for ClamAV to be ready
+ - name: Migrate shard database
run: |
- echo "Waiting for ClamAV daemon to start..."
- max_attempts=60
- attempt=0
-
- until nc -z localhost 3310 || [ $attempt -eq $max_attempts ]; do
- echo "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"
- sleep 5
- attempt=$((attempt+1))
- done
-
- if [ $attempt -eq $max_attempts ]; then
- echo "ClamAV failed to start after $((max_attempts*5)) seconds"
- echo "Checking ClamAV service logs..."
- docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
- exit 1
- fi
-
- echo "ClamAV is ready!"
-
- # Verify ClamAV is responsive
- echo "Testing ClamAV connection..."
- timeout 10 bash -c 'echo "PING" | nc localhost 3310' || {
- echo "ClamAV is not responding to PING"
- docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
- exit 1
- }
+ export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/${DATABASE_NAME}"
+ export DIRECT_URL="$DATABASE_URL"
+ poetry run prisma migrate deploy
- - name: Run Database Migrations
- run: poetry run prisma migrate deploy
+ - name: Test shard
+ shell: bash
env:
- DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
- DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
-
- - name: Run pytest with coverage
+ COVERAGE_FILE: .coverage-${{ matrix.test-shard }}
+ LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
+ EXECUTION_MANAGER_PORT: "18002"
+ DATABASE_API_PORT: "18005"
+ AGENT_API_PORT: "18006"
+ EXECUTION_SCHEDULER_PORT: "18003"
+ NOTIFICATION_SERVICE_PORT: "18007"
+ JWT_VERIFY_KEY: "ci-only-jwt-verify-key-with-at-least-32-characters"
+ REDIS_HOST: localhost
+ ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw="
+ TRIAL_TEST_DATABASE: "1"
run: |
+ export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/${DATABASE_NAME}"
+ export DIRECT_URL="$DATABASE_URL"
+ pytest_args=(
+ -p no:cacheprovider
+ --cov=backend
+ --cov-branch
+ --cov-report=
+ --cov-report="xml:coverage-${TEST_SHARD}.xml"
+ --junitxml="junit-${TEST_SHARD}.xml"
+ )
if [[ "${{ runner.debug }}" == "1" ]]; then
- poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG \
- --cov=backend --cov-branch --cov-report term-missing --cov-report xml
+ pytest_args+=(-s -vv -o log_cli=true -o log_cli_level=DEBUG)
else
- poetry run pytest -s -vv \
- --cov=backend --cov-branch --cov-report term-missing --cov-report xml
+ pytest_args+=(-q)
fi
- env:
- LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
- DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
- DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
- JWT_VERIFY_KEY: "ci-only-jwt-verify-key-with-at-least-32-characters" # DO NOT USE IN PRODUCTION!!
- REDIS_HOST: "localhost"
- REDIS_PORT: "17000"
- ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
- # Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its
- # own isolated 3-shard cluster (ports 27110–27112) and exercise
- # ``docker restart <shard>`` mid-stream. Off locally so a
- # contributor's ``poetry run test`` doesn't pay the ~15s cost.
- E2E_RESTART_ISOLATED: "1"
- TRIAL_TEST_DATABASE: "1"
+
+ python ../../.github/scripts/backend_test_shard.py "$TEST_SHARD" -- "${pytest_args[@]}"
+
+ - name: Validate test reports
+ if: ${{ always() }}
+ run: >-
+ python ../../.github/scripts/validate_junit.py --synthesize-invalid
+ --allow-skips-from ../../.github/scripts/backend-allowed-skips.json
+ "junit-${TEST_SHARD}.xml"
+
+ - name: Validate coverage reports
+ if: ${{ always() }}
+ run: |
+ python - "coverage-${TEST_SHARD}.xml" <<'PY'
+ import sys
+ from pathlib import Path
+ from xml.etree import ElementTree
+
+ report = Path(sys.argv[1])
+ if not report.is_file() or not report.stat().st_size:
+ raise SystemExit(f"Missing or empty report: {report}")
+ ElementTree.parse(report)
+ print(f"Validated nonempty XML report: {report}")
+ PY
+
+ - name: Upload test and coverage reports
+ if: ${{ always() }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: backend-test-reports-py${{ matrix.python-version }}-${{ matrix.test-shard }}
+ path: |
+ autogpt_platform/backend/junit-${{ matrix.test-shard }}.xml
+ autogpt_platform/backend/coverage-${{ matrix.test-shard }}.xml
+ if-no-files-found: error
- name: Upload coverage reports to Codecov
if: ${{ !cancelled() }}
@@ -473,12 +604,8 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: platform-backend
- files: ./autogpt_platform/backend/coverage.xml
- # On a manual workflow_dispatch, attach the upload to the given PR so
- # codecov re-evaluates that PR's project status against its base.
- # Empty for all automatic events (push/pull_request/merge_group), so
- # their normal PR/commit detection is unchanged.
override_pr: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || '' }}
+ files: ./autogpt_platform/backend/coverage-${{ matrix.test-shard }}.xml
env:
CI: true
diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/logging/formatters.py b/autogpt_platform/autogpt_libs/autogpt_libs/logging/formatters.py
index 28b998f2ce..c3aa997c8c 100644
--- a/autogpt_platform/autogpt_libs/autogpt_libs/logging/formatters.py
+++ b/autogpt_platform/autogpt_libs/autogpt_libs/logging/formatters.py
@@ -1,4 +1,5 @@
import logging
+from copy import copy
from colorama import Fore, Style
@@ -26,6 +27,7 @@ class FancyConsoleFormatter(logging.Formatter):
}
def format(self, record: logging.LogRecord) -> str:
+ record = copy(record)
# Make sure `msg` is a string
if not hasattr(record, "msg"):
record.msg = ""
@@ -55,6 +57,7 @@ class AGPTFormatter(FancyConsoleFormatter):
self.no_color = no_color
def format(self, record: logging.LogRecord) -> str:
+ record = copy(record)
# Make sure `msg` is a string
if not hasattr(record, "msg"):
record.msg = ""
diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/logging/test_formatters.py b/autogpt_platform/autogpt_libs/autogpt_libs/logging/test_formatters.py
new file mode 100644
index 0000000000..8484665782
--- /dev/null
+++ b/autogpt_platform/autogpt_libs/autogpt_libs/logging/test_formatters.py
@@ -0,0 +1,28 @@
+import logging
+
+from colorama import Fore, Style
+
+from .formatters import AGPTFormatter
+
+
+def test_formatter_does_not_mutate_log_record() -> None:
+ record = logging.LogRecord(
+ name="test",
+ level=logging.ERROR,
+ pathname=__file__,
+ lineno=1,
+ msg="failed %s",
+ args=("safely",),
+ exc_info=None,
+ )
+ formatter = AGPTFormatter("%(levelname)s %(title)s%(message)s")
+
+ output = formatter.format(record)
+
+ assert output == (
+ f"{Fore.RED}ERROR{Style.RESET_ALL} {Fore.RED}failed safely{Style.RESET_ALL}"
+ )
+ assert record.levelname == "ERROR"
+ assert record.msg == "failed %s"
+ assert record.args == ("safely",)
+ assert not hasattr(record, "title")
diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.py
index 0a21ce30b8..739a79e27c 100644
--- a/autogpt_platform/backend/backend/copilot/executor/utils.py
+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py
@@ -31,6 +31,7 @@ from backend.copilot.tree import (
)
from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
from backend.util.logging import TruncatedLogger, is_structured_logging_enabled
+from backend.util.settings import Config
logger = logging.getLogger(__name__)
@@ -174,7 +175,7 @@ def create_copilot_queue_config() -> RabbitMQConfig:
arguments={"x-queue-type": "quorum"},
)
return RabbitMQConfig(
- vhost="/",
+ vhost=Config().rabbitmq_vhost,
exchanges=[COPILOT_EXECUTION_EXCHANGE, COPILOT_CANCEL_EXCHANGE],
queues=[run_queue, cancel_queue],
)
diff --git a/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.py b/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.py
index a273e90da7..350ddd0a88 100644
--- a/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.py
+++ b/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.py
@@ -408,6 +408,7 @@ async def test_graph_execution_queue_publish_and_consume() -> None:
→ consume → payload round-trips intact. Uses a unique routing key so
the live executor consumer (if any) doesn't race for the message."""
from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
+ from backend.util.settings import Config
test_queue_name = f"e2e_test_{uuid4().hex[:8]}_v2"
test_routing_key = f"e2e.test.{uuid4().hex[:8]}"
@@ -426,7 +427,11 @@ async def test_graph_execution_queue_publish_and_consume() -> None:
routing_key=test_routing_key,
arguments={"x-queue-type": "quorum"},
)
- cfg = RabbitMQConfig(vhost="/", exchanges=[test_exchange], queues=[test_queue])
+ cfg = RabbitMQConfig(
+ vhost=Config().rabbitmq_vhost,
+ exchanges=[test_exchange],
+ queues=[test_queue],
+ )
publisher = AsyncRabbitMQ(cfg)
await publisher.connect()
diff --git a/autogpt_platform/backend/backend/data/rabbitmq_test.py b/autogpt_platform/backend/backend/data/rabbitmq_test.py
index 6f666148ca..93cec9fa83 100644
--- a/autogpt_platform/backend/backend/data/rabbitmq_test.py
+++ b/autogpt_platform/backend/backend/data/rabbitmq_test.py
@@ -27,10 +27,25 @@ from backend.executor.utils import (
GRAPH_EXECUTION_ROUTING_KEY,
create_execution_queue_config,
)
+from backend.notifications.queue import create_notification_config
# ---------- Quorum queue config: classic→quorum rollover guard ----------
+ "factory",
+ [
+ create_execution_queue_config,
+ create_copilot_queue_config,
+ create_notification_config,
+ ],
+)
[email protected]("vhost", ["/", "ci-shard"])
+def test_queue_config_uses_configured_vhost(monkeypatch, factory, vhost: str) -> None:
+ monkeypatch.setenv("RABBITMQ_VHOST", vhost)
+ assert factory().vhost == vhost
+
+
def test_graph_execution_queue_is_quorum() -> None:
"""Run queue must declare `x-queue-type=quorum` to survive a single
broker-node outage (AUTOGPT-SERVER-8ST/SV/SW)."""
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index fc8c5f65e2..173d62adb4 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -1033,7 +1033,7 @@ def create_execution_queue_config() -> RabbitMQConfig:
arguments={"x-queue-type": "quorum"},
)
return RabbitMQConfig(
- vhost="/",
+ vhost=Config().rabbitmq_vhost,
exchanges=[GRAPH_EXECUTION_EXCHANGE, GRAPH_EXECUTION_CANCEL_EXCHANGE],
queues=[run_queue, cancel_queue],
)
diff --git a/autogpt_platform/backend/backend/notifications/queue.py b/autogpt_platform/backend/backend/notifications/queue.py
index 82caa32761..241c6f356c 100644
--- a/autogpt_platform/backend/backend/notifications/queue.py
+++ b/autogpt_platform/backend/backend/notifications/queue.py
@@ -20,6 +20,7 @@ from backend.data.notifications import (
)
from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
from backend.util.logging import TruncatedLogger
+from backend.util.settings import Config
logger = TruncatedLogger(logging.getLogger(__name__), "[NotificationQueue]")
@@ -54,7 +55,9 @@ def create_notification_config() -> RabbitMQConfig:
arguments={"x-queue-type": _QUORUM},
),
]
- return RabbitMQConfig(exchanges=EXCHANGES, queues=queues)
+ return RabbitMQConfig(
+ vhost=Config().rabbitmq_vhost, exchanges=EXCHANGES, queues=queues
+ )
def get_routing_key(event_type: NotificationType) -> str: