AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
ci(frontend): shard integration tests with verified coverage (#14280)
Nicholas Tindle14f8d6539a52
diff --git a/.github/scripts/run_command_junit.py b/.github/scripts/run_command_junit.pynew file mode 100644index 0000000000..73ace7ce73--- /dev/null+++ b/.github/scripts/run_command_junit.py@@ -0,0 +1,97 @@+#!/usr/bin/env python3++import argparse+import subprocess+import sys+import time+import xml.etree.ElementTree as ET+from pathlib import Path+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser()+ parser.add_argument("--name", required=True)+ parser.add_argument("--classname", required=True)+ parser.add_argument("--output", type=Path, required=True)+ parser.add_argument("command", nargs=argparse.REMAINDER)+ args = parser.parse_args()+ if args.command[:1] == ["--"]:+ args.command = args.command[1:]+ if not args.command:+ parser.error("a command is required after --")+ return args+++def write_junit(+ path: Path,+ name: str,+ classname: str,+ command: list[str],+ returncode: int | None,+ duration: float,+ spawn_error: str | None = None,+) -> None:+ suite = ET.Element(+ "testsuite",+ {+ "name": classname,+ "tests": "1",+ "failures": "1" if returncode and spawn_error is None else "0",+ "errors": "1" if spawn_error is not None else "0",+ "skipped": "0",+ "time": f"{duration:.6f}",+ },+ )+ case = ET.SubElement(+ suite,+ "testcase",+ {"classname": classname, "name": name, "time": f"{duration:.6f}"},+ )+ if spawn_error is not None:+ error = ET.SubElement(+ case,+ "error",+ {"message": "command could not be started"},+ )+ error.text = spawn_error+ elif returncode:+ failure = ET.SubElement(+ case,+ "failure",+ {"message": f"command exited with status {returncode}"},+ )+ failure.text = " ".join(command)++ path.parent.mkdir(parents=True, exist_ok=True)+ ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True)+++def main() -> int:+ args = parse_args()+ started_at = time.monotonic()+ spawn_error = None+ try:+ completed = subprocess.run(args.command, check=False)+ returncode = completed.returncode+ except OSError as error:+ spawn_error = f"{type(error).__name__}: {error}"+ returncode = 127+ duration = time.monotonic() - started_at+ try:+ write_junit(+ args.output,+ args.name,+ args.classname,+ args.command,+ None if spawn_error is not None else returncode,+ duration,+ spawn_error=spawn_error,+ )+ except OSError as error:+ print(f"Failed to write JUnit report {args.output}: {error}", file=sys.stderr)+ return returncode or 1+ return returncode+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/scripts/run_unittest_junit.py b/.github/scripts/run_unittest_junit.pynew file mode 100644index 0000000000..beef0db8b7--- /dev/null+++ b/.github/scripts/run_unittest_junit.py@@ -0,0 +1,261 @@+#!/usr/bin/env python3++import argparse+import sys+import time+import unittest+import xml.etree.ElementTree as ET+from dataclasses import dataclass+from pathlib import Path+++@dataclass+class TestRecord:+ classname: str+ name: str+ duration: float+ outcome: str+ detail: str = ""+++class JUnitResult(unittest.TextTestResult):+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.records: list[TestRecord] = []+ self._started_at: dict[int, float] = {}+ self._outcomes: dict[int, list[tuple[unittest.TestCase, str, str]]] = {}+ self._recorded: set[int] = set()++ def _append_record(self, test, duration: float, outcome: str, detail: str):+ test_id = test.id()+ classname, _, name = test_id.rpartition(".")+ self.records.append(+ TestRecord(+ classname=classname,+ name=name or test_id,+ duration=duration,+ outcome=outcome,+ detail=detail,+ )+ )+ self._recorded.add(id(test))++ def startTest(self, test):+ self._started_at[id(test)] = time.monotonic()+ self._outcomes[id(test)] = []+ super().startTest(test)++ def addFailure(self, test, err):+ self._outcomes[id(test)].append(+ (test, "failure", self._exc_info_to_string(err, test))+ )+ super().addFailure(test, err)++ def addError(self, test, err):+ detail = self._exc_info_to_string(err, test)+ if id(test) in self._started_at:+ self._outcomes[id(test)].append((test, "error", detail))+ else:+ self._append_record(test, 0.0, "error", detail)+ super().addError(test, err)++ def addSkip(self, test, reason):+ if id(test) in self._started_at:+ self._outcomes[id(test)].append((test, "skipped", reason))+ else:+ self._append_record(test, 0.0, "skipped", reason)+ super().addSkip(test, reason)++ def addExpectedFailure(self, test, err):+ self._outcomes[id(test)].append(+ (+ test,+ "expected_failure",+ f"expected failure\n{self._exc_info_to_string(err, test)}",+ )+ )+ super().addExpectedFailure(test, err)++ def addUnexpectedSuccess(self, test):+ self._outcomes[id(test)].append((test, "failure", "unexpected success"))+ super().addUnexpectedSuccess(test)++ def addSubTest(self, test, subtest, err):+ if err is not None:+ outcome = (+ "failure" if issubclass(err[0], test.failureException) else "error"+ )+ detail = self._exc_info_to_string(err, subtest)+ self._outcomes[id(test)].append((subtest, outcome, detail))+ super().addSubTest(test, subtest, err)++ def stopTest(self, test):+ started_at = self._started_at.pop(id(test), time.monotonic())+ outcomes = self._outcomes.pop(id(test), []) or [(test, "success", "")]+ if id(test) not in self._recorded:+ duration = time.monotonic() - started_at+ for recorded_test, outcome, detail in outcomes:+ self._append_record(recorded_test, duration, outcome, detail)+ self._recorded.discard(id(recorded_test))+ duration = 0.0+ self._recorded.discard(id(test))+ super().stopTest(test)+++def write_junit(+ path: Path,+ suite_name: str,+ records: list[TestRecord],+ duration: float,+) -> None:+ failures = sum(record.outcome == "failure" for record in records)+ errors = sum(record.outcome == "error" for record in records)+ skipped = sum(record.outcome == "skipped" for record in records)+ suite = ET.Element(+ "testsuite",+ {+ "name": suite_name,+ "tests": str(len(records)),+ "failures": str(failures),+ "errors": str(errors),+ "skipped": str(skipped),+ "time": f"{duration:.6f}",+ },+ )+ for record in records:+ case = ET.SubElement(+ suite,+ "testcase",+ {+ "classname": record.classname,+ "name": record.name,+ "time": f"{record.duration:.6f}",+ },+ )+ if record.outcome != "success":+ if record.outcome == "skipped":+ tag = "skipped"+ elif record.outcome == "error":+ tag = "error"+ else:+ tag = "failure"+ detail = xml_safe(record.detail)+ message = detail.splitlines()[0] if detail else record.outcome+ child = ET.SubElement(case, tag, {"message": message})+ child.text = detail++ path.parent.mkdir(parents=True, exist_ok=True)+ ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True)+++def xml_safe(text: str) -> str:+ # unittest colourises tracebacks on Python 3.13+, and ANSI escapes are+ # not valid XML 1.0 characters, so a failing run would emit an+ # unparseable report.+ return "".join(+ character+ for character in text+ if character in "\t\n\r"+ or 0x20 <= ord(character) <= 0xD7FF+ or 0xE000 <= ord(character) <= 0xFFFD+ or 0x10000 <= ord(character) <= 0x10FFFF+ )+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser()+ parser.add_argument("--start-directory", required=True)+ parser.add_argument("--pattern", default="test*.py")+ parser.add_argument("--suite-name", default="single-container-runtime-helpers")+ parser.add_argument("--output", type=Path, required=True)+ parser.add_argument(+ "--allow-skip",+ action="append",+ default=[],+ metavar="TEST_ID",+ help="exact unittest ID allowed to report as skipped (repeatable)",+ )+ return parser.parse_args()+++def enforce_test_policy(+ records: list[TestRecord],+ suite_name: str,+ allowed_skips: set[str],+) -> bool:+ """Reject omitted tests unless an exact skip ID is explicitly allowlisted."""+ if not records:+ records.append(+ TestRecord(+ classname=suite_name,+ name="test_discovery",+ duration=0.0,+ outcome="error",+ detail="unittest discovery found zero tests",+ )+ )+ return False++ valid = True+ for record in records:+ test_id = (+ f"{record.classname}.{record.name}" if record.classname else record.name+ )+ if record.outcome == "expected_failure":+ record.outcome = "failure"+ record.detail = f"expected failures are not allowed\n{record.detail}"+ valid = False+ elif record.outcome == "skipped" and test_id not in allowed_skips:+ record.outcome = "failure"+ record.detail = f"disallowed skip for {test_id}\n{record.detail}"+ valid = False+ return valid+++def main() -> int:+ args = parse_args()+ started_at = time.monotonic()+ try:+ suite = unittest.defaultTestLoader.discover(+ args.start_directory,+ pattern=args.pattern,+ )+ runner = unittest.TextTestRunner(+ stream=sys.stderr,+ verbosity=2,+ resultclass=JUnitResult,+ )+ result = runner.run(suite)+ records = result.records+ policy_passed = enforce_test_policy(+ records,+ args.suite_name,+ set(args.allow_skip),+ )+ successful = result.wasSuccessful() and policy_passed+ except Exception as error:+ records = [+ TestRecord(+ classname=args.suite_name,+ name="test_discovery",+ duration=0.0,+ outcome="error",+ detail=f"{type(error).__name__}: {error}",+ )+ ]+ successful = False+ try:+ write_junit(+ args.output,+ args.suite_name,+ records,+ time.monotonic() - started_at,+ )+ except OSError as error:+ print(f"Failed to write JUnit report {args.output}: {error}", file=sys.stderr)+ return 1+ return 0 if successful else 1+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/scripts/test_frontend_cobertura_contract.py b/.github/scripts/test_frontend_cobertura_contract.pynew file mode 100644index 0000000000..a1180ae6c9--- /dev/null+++ b/.github/scripts/test_frontend_cobertura_contract.py@@ -0,0 +1,24 @@+import unittest+from pathlib import Path+++class FrontendCoverageContractTests(unittest.TestCase):+ def test_codecov_requires_validated_coverage(self):+ workflow = (+ Path(__file__).resolve().parents[1] / "workflows/platform-frontend-ci.yml"+ ).read_text(encoding="utf-8")+ validator = workflow.split("- name: Validate coverage report\n", 1)[1].split(+ "\n - name:", 1+ )[0]+ upload = workflow.split("- name: Upload coverage reports to Codecov\n", 1)[1]+ self.assertIn("id: validate-frontend-coverage", validator)+ self.assertIn("if: ${{ always() }}", validator)+ self.assertNotIn("continue-on-error", validator)+ self.assertIn(+ "if: ${{ !cancelled() && steps.validate-frontend-coverage.outcome == 'success' }}",+ upload,+ )+++if __name__ == "__main__":+ unittest.main()diff --git a/.github/scripts/test_validate_cobertura.py b/.github/scripts/test_validate_cobertura.pynew file mode 100644index 0000000000..79f5a2c54c--- /dev/null+++ b/.github/scripts/test_validate_cobertura.py@@ -0,0 +1,129 @@+import tempfile+import unittest+import xml.etree.ElementTree as ET+from pathlib import Path++from validate_cobertura import main, validate_report+++def valid_report() -> ET.Element:+ return ET.fromstring(+ '<coverage lines-valid="1" lines-covered="0" line-rate="0">'+ '<packages><package name="src"><classes>'+ '<class name="example" filename="src/example.ts">'+ '<lines><line number="1" hits="0"/></lines>'+ "</class></classes></package></packages></coverage>"+ )+++class ValidateCoberturaTests(unittest.TestCase):+ def test_accepts_real_uncovered_lines_without_imposing_a_threshold(self):+ validate_report(valid_report())++ def test_rejects_arbitrary_xml_and_empty_coverage(self):+ for report in ("<report/>", "<coverage/>"):+ with self.subTest(report=report), self.assertRaises(ValueError):+ validate_report(ET.fromstring(report))++ def test_rejects_invalid_summary(self):+ for attribute, value in (+ ("lines-valid", "0"),+ ("lines-valid", "-1"),+ ("lines-covered", "2"),+ ("lines-covered", "0.5"),+ ("line-rate", "nan"),+ ("line-rate", "inf"),+ ("line-rate", "-0.1"),+ ("line-rate", "1.1"),+ ):+ with self.subTest(attribute=attribute, value=value):+ report = valid_report()+ report.set(attribute, value)+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_requires_source_classes_and_lines(self):+ for path in ("packages", "packages/package/classes/class/lines"):+ with self.subTest(path=path):+ report = valid_report()+ report.find(path).clear()+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_rejects_invalid_line_data(self):+ for attribute, value in (("number", "0"), ("hits", "-1")):+ with self.subTest(attribute=attribute):+ report = valid_report()+ report.find(".//line").set(attribute, value)+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_rejects_missing_filename(self):+ report = valid_report()+ del report.find(".//class").attrib["filename"]+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_rejects_summary_counts_that_disagree_with_source_lines(self):+ for attribute, value in (("lines-valid", "2"), ("lines-covered", "1")):+ with self.subTest(attribute=attribute):+ report = valid_report()+ report.set(attribute, value)+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_rejects_uncovered_summary_with_a_covered_source_line(self):+ report = valid_report()+ report.find(".//class/lines/line").set("hits", "1")+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_rejects_rate_that_disagrees_with_source_line_counts(self):+ report = valid_report()+ report.set("line-rate", "0.5")+ with self.assertRaises(ValueError):+ validate_report(report)++ def test_accepts_rounded_reporter_rate_and_ignores_method_line_duplicates(self):+ report = valid_report()+ report.set("lines-valid", "3")+ report.set("lines-covered", "1")+ lines = report.find(".//class/lines")+ ET.SubElement(lines, "line", number="2", hits="1")+ ET.SubElement(lines, "line", number="3", hits="0")+ methods = ET.SubElement(report.find(".//class"), "methods")+ method_lines = ET.SubElement(ET.SubElement(methods, "method"), "lines")+ ET.SubElement(method_lines, "line", number="2", hits="1")+ for rate in ("0.3333", "0.33329999999999999", str(1 / 3)):+ with self.subTest(rate=rate):+ report.set("line-rate", rate)+ validate_report(report)++ def test_counts_nested_packages_in_e2e_reports(self):+ report = valid_report()+ classes = report.find("packages/package/classes")+ source = classes.find("class")+ classes.remove(source)+ nested = ET.SubElement(classes, "package", name="nested")+ ET.SubElement(nested, "classes").append(source)+ validate_report(report)++ def test_cli_fails_for_missing_empty_malformed_or_noncoverage_reports(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "coverage.xml"+ self.assertEqual(main([str(path)]), 1)+ for text in ("", "<coverage", "<report/>"):+ with self.subTest(text=text):+ path.write_text(text, encoding="utf-8")+ self.assertEqual(main([str(path)]), 1)+ self.assertEqual(path.read_text(encoding="utf-8"), text)++ def test_cli_accepts_valid_report(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "coverage.xml"+ ET.ElementTree(valid_report()).write(path, encoding="utf-8")+ self.assertEqual(main([str(path)]), 0)+++if __name__ == "__main__":+ unittest.main()diff --git a/.github/scripts/validate_cobertura.py b/.github/scripts/validate_cobertura.pynew file mode 100644index 0000000000..e55aa51cdd--- /dev/null+++ b/.github/scripts/validate_cobertura.py@@ -0,0 +1,61 @@+import argparse+import math+import sys+import xml.etree.ElementTree as ET+from pathlib import Path+++def nonnegative_integer(element: ET.Element, attribute: str) -> int:+ value = element.get(attribute, "")+ if not value.isascii() or not value.isdecimal():+ raise ValueError(f"{element.tag}.{attribute} must be a non-negative integer")+ return int(value)+++def validate_report(root: ET.Element) -> None:+ if root.tag != "coverage":+ raise ValueError("report is not Cobertura coverage XML")+ valid = nonnegative_integer(root, "lines-valid")+ covered = nonnegative_integer(root, "lines-covered")+ if valid == 0 or covered > valid:+ raise ValueError("coverage must contain lines with valid coverage counts")+ rate = float(root.get("line-rate", "nan"))+ if not math.isfinite(rate) or not 0 <= rate <= 1:+ raise ValueError("coverage.line-rate must be between 0 and 1")++ classes = root.findall("packages//class")+ if not classes:+ raise ValueError("coverage contains no source classes")+ line_count = 0+ covered_count = 0+ for source in classes:+ if not source.get("filename", "").strip():+ raise ValueError("coverage class is missing a filename")+ for line in source.findall("lines/line"):+ if nonnegative_integer(line, "number") == 0:+ raise ValueError("coverage line numbers must be positive")+ covered_count += nonnegative_integer(line, "hits") > 0+ line_count += 1+ if line_count == 0:+ raise ValueError("coverage contains no source lines")+ if valid != line_count or covered != covered_count:+ raise ValueError("coverage summary counts do not match source lines")+ # Istanbul truncates percentages to two decimals before converting to a rate.+ if not math.isclose(rate, covered_count / line_count, rel_tol=0, abs_tol=1e-4):+ raise ValueError("coverage.line-rate does not match source lines")+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser()+ parser.add_argument("report", type=Path)+ args = parser.parse_args(argv)+ try:+ validate_report(ET.parse(args.report).getroot())+ except (OSError, ET.ParseError, ValueError) as error:+ print(f"{args.report}: {error}", file=sys.stderr)+ return 1+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/workflows/platform-frontend-ci.yml b/.github/workflows/platform-frontend-ci.ymlindex fc08440d91..363a1602a4 100644--- a/.github/workflows/platform-frontend-ci.yml+++ b/.github/workflows/platform-frontend-ci.yml@@ -5,6 +5,12 @@ on:branches: [master, dev]paths:- ".github/workflows/platform-frontend-ci.yml"+ - ".github/scripts/validate_junit.py"+ - ".github/scripts/test_validate_junit.py"+ - ".github/scripts/validate_cobertura.py"+ - ".github/scripts/test_validate_cobertura.py"+ - ".github/scripts/test_frontend_cobertura_contract.py"+ - ".github/scripts/run_unittest_junit.py"- "autogpt_platform/frontend/**"- "autogpt_platform/backend/Dockerfile"- "autogpt_platform/docker-compose.yml"@@ -12,6 +18,12 @@ on:pull_request:paths:- ".github/workflows/platform-frontend-ci.yml"+ - ".github/scripts/validate_junit.py"+ - ".github/scripts/test_validate_junit.py"+ - ".github/scripts/validate_cobertura.py"+ - ".github/scripts/test_validate_cobertura.py"+ - ".github/scripts/test_frontend_cobertura_contract.py"+ - ".github/scripts/run_unittest_junit.py"- "autogpt_platform/frontend/**"- "autogpt_platform/backend/Dockerfile"- "autogpt_platform/docker-compose.yml"@@ -20,7 +32,7 @@ on:workflow_dispatch:concurrency:- group: ${{ github.workflow }}-${{ github.event_name == 'merge_group' && format('merge-queue-{0}', github.ref) || format('{0}-{1}', github.ref, github.event.pull_request.number || github.sha) }}+ group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && format('dispatch-{0}', github.run_id) || github.event_name == 'merge_group' && format('merge-queue-{0}', github.ref) || format('{0}-{1}', github.ref, github.event.pull_request.number || github.sha) }}cancel-in-progress: ${{ github.event_name == 'pull_request' }}defaults:@@ -29,45 +41,30 @@ defaults:working-directory: autogpt_platform/frontendjobs:- setup:+ lint:runs-on: ubuntu-latest- outputs:- components-changed: ${{ steps.filter.outputs.components }}steps:- name: Checkout repositoryuses: actions/checkout@v6-- - name: Check for component changes- uses: dorny/paths-filter@v3- id: filterwith:- filters: |- components:- - 'autogpt_platform/frontend/src/components/**'+ ref: ${{ github.sha }}- - name: Set up pnpm- uses: pnpm/action-setup@v4- with:- package_json_file: autogpt_platform/frontend/package.json+ - name: Test coverage report validator+ run: >-+ python3 ../../.github/scripts/run_unittest_junit.py+ --start-directory ../../.github/scripts+ --pattern 'test_*cobertura*.py'+ --suite-name frontend-coverage-validator+ --output coverage-validator-tests.xml- - name: Set up Node- uses: actions/setup-node@v6+ - name: Upload coverage validator test results+ if: ${{ always() }}+ uses: actions/upload-artifact@v7with:- node-version: "24.18.0"- cache: "pnpm"- cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml-- - name: Install dependencies to populate cache- run: pnpm install --frozen-lockfile-- lint:- runs-on: ubuntu-latest- needs: setup-- steps:- - name: Checkout repository- uses: actions/checkout@v6+ name: frontend-coverage-validator-tests+ path: autogpt_platform/frontend/coverage-validator-tests.xml+ if-no-files-found: error- name: Set up pnpmuses: pnpm/action-setup@v4@@ -82,14 +79,13 @@ jobs:cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml- name: Install dependencies- run: pnpm install --frozen-lockfile+ run: pnpm install --frozen-lockfile --prefer-offline- name: Run lintrun: pnpm lintknip:runs-on: ubuntu-latest- needs: setuppermissions:contents: read@@ -127,12 +123,9 @@ jobs:chromatic:runs-on: ubuntu-latest- needs: setup- # Disabled: to re-enable, remove 'false &&' from the condition below- if: >-- false- && (github.ref == 'refs/heads/dev' || github.base_ref == 'dev')- && needs.setup.outputs.components-changed == 'true'+ # Disabled. Re-enable only for github.ref == 'refs/heads/dev' || github.base_ref == 'dev'+ # and a paths-filter match on autogpt_platform/frontend/src/components/**.+ if: ${{ false }}steps:- name: Checkout repository@@ -165,13 +158,18 @@ jobs:exitOnceUploaded: trueintegration_test:+ name: integration_test (shard ${{ matrix.shard }}/4)runs-on: ubuntu-latest- needs: setup+ strategy:+ fail-fast: false+ matrix:+ shard: [1, 2, 3, 4]steps:- name: Checkout repositoryuses: actions/checkout@v6with:+ ref: ${{ github.sha }}submodules: recursive- name: Set up pnpm@@ -187,18 +185,53 @@ jobs:cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml- name: Install dependencies- run: pnpm install --frozen-lockfile+ run: pnpm install --frozen-lockfile --prefer-offline- name: Generate API clientrun: pnpm generate:api- name: Run Integration Tests- run: pnpm test:unit+ run: >-+ pnpm exec vitest run+ --coverage+ --coverage.reportOnFailure+ --shard=${{ matrix.shard }}/4+ --reporter=default+ --reporter=github-actions+ --reporter=junit+ --outputFile.junit=./junit.xml++ - name: Validate JUnit test report+ if: ${{ always() }}+ run: >-+ python3 ../../.github/scripts/validate_junit.py+ --synthesize-invalid+ --require-no-skips+ junit.xml++ - name: Validate coverage report+ id: validate-frontend-coverage+ if: ${{ always() }}+ run: >-+ python3 ../../.github/scripts/validate_cobertura.py+ coverage/cobertura-coverage.xml++ - name: Upload machine-readable test results+ if: ${{ always() }}+ uses: actions/upload-artifact@v7+ with:+ name: frontend-integration-shard-${{ matrix.shard }}-of-4+ path: |+ autogpt_platform/frontend/junit.xml+ autogpt_platform/frontend/coverage/cobertura-coverage.xml+ if-no-files-found: error- name: Upload coverage reports to Codecov- if: ${{ !cancelled() }}+ if: ${{ !cancelled() && steps.validate-frontend-coverage.outcome == 'success' }}uses: codecov/codecov-action@v5with:token: ${{ secrets.CODECOV_TOKEN }}flags: platform-frontendfiles: ./autogpt_platform/frontend/coverage/cobertura-coverage.xml+ disable_search: true+ name: frontend-shard-${{ matrix.shard }}-of-4diff --git a/autogpt_platform/single-container/tests/test_ci_junit_reporters.py b/autogpt_platform/single-container/tests/test_ci_junit_reporters.pynew file mode 100644index 0000000000..2209c82a23--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_junit_reporters.py@@ -0,0 +1,423 @@+import runpy+import subprocess+import sys+import tempfile+import unittest+import xml.etree.ElementTree as ET+from pathlib import Path+++REPOSITORY_ROOT = Path(__file__).resolve().parents[3]+COMMAND_REPORTER = REPOSITORY_ROOT / ".github/scripts/run_command_junit.py"+UNITTEST_REPORTER = REPOSITORY_ROOT / ".github/scripts/run_unittest_junit.py"+++class XMLSafeTests(unittest.TestCase):+ def test_xml_character_range_boundaries(self):+ xml_safe = runpy.run_path(str(UNITTEST_REPORTER))["xml_safe"]+ allowed = "\t\n\r\x20\ud7ff\ue000\ufffd\U00010000\U0010ffff"+ forbidden = "\x00\x08\x0b\x0c\x0e\x1f\ud800\udfff\ufffe\uffff"+ self.assertEqual(xml_safe(allowed + forbidden), allowed)+ root = ET.Element("failure")+ root.text = xml_safe(allowed + forbidden)+ parsed = ET.fromstring(ET.tostring(root))+ self.assertEqual(parsed.text, allowed.replace("\r", "\n"))+++class CommandReporterTests(unittest.TestCase):+ def test_success_report_preserves_name_and_status(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ output = Path(temp_dir) / "command.xml"+ result = subprocess.run(+ [+ sys.executable,+ str(COMMAND_REPORTER),+ "--name",+ "safe <command>",+ "--classname",+ "ci.command",+ "--output",+ str(output),+ "--",+ sys.executable,+ "-c",+ "print('ok')",+ ],+ check=False,+ )++ self.assertEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["failures"], "0")+ self.assertEqual(suite.find("testcase").attrib["name"], "safe <command>")++ def test_failure_report_preserves_child_exit_status(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ output = Path(temp_dir) / "command.xml"+ result = subprocess.run(+ [+ sys.executable,+ str(COMMAND_REPORTER),+ "--name",+ "failure",+ "--classname",+ "ci.command",+ "--output",+ str(output),+ "--",+ sys.executable,+ "-c",+ "raise SystemExit(7)",+ ],+ check=False,+ )++ self.assertEqual(result.returncode, 7)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["failures"], "1")+ self.assertIsNotNone(suite.find("testcase/failure"))++ def test_spawn_error_is_recorded_as_machine_error(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ output = Path(temp_dir) / "command.xml"+ result = subprocess.run(+ [+ sys.executable,+ str(COMMAND_REPORTER),+ "--name",+ "spawn-error",+ "--classname",+ "ci.command",+ "--output",+ str(output),+ "--",+ str(Path(temp_dir) / "missing-command"),+ ],+ check=False,+ )++ self.assertNotEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["failures"], "0")+ self.assertEqual(suite.attrib["errors"], "1")+ self.assertIsNotNone(suite.find("testcase/error"))+++class ReportWriteFailureTests(unittest.TestCase):+ def test_report_write_errors_are_diagnostic_and_preserve_failure_status(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ root = Path(temp_dir)+ tests = root / "tests"+ tests.mkdir()+ (tests / "test_healthy.py").write_text(+ "import unittest\n"+ "class Healthy(unittest.TestCase):\n"+ " def test_passes(self): pass\n",+ encoding="utf-8",+ )+ existing_directory = root / "directory.xml"+ existing_directory.mkdir()+ existing_file = root / "not-a-directory"+ existing_file.write_text("occupied", encoding="utf-8")+ cases = [+ (+ "command-success",+ COMMAND_REPORTER,+ ["--name", "command", "--classname", "ci"],+ ["--", sys.executable, "-c", "pass"],+ 1,+ ),+ (+ "command-failure",+ COMMAND_REPORTER,+ ["--name", "command", "--classname", "ci"],+ ["--", sys.executable, "-c", "raise SystemExit(7)"],+ 7,+ ),+ (+ "command-spawn-error",+ COMMAND_REPORTER,+ ["--name", "command", "--classname", "ci"],+ ["--", str(root / "missing-command")],+ 127,+ ),+ (+ "unittest-success",+ UNITTEST_REPORTER,+ ["--start-directory", str(tests)],+ [],+ 1,+ ),+ (+ "unittest-discovery-error",+ UNITTEST_REPORTER,+ ["--start-directory", str(root / "missing-tests")],+ [],+ 1,+ ),+ ]+ for name, reporter, args, command, status in cases:+ for output in [existing_directory, existing_file / "report.xml"]:+ with self.subTest(name=name, output=str(output)):+ result = subprocess.run(+ [+ sys.executable,+ str(reporter),+ *args,+ "--output",+ str(output),+ *command,+ ],+ capture_output=True,+ text=True,+ check=False,+ )+ self.assertEqual(result.returncode, status, result.stderr)+ self.assertIn("Failed to write JUnit report", result.stderr)+ self.assertIn(str(output), result.stderr)+ self.assertNotIn("Traceback", result.stderr)+++class UnittestReporterTests(unittest.TestCase):+ def test_discovery_report_contains_every_test_id(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ test_directory = Path(temp_dir) / "tests"+ test_directory.mkdir()+ (test_directory / "test_sample.py").write_text(+ "import unittest\n"+ "class SampleTests(unittest.TestCase):\n"+ " def test_first(self):\n"+ " self.assertTrue(True)\n"+ " def test_second(self):\n"+ " self.assertEqual(2 + 2, 4)\n",+ encoding="utf-8",+ )+ output = Path(temp_dir) / "unittest.xml"++ result = subprocess.run(+ [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(test_directory),+ "--pattern",+ "test_*.py",+ "--output",+ str(output),+ ],+ check=False,+ )++ self.assertEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["tests"], "2")+ names = {case.attrib["name"] for case in suite.findall("testcase")}+ self.assertEqual(names, {"test_first", "test_second"})++ def test_failure_report_preserves_test_exit_status(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ test_directory = Path(temp_dir) / "tests"+ test_directory.mkdir()+ (test_directory / "test_failure.py").write_text(+ "import unittest\n"+ "class FailureTests(unittest.TestCase):\n"+ " def test_failure(self):\n"+ " self.fail('expected failure')\n",+ encoding="utf-8",+ )+ output = Path(temp_dir) / "unittest.xml"++ result = subprocess.run(+ [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(test_directory),+ "--output",+ str(output),+ ],+ check=False,+ )++ self.assertNotEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["failures"], "1")+ self.assertIsNotNone(suite.find("testcase/failure"))++ def test_class_fixture_error_is_recorded(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ test_directory = Path(temp_dir) / "tests"+ test_directory.mkdir()+ (test_directory / "test_fixture_error.py").write_text(+ "import unittest\n"+ "class FixtureErrorTests(unittest.TestCase):\n"+ " @classmethod\n"+ " def setUpClass(cls):\n"+ " raise RuntimeError('fixture failed')\n"+ " def test_never_runs(self):\n"+ " pass\n"+ "class Healthy(unittest.TestCase):\n"+ " def test_passes(self):\n"+ " self.assertTrue(True)\n",+ encoding="utf-8",+ )+ output = Path(temp_dir) / "unittest.xml"++ result = subprocess.run(+ [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(test_directory),+ "--output",+ str(output),+ ],+ check=False,+ )++ self.assertNotEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["tests"], "2")+ self.assertEqual(suite.attrib["errors"], "1")+ self.assertIn("fixture failed", suite.find("testcase/error").text)+ self.assertIn(+ "test_passes",+ {case.attrib["name"] for case in suite.findall("testcase")},+ )++ def test_zero_discovered_tests_is_a_machine_error(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ test_directory = Path(temp_dir) / "tests"+ test_directory.mkdir()+ output = Path(temp_dir) / "unittest.xml"++ result = subprocess.run(+ [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(test_directory),+ "--output",+ str(output),+ ],+ check=False,+ )++ self.assertNotEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["tests"], "1")+ self.assertEqual(suite.attrib["errors"], "1")++ def test_discovery_exception_is_a_machine_error(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ output = Path(temp_dir) / "unittest.xml"+ result = subprocess.run(+ [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(Path(temp_dir) / "missing"),+ "--output",+ str(output),+ ],+ check=False,+ )++ self.assertNotEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["errors"], "1")+ case = suite.find("testcase")+ self.assertEqual(case.attrib["name"], "test_discovery")+ self.assertIn("ImportError", case.find("error").text)++ def test_skip_is_failure_unless_exactly_allowlisted(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ test_directory = Path(temp_dir) / "tests"+ test_directory.mkdir()+ (test_directory / "test_skip.py").write_text(+ "import unittest\n"+ "class SkipTests(unittest.TestCase):\n"+ " @unittest.skip('missing optional tool')\n"+ " def test_optional(self):\n"+ " pass\n",+ encoding="utf-8",+ )+ rejected = Path(temp_dir) / "rejected.xml"+ allowed = Path(temp_dir) / "allowed.xml"+ command = [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(test_directory),+ "--output",+ ]++ rejected_result = subprocess.run(command + [str(rejected)], check=False)+ allowed_result = subprocess.run(+ command+ + [+ str(allowed),+ "--allow-skip",+ "test_skip.SkipTests.test_optional",+ ],+ check=False,+ )++ self.assertNotEqual(rejected_result.returncode, 0)+ rejected_suite = ET.parse(rejected).getroot()+ self.assertEqual(rejected_suite.attrib["failures"], "1")+ self.assertEqual(allowed_result.returncode, 0)+ allowed_suite = ET.parse(allowed).getroot()+ self.assertEqual(allowed_suite.attrib["skipped"], "1")++ other = Path(temp_dir) / "other.xml"+ other_result = subprocess.run(+ command+ + [str(other), "--allow-skip", "test_skip.SkipTests.test_other"],+ check=False,+ )+ self.assertNotEqual(other_result.returncode, 0)+ other_suite = ET.parse(other).getroot()+ self.assertEqual(other_suite.attrib["failures"], "1")+ self.assertEqual(other_suite.attrib["skipped"], "0")+ self.assertIn(+ "test_skip.SkipTests.test_optional",+ other_suite.find("testcase/failure").text,+ )++ def test_expected_failure_is_never_allowlisted_as_a_skip(self):+ with tempfile.TemporaryDirectory() as temp_dir:+ test_directory = Path(temp_dir) / "tests"+ test_directory.mkdir()+ (test_directory / "test_expected.py").write_text(+ "import unittest\n"+ "class ExpectedTests(unittest.TestCase):\n"+ " @unittest.expectedFailure\n"+ " def test_broken(self):\n"+ " self.fail('still broken')\n",+ encoding="utf-8",+ )+ output = Path(temp_dir) / "unittest.xml"++ result = subprocess.run(+ [+ sys.executable,+ str(UNITTEST_REPORTER),+ "--start-directory",+ str(test_directory),+ "--output",+ str(output),+ "--allow-skip",+ "test_expected.ExpectedTests.test_broken",+ ],+ check=False,+ )++ self.assertNotEqual(result.returncode, 0)+ suite = ET.parse(output).getroot()+ self.assertEqual(suite.attrib["failures"], "1")+ self.assertEqual(suite.attrib["skipped"], "0")+++if __name__ == "__main__":+ unittest.main()diff --git a/autogpt_platform/single-container/tests/test_ci_junit_skips.py b/autogpt_platform/single-container/tests/test_ci_junit_skips.pynew file mode 100644index 0000000000..2ee5d5e2cc--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_junit_skips.py@@ -0,0 +1,172 @@+import subprocess+import sys+import tempfile+import textwrap+import unittest+import xml.etree.ElementTree as ET+from pathlib import Path+++REPORTER = Path(__file__).resolve().parents[3] / ".github/scripts/run_unittest_junit.py"+++class UnittestSkipReportingTests(unittest.TestCase):+ def run_reporter(self, source):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ tests = root / "tests"+ tests.mkdir()+ (tests / "test_healthy.py").write_text(+ "import unittest\n"+ "class Healthy(unittest.TestCase):\n"+ " def test_passes(self): pass\n",+ encoding="utf-8",+ )+ (tests / "test_skipped.py").write_text(+ textwrap.dedent(source), encoding="utf-8"+ )+ output = root / "junit.xml"+ result = subprocess.run(+ [+ sys.executable,+ str(REPORTER),+ "--start-directory",+ str(tests),+ "--output",+ str(output),+ ],+ capture_output=True,+ text=True,+ check=False,+ )+ return result, ET.parse(output).getroot()++ def assert_skip_rejected(self, source, expected_name):+ result, report = self.run_reporter(source)+ self.assertNotEqual(result.returncode, 0, result.stderr)+ rejected = [+ case+ for case in report.findall("testcase")+ if case.find("failure") is not None+ ]+ self.assertEqual(len(rejected), 1)+ self.assertIn(expected_name, rejected[0].get("name"))+ self.assertIn("disallowed skip", rejected[0].find("failure").text)+ self.assertIn("required service unavailable", rejected[0].find("failure").text)+ self.assertEqual(report.get("failures"), "1")++ def test_class_fixture_skip_is_reported_and_rejected(self):+ self.assert_skip_rejected(+ """+ import unittest+ class Required(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ raise unittest.SkipTest('required service unavailable')+ def test_required(self):+ self.fail('must not execute after skipped setup')+ """,+ "Required",+ )++ def test_module_fixture_skip_is_reported_and_rejected(self):+ self.assert_skip_rejected(+ """+ import unittest+ def setUpModule():+ raise unittest.SkipTest('required service unavailable')+ class Required(unittest.TestCase):+ def test_required(self):+ self.fail('must not execute after skipped setup')+ """,+ "setUpModule",+ )++ def test_subtest_skip_is_reported_and_rejected(self):+ self.assert_skip_rejected(+ """+ import unittest+ class Required(unittest.TestCase):+ def test_required(self):+ with self.subTest(service='database'):+ self.skipTest('required service unavailable')+ with self.subTest(service='queue'):+ self.assertTrue(True)+ """,+ "test_required (service='database')",+ )++ def test_subtest_skip_cannot_replace_an_assertion_failure(self):+ result, report = self.run_reporter(+ """+ import unittest+ class Required(unittest.TestCase):+ def test_required(self):+ with self.subTest(service='database'):+ self.fail('database assertion failed')+ with self.subTest(service='queue'):+ self.skipTest('required service unavailable')+ """+ )+ self.assertNotEqual(result.returncode, 0)+ failures = report.findall("testcase/failure")+ self.assertEqual(len(failures), 2)+ self.assertTrue(+ any("database assertion failed" in case.text for case in failures)+ )+ self.assertTrue(any("disallowed skip" in case.text for case in failures))++ def test_repeated_subtest_failures_and_errors_are_all_reported(self):+ result, report = self.run_reporter(+ """+ import unittest+ class Required(unittest.TestCase):+ def test_required(self):+ with self.subTest(service='database'):+ self.fail('database assertion failed')+ with self.subTest(service='queue'):+ self.fail('queue assertion failed')+ with self.subTest(service='storage'):+ raise RuntimeError('storage unavailable')+ with self.subTest(service='optional'):+ self.skipTest('required service unavailable')+ """+ )+ self.assertNotEqual(result.returncode, 0)+ cases = report.findall("testcase")+ self.assertEqual(report.get("tests"), str(len(cases)))+ self.assertEqual(report.get("failures"), "3")+ self.assertEqual(report.get("errors"), "1")+ failures = report.findall("testcase/failure")+ self.assertEqual(len(failures), 3)+ for message in [+ "database assertion failed",+ "queue assertion failed",+ "disallowed skip",+ ]:+ self.assertTrue(any(message in failure.text for failure in failures))+ self.assertIn("storage unavailable", report.find("testcase/error").text)+ self.assertEqual(len({case.get("name") for case in cases}), len(cases))++ def test_cleanup_error_preserves_subtest_failure(self):+ result, report = self.run_reporter(+ """+ import unittest+ class Required(unittest.TestCase):+ def cleanup(self):+ raise RuntimeError('cleanup failed')+ def test_required(self):+ self.addCleanup(self.cleanup)+ with self.subTest(service='database'):+ self.fail('database assertion failed')+ """+ )+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(report.get("failures"), "1")+ self.assertEqual(report.get("errors"), "1")+ self.assertIn("database assertion failed", report.find("testcase/failure").text)+ self.assertIn("cleanup failed", report.find("testcase/error").text)+++if __name__ == "__main__":+ unittest.main()