AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
ci(backend): let secret-gated skips pass on fork PRs (#14606)
Reinier van der Leercb1d3bb0c752
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.jsonindex 39c4cc232d..ca0440b1c7 100644--- a/.github/scripts/backend-allowed-skips.json+++ b/.github/scripts/backend-allowed-skips.json@@ -141,5 +141,10 @@"3.11": [],"3.12": [],"3.13": []+ },+ "secret_gated": {+ "OPENAI_API_KEY": [+ "backend.copilot.tools.run_agent_test.test_run_agent_with_llm_credentials"+ ]}}diff --git a/.github/scripts/test_validate_backend_skips.py b/.github/scripts/test_validate_backend_skips.pyindex a02ea58b21..79b045fbc0 100644--- a/.github/scripts/test_validate_backend_skips.py+++ b/.github/scripts/test_validate_backend_skips.py@@ -166,6 +166,15 @@ class AggregateSkipTests(unittest.TestCase):self.assertEqual(result, 1)self.assertIn("unapproved skipped test IDs", diagnostic)+ def test_secret_gated_skip_passes_whether_or_not_the_run_had_the_secret(self):+ self.policy["secret_gated"] = {+ "EXAMPLE_API_KEY": ["backend.example.test_needs_secret"]+ }+ self.write_report("3.11", "data", ["test_needs_secret"])+ self.assertEqual(self.run_validator()[0], 0)+ self.write_report("3.11", "data", [])+ self.assertEqual(self.run_validator()[0], 0)+def test_aggregate_validation_requires_explicit_versions(self):self.policy = ["backend.example.test_shared"]result, diagnostic = self.run_validator()@@ -205,6 +214,27 @@ class AggregateSkipTests(unittest.TestCase):{"common": ["backend.test"], "python_versions": {"3.11": ["backend.test"]}},{"common": [], "python_versions": {"3.11": [""]}},{"common": [], "python_versions": {"3.11": []}, "typo": []},+ {"common": [], "python_versions": {"3.11": []}, "secret_gated": []},+ {+ "common": [],+ "python_versions": {"3.11": []},+ "secret_gated": {"lowercase_key": ["backend.test"]},+ },+ {+ "common": ["backend.test"],+ "python_versions": {"3.11": []},+ "secret_gated": {"EXAMPLE_API_KEY": ["backend.test"]},+ },+ {+ "common": [],+ "python_versions": {"3.11": ["backend.test"]},+ "secret_gated": {"EXAMPLE_API_KEY": ["backend.test"]},+ },+ {+ "common": [],+ "python_versions": {"3.11": []},+ "secret_gated": {"EXAMPLE_API_KEY": ["backend.test", "backend.test"]},+ },)for policy in invalid:with self.subTest(policy=policy):@@ -292,6 +322,13 @@ class WorkflowSkipPolicyTests(unittest.TestCase):self.assertIn("reports may be incomplete", run.stderr)self.assertFalse(invocation.exists())+ def test_every_secret_gated_variable_reaches_the_test_job_as_a_secret(self):+ policy = load_skip_policy(self.github / "scripts/backend-allowed-skips.json")+ self.assertTrue(policy.secret_gated)+ environment = self.workflow["jobs"]["test"]["env"]+ for variable in policy.secret_gated:+ self.assertEqual(environment.get(variable), "${{ secrets.%s }}" % variable)+def test_new_helper_and_test_changes_trigger_backend_ci(self):events = self.workflow["on"] if "on" in self.workflow else self.workflow[True]for event in ("push", "pull_request"):diff --git a/.github/scripts/test_validate_junit.py b/.github/scripts/test_validate_junit.pyindex dc66e2daf2..e6c8b4f3c2 100644--- a/.github/scripts/test_validate_junit.py+++ b/.github/scripts/test_validate_junit.py@@ -1,4 +1,5 @@import json+import osimport tempfileimport unittestfrom pathlib import Path@@ -147,6 +148,44 @@ class ValidateJUnitTests(unittest.TestCase):self.assertIn(str(allowlist), output)self.assertIn("do not allowlist a regression", output)+ def test_secret_gated_skip_needs_its_variable_absent_or_empty(self):+ with tempfile.TemporaryDirectory() as directory:+ report = Path(directory) / "skipped.xml"+ allowlist = Path(directory) / "allowed.json"+ root = report_xml(skipped=1)+ for case in root.iter("testcase"):+ case.set("classname", "backend.example")+ ElementTree.ElementTree(root).write(report)+ allowlist.write_text(+ json.dumps(+ {+ "common": [],+ "python_versions": {"3.13": []},+ "secret_gated": {+ "EXAMPLE_API_KEY": ["backend.example.skipped-0"]+ },+ }+ )+ )+ args = [+ "--allow-skips-from",+ str(allowlist),+ "--python-version",+ "3.13",+ str(report),+ ]+ # A fork run renders an unset secret as the empty string rather than+ # dropping the variable, so both spellings have to be tolerated.+ for value, expected in (({}, 0), ({"EXAMPLE_API_KEY": ""}, 0)):+ with self.subTest(environment=value):+ with patch.dict(os.environ, value, clear=True):+ self.assertEqual(main(args), expected)+ with patch.dict(os.environ, {"EXAMPLE_API_KEY": "present"}, clear=True):+ with patch("sys.stderr") as stderr:+ self.assertEqual(main(args), 1)+ output = "".join(call.args[0] for call in stderr.write.call_args_list)+ self.assertIn("backend.example.skipped-0", output)+def test_skip_allowlist_must_be_a_valid_list_of_exact_ids(self):self._check_invalid_allowlists()diff --git a/.github/scripts/validate_backend_skips.py b/.github/scripts/validate_backend_skips.pyindex 112f79c5a6..32729d045e 100644--- a/.github/scripts/validate_backend_skips.py+++ b/.github/scripts/validate_backend_skips.py@@ -55,10 +55,14 @@ def main(argv: list[str] | None = None) -> int:f"unexpected: {sorted(actual - expected)}")problems = []+ # No secrets in this job, so secret-gated IDs are accepted and left out+ # of the staleness sweep; the per-shard step is the strict check.for version in policy.python_versions:allowed = policy.for_python_version(version)try:- observed = validate_shard_reports(args.reports_dir, version, allowed)+ observed = validate_shard_reports(+ args.reports_dir, version, allowed | policy.secret_gated_ids()+ )if stale := allowed - observed:raise ValueError(f"Python {version} has stale skip IDs in {args.allow_skips_from}: "diff --git a/.github/scripts/validate_junit.py b/.github/scripts/validate_junit.pyindex efaf563b21..7728d69db5 100644--- a/.github/scripts/validate_junit.py+++ b/.github/scripts/validate_junit.py@@ -2,8 +2,10 @@import argparseimport json+import osimport sys-from dataclasses import dataclass+from collections.abc import Mapping+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom xml.etree import ElementTree@@ -24,6 +26,7 @@ class JUnitSummary:class SkipPolicy:common: frozenset[str]python_versions: dict[str, frozenset[str]] | None = None+ secret_gated: Mapping[str, frozenset[str]] = field(default_factory=dict)def for_python_version(self, version: str | None) -> set[str]:if self.python_versions is None:@@ -34,6 +37,19 @@ class SkipPolicy:)return set(self.common | self.python_versions[version])+ def secret_gated_ids(self, env: Mapping[str, str] | None = None) -> set[str]:+ """IDs a missing repository secret is allowed to skip.++ Given ``env``, only the IDs whose variable is absent there; without it,+ all of them, for callers that cannot see the secrets themselves.+ """+ return {+ skip_id+ for variable, skip_ids in self.secret_gated.items()+ if env is None or not env.get(variable)+ for skip_id in skip_ids+ }+def _skip_ids(entries: object) -> frozenset[str]:if (@@ -53,12 +69,37 @@ def _skip_ids(entries: object) -> frozenset[str]:return frozenset(entries)+def _secret_gated(+ entries: object, unconditional: frozenset[str]+) -> dict[str, frozenset[str]]:+ if not isinstance(entries, dict):+ raise ValueError("secret_gated must map environment variables to skip IDs")+ gated = {}+ for variable, skip_ids in entries.items():+ if not (variable.isidentifier() and variable.isupper()):+ raise ValueError(f"invalid secret_gated environment variable: {variable!r}")+ gated[variable] = _skip_ids(skip_ids)+ if repeated := unconditional & gated[variable]:+ raise ValueError(+ f"{variable} repeats unconditionally allowed skip IDs: "+ f"{', '.join(sorted(repeated))}"+ )+ return gated++def load_skip_policy(path: Path) -> SkipPolicy:entries = json.loads(path.read_text(encoding="utf-8"))if isinstance(entries, list):return SkipPolicy(_skip_ids(entries))- if not isinstance(entries, dict) or set(entries) != {"common", "python_versions"}:- raise ValueError("skip policy must contain common and python_versions")+ if not isinstance(entries, dict) or not (+ {"common", "python_versions"}+ <= set(entries)+ <= {"common", "python_versions", "secret_gated"}+ ):+ raise ValueError(+ "skip policy must contain common and python_versions, "+ "and may contain secret_gated"+ )common = _skip_ids(entries["common"])versions = entries["python_versions"]if not isinstance(versions, dict) or not versions:@@ -76,7 +117,13 @@ def load_skip_policy(path: Path) -> SkipPolicy:f"Python {version} repeats common skip IDs: {', '.join(sorted(duplicate))}")version_skips[version] = specific- return SkipPolicy(common, version_skips)+ return SkipPolicy(+ common,+ version_skips,+ _secret_gated(+ entries.get("secret_gated", {}), common.union(*version_skips.values())+ ),+ )def _declared_count(element: ElementTree.Element, key: str, label: str) -> int | None:@@ -226,9 +273,10 @@ def main(argv: list[str] | None = None) -> int:allowed_skips = Noneif args.allow_skips_from:try:- allowed_skips = load_skip_policy(args.allow_skips_from).for_python_version(+ policy = load_skip_policy(args.allow_skips_from)+ allowed_skips = policy.for_python_version(args.python_version- )+ ) | policy.secret_gated_ids(os.environ)except (OSError, ValueError) as exc:print(f"Invalid skip allowlist: {exc}", file=sys.stderr)return 1