Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Run complete documentation safety suites on docs-only changes

pulse-triage[bot]54f40a6a7634
diff --git a/.github/workflows/public-docs.yml b/.github/workflows/public-docs.yml
index 700293558e..c5ef3af498 100644
--- a/.github/workflows/public-docs.yml
+++ b/.github/workflows/public-docs.yml
@@ -7,23 +7,11 @@ on:
- "docs/**"
- "frontend-modern/public/docs/**"
- ".github/ISSUE_TEMPLATE/**"
+ - ".github/PULL_REQUEST_TEMPLATE.md"
- "scripts/check_public_docs.py"
- "scripts/check_docs_mirror.py"
- "scripts/readme_entrypoints_test.py"
- - "scripts/tests/run.sh"
- - "scripts/tests/test_public_docs_workflow.py"
- - "scripts/tests/test_production_rollout_docs.py"
- - "scripts/tests/test_pmg_docs.py"
- - "scripts/tests/test_agent_profile_docs.py"
- - "scripts/tests/test_pbs_docs.py"
- - "scripts/tests/test_audit_recovery_docs.py"
- - "scripts/tests/test_notification_troubleshooting_docs.py"
- - "scripts/tests/test_webhook_verification_docs.py"
- - "scripts/tests/test_pve_backup_troubleshooting_docs.py"
- - "scripts/tests/test_vm_disk_diagnostics.py"
- - "scripts/tests/test_memory_troubleshooting_docs.py"
- - "scripts/tests/test_performance_troubleshooting_docs.py"
- - "scripts/tests/test_troubleshooting_logs.py"
+ - "scripts/tests/**"
- "scripts/test-vm-disk.sh"
- ".github/workflows/public-docs.yml"
push:
@@ -34,23 +22,11 @@ on:
- "docs/**"
- "frontend-modern/public/docs/**"
- ".github/ISSUE_TEMPLATE/**"
+ - ".github/PULL_REQUEST_TEMPLATE.md"
- "scripts/check_public_docs.py"
- "scripts/check_docs_mirror.py"
- "scripts/readme_entrypoints_test.py"
- - "scripts/tests/run.sh"
- - "scripts/tests/test_public_docs_workflow.py"
- - "scripts/tests/test_production_rollout_docs.py"
- - "scripts/tests/test_pmg_docs.py"
- - "scripts/tests/test_agent_profile_docs.py"
- - "scripts/tests/test_pbs_docs.py"
- - "scripts/tests/test_audit_recovery_docs.py"
- - "scripts/tests/test_notification_troubleshooting_docs.py"
- - "scripts/tests/test_webhook_verification_docs.py"
- - "scripts/tests/test_pve_backup_troubleshooting_docs.py"
- - "scripts/tests/test_vm_disk_diagnostics.py"
- - "scripts/tests/test_memory_troubleshooting_docs.py"
- - "scripts/tests/test_performance_troubleshooting_docs.py"
- - "scripts/tests/test_troubleshooting_logs.py"
+ - "scripts/tests/**"
- "scripts/test-vm-disk.sh"
- ".github/workflows/public-docs.yml"
@@ -62,7 +38,30 @@ concurrency:
cancel-in-progress: true
jobs:
+ recipes:
+ name: Documentation recipes (${{ matrix.shard }})
+ runs-on: ubuntu-24.04
+ timeout-minutes: 10
+ strategy:
+ fail-fast: false
+ matrix:
+ shard: [0, 1, 2, 3]
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ # Build and Test skips smoke for docs-only changes. Discover the complete
+ # current docs suite here, including privacy, authentication and recovery.
+ # Isolated shards retain all recipe deadlines without a longer CI path.
+ - name: Exercise documentation safety and diagnostic recipes
+ run: scripts/tests/run.sh --docs-shard "${{ matrix.shard }}"
+
+ # Preserve the existing required check name and fail closed on every shard.
check:
+ needs: recipes
+ if: ${{ always() }}
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
@@ -71,6 +70,15 @@ jobs:
with:
persist-credentials: false
+ - name: Require every documentation recipe shard
+ env:
+ RECIPE_RESULT: ${{ needs.recipes.result }}
+ run: |
+ if [[ "$RECIPE_RESULT" != success ]]; then
+ echo "Documentation recipe shards did not all succeed: $RECIPE_RESULT" >&2
+ exit 1
+ fi
+
- name: Validate public documentation
run: python3 scripts/check_public_docs.py
@@ -82,22 +90,3 @@ jobs:
- name: Check documentation safety workflow contract
run: python3 scripts/tests/test_public_docs_workflow.py
-
- # Build and Test intentionally skips its smoke suite for docs-only
- # changes. Exercise the existing safety guidance and copied recipes here
- # too, without installing frontend/Go dependencies or calling providers.
- - name: Exercise backup safety and diagnostic recipes
- run: |
- scripts/tests/run.sh \
- test_production_rollout_docs.py \
- test_pmg_docs.py \
- test_agent_profile_docs.py \
- test_pbs_docs.py \
- test_audit_recovery_docs.py \
- test_notification_troubleshooting_docs.py \
- test_webhook_verification_docs.py \
- test_pve_backup_troubleshooting_docs.py \
- test_vm_disk_diagnostics.py \
- test_memory_troubleshooting_docs.py \
- test_performance_troubleshooting_docs.py \
- test_troubleshooting_logs.py
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index b066faa26c..9717c10c24 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -10,20 +10,30 @@ TEST_DIR="${ROOT_DIR}/scripts/tests"
usage() {
cat <<'EOF'
Usage: scripts/tests/run.sh [test-script ...]
+ scripts/tests/run.sh --docs-shard <0|1|2|3>
Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subset
-when specified.
+when specified. Documentation CI discovers every documentation suite and runs
+one of four disjoint, sorted shards; it does not maintain a second suite list.
EOF
}
discover_tests() {
local -n ref=$1
+ local mode="${2:-all}"
local inventory
+ local -a patterns=(-name 'test-*.sh' -o -name 'test_*.py')
+ if [[ "${mode}" == docs ]]; then
+ # Keep the established safety helpers whose names predate the docs suffix.
+ patterns=(-name 'test_*docs*.py' -o -name 'test_vm_disk_diagnostics.py'
+ -o -name 'test_troubleshooting_logs.py'
+ -o -name 'test-retired-trial-acquisition-docs.sh')
+ fi
# Process substitution hides producer failures from mapfile. Admit the
# complete discovery pipeline before running any tests, never a partial list.
if ! inventory="$(
find "${TEST_DIR}" -maxdepth 1 -type f \
- \( -name 'test-*.sh' -o -name 'test_*.py' \) | sort
+ \( "${patterns[@]}" \) | LC_ALL=C sort
)"; then
echo "Failed to discover the complete smoke test inventory under ${TEST_DIR}" >&2
return 1
@@ -99,7 +109,21 @@ main() {
fi
local -a tests=()
- if [[ $# -gt 0 ]]; then
+ if [[ "${1:-}" == --docs-shard ]]; then
+ if [[ $# -ne 2 || ! "${2:-}" =~ ^[0-3]$ ]]; then
+ echo "Usage: scripts/tests/run.sh --docs-shard <0|1|2|3>" >&2
+ return 1
+ fi
+ local -a doc_tests=()
+ if ! discover_tests doc_tests docs; then
+ return 1
+ fi
+ local index
+ for ((index = $2; index < ${#doc_tests[@]}; index += 4)); do
+ tests+=("${doc_tests[index]}")
+ done
+ printf 'Documentation shard %s/4: %s of %s suites\n' "$2" "${#tests[@]}" "${#doc_tests[@]}"
+ elif [[ $# -gt 0 ]]; then
local arg resolved
for arg in "$@"; do
if ! resolved="$(resolve_test_path "${arg}")"; then
diff --git a/scripts/tests/test_public_docs_workflow.py b/scripts/tests/test_public_docs_workflow.py
index 76e6e8441b..411f722e17 100644
--- a/scripts/tests/test_public_docs_workflow.py
+++ b/scripts/tests/test_public_docs_workflow.py
@@ -1,13 +1,13 @@
#!/usr/bin/env python3
-"""Docs-only CI must exercise safety guidance, not just its Markdown shape.
+"""Docs-only CI must execute every existing safety suite and fail closed.
-Read the canonical block-YAML surface (also checked by workflow trust), then
-execute its actual smoke command with disposable test adapters. No provider,
-host service, backup or guest-agent operation is used.
+Exercise the actual workflow commands with disposable test adapters. Real
+recipes run separately in the source-proof VM, never against customer services.
"""
from __future__ import annotations
+import fnmatch
import os
from pathlib import Path
import re
@@ -21,21 +21,38 @@ import unittest
ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".github/workflows/public-docs.yml"
-SAFETY_STEP = "Exercise backup safety and diagnostic recipes"
-TESTS = (
- "test_production_rollout_docs.py",
- "test_pmg_docs.py",
- "test_agent_profile_docs.py",
- "test_pbs_docs.py",
- "test_audit_recovery_docs.py",
- "test_notification_troubleshooting_docs.py",
- "test_webhook_verification_docs.py",
- "test_pve_backup_troubleshooting_docs.py",
- "test_vm_disk_diagnostics.py",
- "test_memory_troubleshooting_docs.py",
- "test_performance_troubleshooting_docs.py",
- "test_troubleshooting_logs.py",
+SAFETY_STEP = "Exercise documentation safety and diagnostic recipes"
+VERDICT_STEP = "Require every documentation recipe shard"
+# Independent witnesses: all twelve formerly covered suites, plus the critical
+# privacy/access/recovery checks that the former literal workflow omitted.
+REQUIRED_SUITES = (
+ "test_production_rollout_docs.py", "test_pmg_docs.py",
+ "test_agent_profile_docs.py", "test_pbs_docs.py", "test_audit_recovery_docs.py",
+ "test_notification_troubleshooting_docs.py", "test_webhook_verification_docs.py",
+ "test_pve_backup_troubleshooting_docs.py", "test_vm_disk_diagnostics.py",
+ "test_memory_troubleshooting_docs.py", "test_performance_troubleshooting_docs.py",
+ "test_troubleshooting_logs.py", "test_admin_docs.py", "test_api_auth_docs.py",
+ "test_authentication_diagnostic_docs.py", "test_update_recovery_docs.py",
+ "test_guest_disk_reading_docs.py", "test_guest_network_troubleshooting_docs.py",
+ "test_truenas_docs.py", "test_repo_docs_link_drift.py", "test_security_docs.py",
)
+LEGACY_SUITES = {
+ "test_vm_disk_diagnostics.py", "test_troubleshooting_logs.py",
+ "test-retired-trial-acquisition-docs.sh",
+}
+
+
+def documentation_suites() -> list[str]:
+ return sorted(path.name for path in (ROOT / "scripts/tests").iterdir()
+ if path.is_file() and (fnmatch.fnmatch(path.name, "test_*docs*.py")
+ or path.name in LEGACY_SUITES))
+
+
+def job_body(source: str, name: str) -> str:
+ marker = f"\n {name}:\n"
+ if source.count(marker) != 1:
+ raise AssertionError(f"expected exactly one {name} job")
+ return re.split(r"\n [\w-]+:\n", source.split(marker, 1)[1], maxsplit=1)[0]
def step_body(source: str, name: str) -> str:
@@ -45,11 +62,19 @@ def step_body(source: str, name: str) -> str:
return source.split(marker, 1)[1].split("\n - name:", 1)[0]
-def safety_command(source: str) -> str:
- body = step_body(source, SAFETY_STEP)
+def safety_command(source: str, shard: int) -> str:
+ body = step_body(job_body(source, "recipes"), SAFETY_STEP)
+ run = re.findall(r"^ run: (.+)$", body, re.MULTILINE)
+ if len(run) != 1:
+ raise AssertionError("expected one literal docs-shard command")
+ return run[0].replace("${{ matrix.shard }}", str(shard))
+
+
+def verdict_command(source: str) -> str:
+ body = step_body(job_body(source, "check"), VERDICT_STEP)
marker = " run: |\n"
if body.count(marker) != 1:
- raise AssertionError("expected one literal safety command")
+ raise AssertionError("expected one literal verdict command")
return textwrap.dedent(body.split(marker, 1)[1]).strip()
@@ -57,15 +82,14 @@ class PublicDocsWorkflowTest(unittest.TestCase):
def setUp(self):
self.source = WORKFLOW.read_text(encoding="utf-8")
- def test_both_events_cover_docs_checks_runner_and_helper(self):
+ def test_both_events_cover_all_suite_and_helper_changes(self):
triggers = self.source.split("\non:\n", 1)[1].split("\npermissions:", 1)[0]
required = {
"*.md", "docs/**", "frontend-modern/public/docs/**",
+ ".github/ISSUE_TEMPLATE/**", ".github/PULL_REQUEST_TEMPLATE.md",
"scripts/check_public_docs.py", "scripts/check_docs_mirror.py",
- "scripts/readme_entrypoints_test.py", "scripts/tests/run.sh",
- "scripts/tests/test_public_docs_workflow.py", "scripts/test-vm-disk.sh",
- ".github/workflows/public-docs.yml",
- *(f"scripts/tests/{name}" for name in TESTS),
+ "scripts/readme_entrypoints_test.py", "scripts/tests/**",
+ "scripts/test-vm-disk.sh", ".github/workflows/public-docs.yml",
}
for event in ("pull_request", "push"):
with self.subTest(event=event):
@@ -76,15 +100,16 @@ class PublicDocsWorkflowTest(unittest.TestCase):
self.assertEqual(len(paths), len(set(paths)))
self.assertIn(" push:\n branches:\n - main\n", triggers)
- def test_exact_existing_suites_run_without_dependency_setup(self):
- command = safety_command(self.source).replace("\\\n", " ")
- self.assertEqual(shlex.split(command), ["scripts/tests/run.sh", *TESTS])
- for name in TESTS:
- self.assertTrue((ROOT / "scripts/tests" / name).is_file())
+ def test_shards_discover_current_suites_without_dependency_setup(self):
+ self.assertTrue(set(REQUIRED_SUITES).issubset(documentation_suites()))
+ for shard in range(4):
+ self.assertEqual(shlex.split(safety_command(self.source, shard)),
+ ["scripts/tests/run.sh", "--docs-shard", str(shard)])
self.assertNotIn("actions/setup-", self.source)
self.assertNotRegex(self.source, r"\b(?:npm|pip|go) (?:ci|install|build|test)\b")
def test_existing_document_admissions_are_preserved(self):
+ check = job_body(self.source, "check")
for name, command in (
("Validate public documentation", "python3 scripts/check_public_docs.py"),
("Check getting-started safety and current plans", "python3 scripts/readme_entrypoints_test.py"),
@@ -92,49 +117,77 @@ class PublicDocsWorkflowTest(unittest.TestCase):
("Check documentation safety workflow contract", "python3 scripts/tests/test_public_docs_workflow.py"),
):
with self.subTest(step=name):
- self.assertIn(f" run: {command}\n", step_body(self.source, name))
-
- def test_safety_checks_cannot_be_conditionally_skipped_or_softened(self):
- self.assertNotRegex(self.source, r"(?m)^\s*(?:if|continue-on-error):")
+ self.assertIn(f" run: {command}\n", step_body(check, name))
+
+ def test_all_four_isolated_shards_are_mandatory_and_bounded(self):
+ recipes = job_body(self.source, "recipes")
+ self.assertIn(" shard: [0, 1, 2, 3]\n", recipes)
+ self.assertIn(" fail-fast: false\n", recipes)
+ self.assertNotRegex(recipes, r"(?m)^\s*(?:if|continue-on-error|exclude):")
+ self.assertNotIn("continue-on-error:", self.source)
+ for name in ("recipes", "check"):
+ job = job_body(self.source, name)
+ self.assertIn(" runs-on: ubuntu-24.04\n", job)
+ self.assertIn(" timeout-minutes: 10\n", job)
+ self.assertIn(" persist-credentials: false\n", job)
self.assertIn("permissions:\n contents: read\n", self.source)
- self.assertIn(" persist-credentials: false\n", self.source)
- self.assertIn(" timeout-minutes: 10\n", self.source)
- def run_adapters(self, failing: str = ""):
+ def test_required_check_runs_even_when_shards_fail_or_are_skipped(self):
+ check = job_body(self.source, "check")
+ self.assertIn(" needs: recipes\n", check)
+ self.assertIn(" if: ${{ always() }}\n", check)
+ self.assertNotRegex(check.replace(" if: ${{ always() }}\n", ""), r"(?m)^\s*if:")
+ self.assertIn(" RECIPE_RESULT: ${{ needs.recipes.result }}\n", check)
+ self.assertLess(check.index(VERDICT_STEP), check.index("Validate public documentation"))
+ for state in ("success", "failure", "cancelled", "skipped", "", "unknown"):
+ with self.subTest(state=state):
+ result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],
+ env=dict(os.environ, RECIPE_RESULT=state), text=True,
+ capture_output=True, timeout=5)
+ self.assertEqual(result.returncode == 0, state == "success", result.stdout + result.stderr)
+
+ def run_adapters(self, shard: int, failing: str = ""):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
tests = root / "scripts/tests"
tests.mkdir(parents=True)
shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")
(tests / "run.sh").chmod(0o700)
- for name in TESTS:
- (tests / name).write_text(
- "import os\nfrom pathlib import Path\n"
- f"with Path(os.environ['CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"
- f"raise SystemExit({1 if name == failing else 0})\n",
- encoding="utf-8",
- )
+ for name in documentation_suites():
+ status = 1 if name == failing else 0
+ if name.endswith(".sh"):
+ code = f'#!/bin/bash\nprintf "%s\\n" {shlex.quote(name)} >> "$CALLS"\nexit {status}\n'
+ else:
+ code = ("import os\nfrom pathlib import Path\n"
+ f"with Path(os.environ['CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"
+ f"raise SystemExit({status})\n")
+ (tests / name).write_text(code, encoding="utf-8")
+ (tests / name).chmod(0o700)
calls = root / "calls"
- result = subprocess.run(
- ["bash", "-e", "-o", "pipefail", "-c", safety_command(self.source)],
- cwd=root, env=dict(os.environ, CALLS=str(calls)),
- text=True, capture_output=True, timeout=15,
- )
- return result, calls.read_text().splitlines()
-
- def test_actual_workflow_command_runs_every_suite(self):
- result, calls = self.run_adapters()
- self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
- self.assertEqual(calls, list(TESTS))
- self.assertIn(f"Summary: {len(TESTS)}/{len(TESTS)} passed", result.stdout)
-
- def test_each_failed_suite_is_a_failed_workflow_command(self):
- for failing in TESTS:
+ result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", safety_command(self.source, shard)],
+ cwd=root, env=dict(os.environ, CALLS=str(calls)),
+ text=True, capture_output=True, timeout=15)
+ return result, calls.read_text().splitlines() if calls.exists() else []
+
+ def test_actual_workflow_commands_cover_every_suite_once(self):
+ seen = []
+ for shard in range(4):
+ result, calls = self.run_adapters(shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(calls, documentation_suites()[shard::4])
+ self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)
+ seen.extend(calls)
+ self.assertEqual(sorted(seen), documentation_suites())
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_each_failed_suite_fails_its_actual_workflow_command(self):
+ for index, failing in enumerate(documentation_suites()):
with self.subTest(failing=failing):
- result, calls = self.run_adapters(failing)
+ shard = index % 4
+ result, calls = self.run_adapters(shard, failing)
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
- self.assertEqual(calls, list(TESTS))
- self.assertIn(f"Summary: {len(TESTS)-1}/{len(TESTS)} passed", result.stdout)
+ self.assertEqual(calls, documentation_suites()[shard::4])
+ self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)
self.assertIn("Failures: 1", result.stdout)
diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.py
index c8d78e4feb..10a0ff9339 100644
--- a/scripts/tests/test_smoke_runner.py
+++ b/scripts/tests/test_smoke_runner.py
@@ -163,6 +163,74 @@ class SmokeRunnerTest(unittest.TestCase):
self.assertEqual(self.observed_calls(), ["test_first.py", "test-second.sh"])
self.assertIn("Summary: 1/2 passed", result.stdout)
+ def test_docs_shards_cover_current_and_future_suites_once_without_unrelated_tests(self):
+ names = ["test_admin_docs.py", "test_api_auth_docs.py", "test_docs_reader.py",
+ "test_future_docs.py", "test_recovery_docs.py", "test_security_docs.py",
+ "test_troubleshooting_logs.py", "test_vm_disk_diagnostics.py"]
+ for name in names:
+ self.python_fixture(name)
+ names.append("test-retired-trial-acquisition-docs.sh")
+ self.shell_fixture(names[-1])
+ self.python_fixture("test_backend.py", 7)
+ self.shell_fixture("test-agent-runtime.sh", 7)
+ nested = self.tests / "nested"
+ nested.mkdir()
+ (nested / "test_nested_docs.py").write_text("raise SystemExit(7)\n")
+ seen = []
+ for shard in range(4):
+ before = len(self.observed_calls())
+ result = self.run_runner("--docs-shard", shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ calls = self.observed_calls()[before:]
+ self.assertEqual(calls, sorted(names)[shard::4])
+ seen.extend(calls)
+ self.assertIn(f"Documentation shard {shard}/4:", result.stdout)
+ self.assertEqual(sorted(seen), sorted(names))
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_every_doc_failure_remains_gating_and_collects_the_remaining_shard(self):
+ names = [f"test_{index}_docs.py" for index in range(8)]
+ for index, name in enumerate(names):
+ self.python_fixture(name, 7 if index < 4 else 0)
+ for shard in range(4):
+ before = len(self.observed_calls())
+ result = self.run_runner("--docs-shard", shard)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(self.observed_calls()[before:], names[shard::4])
+ self.assertIn("Summary: 1/2 passed", result.stdout)
+ self.assertIn("Failures: 1", result.stdout)
+
+ def test_docs_discovery_rejects_failed_find_and_sort_before_execution(self):
+ passing = self.python_fixture("test_first_docs.py")
+ self.python_fixture("test_second_docs.py", 7)
+ for name in ("find", "sort"):
+ with self.subTest(producer=name):
+ self.producer(name, 23, [passing])
+ self.assert_discovery_rejected(self.run_runner("--docs-shard", 0))
+ (self.bin / name).unlink()
+
+ def test_invalid_doc_shard_arguments_stop_before_discovery_or_execution(self):
+ self.python_fixture("test_first_docs.py")
+ self.producer("find", 23)
+ for args in ((), ("-1",), ("4",), ("00",), ("one",), ("0", "test_first_docs.py")):
+ with self.subTest(args=args):
+ result = self.run_runner("--docs-shard", *args)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Usage:", result.stderr)
+ self.assertNotIn("Failed to discover", result.stderr)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_empty_doc_inventory_or_shard_is_not_a_pass(self):
+ self.python_fixture("test_unrelated.py")
+ empty = self.run_runner("--docs-shard", 0)
+ self.assertNotEqual(empty.returncode, 0)
+ self.assertIn("No tests found", empty.stderr)
+ self.python_fixture("test_first_docs.py")
+ empty = self.run_runner("--docs-shard", 3)
+ self.assertNotEqual(empty.returncode, 0)
+ self.assertIn("No tests found", empty.stderr)
+ self.assertEqual(self.observed_calls(), [])
+
if __name__ == "__main__":
unittest.main()