Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retain completed backend children before siblings finish

pulse-triage[bot]10cd740593fe
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 3bc64b8091..fc8ca94abe 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -1708,6 +1708,19 @@ to this module. Incomplete JSON execution or failed children cannot be hidden
by another passing command. The existing artifacts retain every task's exit and
duration alongside the unchanged slice coverage. These are four package workers,
not extra hosted jobs; they do not establish a hosted latency improvement.
+The queue atomically retains its planned inventory before submission and each
+returned child result as it arrives, without waiting behind a slower first child.
+`ordinary-packages.json` keeps completed observations in source order and names
+unobserved packages separately. `collection_complete=false` is incomplete
+collection, not a liveness or passing verdict. An interruption may leave that
+snapshot; it cannot erase already retained sibling results or invent outcomes
+for the remaining work. A process-start failure records `started=false` and a
+null exit, not a fabricated shell exit. The existing always-run coverage artifact
+upload retains these snapshots; upload and hosted interruption recovery remain
+separate observations. All children must still return zero for the check to pass.
+`run_go_package_slice_test.py` drives the actual queue with a held first child,
+an early exit-nine child and a successful sibling, checking readable incomplete
+snapshots, source-ordered final results and an unchanged failing aggregate.
Documentation-only changes retain the original explicit
runtime-inapplicable results and required check names.
diff --git a/scripts/release_control/run_go_package_slice_test.py b/scripts/release_control/run_go_package_slice_test.py
index c7b854b9d1..f3d03b50fd 100644
--- a/scripts/release_control/run_go_package_slice_test.py
+++ b/scripts/release_control/run_go_package_slice_test.py
@@ -1,14 +1,19 @@
"""Exercise the actual sliced Go runner, recorder and required-check verdict."""
import copy
+import contextlib
import importlib.util
+import io
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
+import sys
import tempfile
+import threading
+import time
import unittest
from unittest.mock import patch
@@ -460,7 +465,75 @@ class WorkflowTest(unittest.TestCase):
with patch.object(shard.subprocess, 'run', return_value=subprocess.CompletedProcess([],0,'\n'.join(inventory),'')):
with self.assertRaises(ValueError): shard.ordinary_packages(0)
with patch.object(shard.subprocess,'run',side_effect=OSError('start failed')):
- self.assertEqual(shard.run_task(('internal/a',['not-a-command']))['exit'],127)
+ observation = shard.run_task(('internal/a',['not-a-command']))
+ self.assertIsNone(observation['exit'])
+ self.assertFalse(observation['started'])
+ self.assertIn('start_error_errno', observation)
+
+ def test_completed_children_are_retained_before_a_slow_first_child_returns(self):
+ spec = importlib.util.spec_from_file_location('progress_rest_shard', SHARD)
+ shard = importlib.util.module_from_spec(spec); spec.loader.exec_module(shard)
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ output, started, release = root / 'output', root / 'started', root / 'release'
+ names = [runner.MODULE + 'internal/' + name for name in ('held', 'failed', 'passed')]
+ held = '''from pathlib import Path
+import sys,time
+started,release=map(Path,sys.argv[1:])
+started.write_text('started')
+deadline=time.monotonic()+10
+while not release.exists() and time.monotonic()<deadline: time.sleep(.01)
+sys.exit(0 if release.exists() else 13)
+'''
+ admitted = [(names[0], [sys.executable, '-I', '-S', '-c', held, str(started), str(release)]),
+ (names[1], [sys.executable, '-I', '-S', '-c', 'raise SystemExit(9)']),
+ (names[2], [sys.executable, '-I', '-S', '-c', 'raise SystemExit(0)'])]
+ terminal, errors = [], []
+ def execute():
+ try:
+ terminal.append(shard.main())
+ except BaseException as error:
+ errors.append(error)
+ argv = [str(SHARD), '--index', '0', '--output', str(output),
+ '--ordinary-command', shard.ORDINARY_COMMAND]
+ with patch.object(shard, 'ordinary_packages', return_value=names), \
+ patch.object(shard, 'tasks', return_value=admitted), \
+ patch.object(sys, 'stdin', io.StringIO('\n'.join(names) + '\n')), \
+ patch.object(sys, 'argv', argv), contextlib.redirect_stdout(io.StringIO()):
+ worker = threading.Thread(target=execute)
+ worker.start()
+ before = None
+ try:
+ deadline = time.monotonic() + 5
+ while time.monotonic() < deadline:
+ receipt = output / 'ordinary-packages.json'
+ if receipt.exists():
+ # A partial JSON write is a failure, not a poll to
+ # retry until a favourable snapshot appears.
+ before = json.loads(receipt.read_text())
+ if len(before['tasks']) == 2 and started.exists():
+ break
+ time.sleep(.01)
+ self.assertIsNotNone(before, 'no result retained while the first child waited')
+ self.assertEqual(before['planned_packages'], names)
+ self.assertFalse(before['collection_complete'])
+ self.assertEqual(before['unobserved_packages'], names[:1])
+ self.assertEqual([task['package'] for task in before['tasks']], names[1:])
+ self.assertEqual([task['exit'] for task in before['tasks']], [9, 0])
+ self.assertTrue(all(task['started'] for task in before['tasks']))
+ self.assertEqual(terminal, [], 'partial results became an aggregate verdict')
+ finally:
+ release.write_text('release')
+ worker.join(timeout=12)
+ self.assertFalse(worker.is_alive(), 'owned test scheduler did not finish')
+ self.assertEqual(errors, [])
+ self.assertEqual(terminal, [1], 'later successful work hid a failed child')
+ final = json.loads((output / 'ordinary-packages.json').read_text())
+ self.assertTrue(final['collection_complete'])
+ self.assertEqual(final['unobserved_packages'], [])
+ self.assertEqual([task['package'] for task in final['tasks']], names)
+ self.assertEqual([task['exit'] for task in final['tasks']], [0, 9, 0])
+ self.assertEqual(list(output.glob('.ordinary-packages-*')), [])
def test_helpers_are_part_of_the_exact_main_controls_copied_to_patches(self):
stable_patch = (ROOT / 'scripts/release_control/stable_patch.py').read_text()
diff --git a/scripts/release_control/run_go_rest_shard.py b/scripts/release_control/run_go_rest_shard.py
index adc738baaa..c3b1ef6720 100644
--- a/scripts/release_control/run_go_rest_shard.py
+++ b/scripts/release_control/run_go_rest_shard.py
@@ -10,12 +10,13 @@ the source-derived ordinary package inventory.
from __future__ import annotations
import argparse
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
import json
import os
from pathlib import Path
import subprocess
import sys
+import tempfile
import time
@@ -69,14 +70,40 @@ def run_task(task: tuple[str, list[str]]) -> dict:
env["pkgs"] = name
env["GOFLAGS"] = env.get("GOFLAGS", "") + " -p=1"
result = subprocess.run(command, cwd=ROOT, env=env)
- code = result.returncode
+ observation = {"started": True, "exit": result.returncode}
except OSError as error:
print(f"Could not start backend package {name}: {error}", file=sys.stderr)
- code = 127
- return {"package": name, "exit": code,
+ # No process exit was observed. In particular, do not manufacture the
+ # shell's command-not-found exit for a process that never started.
+ observation = {"started": False, "exit": None, "start_error_errno": error.errno}
+ return {"package": name, **observation,
"seconds": round(time.monotonic() - start, 3)}
+def retain_results(output: Path, index: int, planned: list[str], results: dict,
+ start: float, *, complete: bool = False) -> None:
+ # Record only returned observations, in the original source order. A
+ # planned/unobserved task is not evidence of execution or current liveness.
+ receipt = {"schema_version": 1, "index": index, "slots": SLOTS,
+ "planned_packages": planned, "collection_complete": complete,
+ "unobserved_packages": [name for name in planned if name not in results],
+ "tasks": [results[name] for name in planned if name in results],
+ "seconds": round(time.monotonic() - start, 3)}
+ # Readers and the existing always-run artifact upload must see either the
+ # previous complete JSON snapshot or this one, never a truncated overwrite.
+ temporary = None
+ try:
+ with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output,
+ prefix=".ordinary-packages-", delete=False) as stream:
+ temporary = Path(stream.name)
+ json.dump(receipt, stream, indent=2)
+ stream.write("\n")
+ temporary.replace(output / "ordinary-packages.json")
+ finally:
+ if temporary is not None:
+ temporary.unlink(missing_ok=True)
+
+
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--index", type=int, choices=(0, 1), required=True)
@@ -99,14 +126,20 @@ def main() -> int:
args.output.mkdir(parents=True, exist_ok=True)
print("Ordinary packages in this shard:\n" + "\n".join(packages), flush=True)
start = time.monotonic()
- # Drain every admitted task even when a sibling fails. Executor.map retains
- # source scheduling order in the receipt, not concurrent completion order.
+ admitted = tasks(packages, args.index, args.output, args.ordinary_command)
+ planned = [name for name, _ in admitted]
+ results = {}
+ retain_results(args.output, args.index, planned, results, start)
+ # Drain every admitted task even when a sibling fails. Collect in completion
+ # order so a slow first task cannot hide already returned results. Retained
+ # snapshots still use source order, including the final legacy tasks list.
with ThreadPoolExecutor(max_workers=SLOTS) as pool:
- results = list(pool.map(run_task, tasks(packages, args.index, args.output, args.ordinary_command)))
- receipt = {"index": args.index, "slots": SLOTS, "tasks": results,
- "seconds": round(time.monotonic() - start, 3)}
- (args.output / "ordinary-packages.json").write_text(json.dumps(receipt, indent=2) + "\n")
- return 0 if all(result["exit"] == 0 for result in results) else 1
+ futures = {pool.submit(run_task, task): task[0] for task in admitted}
+ for future in as_completed(futures):
+ results[futures[future]] = future.result()
+ retain_results(args.output, args.index, planned, results, start)
+ retain_results(args.output, args.index, planned, results, start, complete=True)
+ return 0 if all(result["exit"] == 0 for result in results.values()) else 1
if __name__ == "__main__":