Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bound notification CI destination reads and retain their failure

pulse-triage[bot]a4b7679cafb6
diff --git a/scripts/check_notification_runtime.py b/scripts/check_notification_runtime.py
index 3efe994f71..715d193f3d 100644
--- a/scripts/check_notification_runtime.py
+++ b/scripts/check_notification_runtime.py
@@ -28,6 +28,10 @@ PUBLIC_URL = 'https://notification-runtime.example.test'
LIMIT = 2 * 1024 * 1024
# prepareWebhookData's ordinary event; resolution is a different path.
ORDINARY_EVENT = 'alert'
+# The owned application must finish one small loopback request within this
+# bound. An idle socket timeout alone still permits an endless header/body
+# trickle, and server_close() waits for every non-daemon handler.
+SINK_REQUEST_TIMEOUT_SECONDS = 2
class Failure(RuntimeError):
@@ -52,10 +56,32 @@ class Sink:
self.error = None
owner = self
+ class Server(http.server.ThreadingHTTPServer):
+ daemon_threads = False
+
+ def finish_request(self, connection, client_address):
+ def expire():
+ owner.reject('TimeoutError')
+ try:
+ connection.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+
+ deadline = threading.Timer(SINK_REQUEST_TIMEOUT_SECONDS, expire)
+ deadline.start()
+ try:
+ super().finish_request(connection, client_address)
+ finally:
+ deadline.cancel()
+ deadline.join()
+
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, *_):
pass
+ def log_error(self, *_):
+ owner.reject('HTTPError')
+
def do_POST(self):
try:
size = int(self.headers.get('Content-Length', '-1'))
@@ -72,17 +98,27 @@ class Sink:
self.end_headers()
self.wfile.write(b'OK')
except Exception as exc:
- owner.error = type(exc).__name__
- self.send_response(400)
- self.send_header('Content-Length', '0')
- self.end_headers()
-
- self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler)
- self.server.daemon_threads = False
+ owner.reject(type(exc).__name__)
+ # A timed-out/closed peer need not receive a response.
+ # Keep the failure, but never print its payload or an
+ # incidental broken-pipe traceback during teardown.
+ try:
+ self.send_response(400)
+ self.send_header('Content-Length', '0')
+ self.end_headers()
+ except OSError:
+ pass
+
+ self.server = Server(('127.0.0.1', 0), Handler)
self.port = self.server.server_port
self.thread = threading.Thread(target=self.server.serve_forever, name='owned-webhook')
self.thread.start()
+ def reject(self, kind):
+ with self.lock:
+ if self.error is None:
+ self.error = kind
+
def snapshot(self):
with self.lock:
return list(self.rows)
@@ -340,7 +376,8 @@ def main():
binary = args.binary.resolve(strict=True)
binding = source_binding(binary, args.expected_source_sha)
report = {'source_binding': binding, 'kind': 'offline-development-source-notification-runtime',
- 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'outcome': 'failed'}
+ 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'outcome': 'failed',
+ 'sink_request_timeout_seconds': SINK_REQUEST_TIMEOUT_SECONDS}
with tempfile.TemporaryDirectory(prefix='pulse-notification-runtime-') as temporary:
data = Path(temporary)
sink = Sink()
@@ -360,6 +397,12 @@ def main():
report['sink_thread_joined_and_listener_closed'] = False
report['cleanup_error'] = type(exc).__name__
report['outcome'] = 'failed'
+ # A destination failure cannot disappear behind a later matching
+ # delivery or a successful resource cleanup. Keep those facts
+ # separate, without retaining request/exception text.
+ if sink.error is not None:
+ report['sink_error'] = sink.error
+ report['outcome'] = 'failed'
if journey:
report.update(phases=journey.phases, child_exits=journey.exits,
requests=journey.requests, incomplete_request=journey.active_request,
diff --git a/scripts/tests/test_notification_runtime.py b/scripts/tests/test_notification_runtime.py
index 1b9b5ba40a..438ea86f83 100644
--- a/scripts/tests/test_notification_runtime.py
+++ b/scripts/tests/test_notification_runtime.py
@@ -1,10 +1,17 @@
#!/usr/bin/env python3
-"""Controls for the source-bound ordinary notification journey, without a server."""
+"""Controls for the source-bound notification journey and its owned loopback sink."""
import contextlib
import copy
+import hashlib
+import http.client
import importlib.util
import io
+import json
from pathlib import Path
+import socket
+import sys
+import tempfile
+import threading
from types import SimpleNamespace
import unittest
from unittest import mock
@@ -190,5 +197,176 @@ class NotificationRuntimeControls(unittest.TestCase):
self.assertNotIn('go build', section)
+class NotificationSinkControls(unittest.TestCase):
+ def test_real_request_preserves_payload_and_closes_every_owned_thread(self):
+ sink = RUNTIME.Sink()
+ payload = b'{"event":"alert","resourceName":"owned.example.test"}'
+ client = http.client.HTTPConnection('127.0.0.1', sink.port, timeout=3)
+ try:
+ client.request('POST', '/owned-alert', body=payload)
+ response = client.getresponse()
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.read(), b'OK')
+ self.assertEqual(sink.snapshot(), [{'body': json.loads(payload),
+ 'bytes': len(payload), 'sha256': hashlib.sha256(payload).hexdigest(),
+ 'path': '/owned-alert'}])
+ finally:
+ client.close()
+ sink.close()
+ self.assertIsNone(sink.error)
+ self.assertFalse(sink.thread.is_alive())
+ self.assertFalse(any(thread.is_alive() for thread in sink.server._threads))
+
+ def stalled_request(self, prefix, trickle=False):
+ # Exercise the real parser, socket and non-daemon request thread. Use
+ # a shorter instance of the fixed timer, not a fake socket/clock.
+ with mock.patch.object(RUNTIME, 'SINK_REQUEST_TIMEOUT_SECONDS', .25):
+ sink = RUNTIME.Sink()
+ accepted = threading.Event()
+ closed = threading.Event()
+ stop_writes = threading.Event()
+ errors = []
+ original = sink.server.finish_request
+
+ def observe(*args):
+ accepted.set()
+ original(*args)
+
+ def close():
+ try:
+ sink.close()
+ except Exception as exc:
+ errors.append(type(exc).__name__)
+ finally:
+ closed.set()
+
+ sink.server.finish_request = observe
+ peer = socket.create_connection(('127.0.0.1', sink.port), timeout=2)
+ closer = None
+ writer = None
+ bounded = False
+ try:
+ peer.sendall(prefix)
+ self.assertTrue(accepted.wait(2), 'loopback request was not accepted')
+ if trickle:
+ def write():
+ while not stop_writes.wait(.02):
+ try:
+ peer.sendall(b' ')
+ except OSError:
+ break
+ writer = threading.Thread(target=write, name='owned-fixture-trickle')
+ writer.start()
+ closer = threading.Thread(target=close, name='owned-fixture-close')
+ closer.start()
+ bounded = closed.wait(2)
+ finally:
+ # A broken regression must still finish the fixture, not hang
+ # its caller. Release the peer even when bounded closure failed.
+ stop_writes.set()
+ try:
+ peer.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+ peer.close()
+ if writer:
+ writer.join(timeout=2)
+ if closer:
+ closer.join(timeout=2)
+ else:
+ sink.close()
+ self.assertTrue(bounded, 'owned sink cleanup waited for the stalled peer')
+ self.assertEqual(errors, [])
+ self.assertFalse(closer.is_alive())
+ self.assertFalse(writer and writer.is_alive())
+ self.assertFalse(any(thread.is_alive() for thread in sink.server._threads))
+ self.assertEqual(sink.error, 'TimeoutError')
+ self.assertEqual(sink.snapshot(), [])
+
+ def test_partial_headers_cannot_block_cleanup(self):
+ self.stalled_request(b'POST /owned-alert HTTP/1.0\r\nContent-Length:')
+
+ def test_partial_body_cannot_block_cleanup(self):
+ self.stalled_request(b'POST /owned-alert HTTP/1.0\r\nContent-Length: 100\r\n\r\n{')
+
+ def test_body_trickle_cannot_reset_the_absolute_deadline(self):
+ self.stalled_request(b'POST /owned-alert HTTP/1.0\r\nContent-Length: 10000\r\n\r\n{',
+ trickle=True)
+
+ def test_invalid_http_records_closed_failure_without_payload_output(self):
+ sink = RUNTIME.Sink()
+ client = http.client.HTTPConnection('127.0.0.1', sink.port, timeout=3)
+ try:
+ with contextlib.redirect_stderr(io.StringIO()) as errors:
+ client.request('PRIVATE_PAYLOAD', '/owned-alert')
+ response = client.getresponse()
+ self.assertEqual(response.status, 501)
+ response.read()
+ client.close()
+ sink.close()
+ self.assertEqual(errors.getvalue(), '')
+ finally:
+ client.close()
+ sink.close()
+ self.assertEqual(sink.error, 'HTTPError')
+ self.assertEqual(sink.snapshot(), [])
+
+ def test_invalid_json_rejects_without_printing_its_private_body(self):
+ sink = RUNTIME.Sink()
+ client = http.client.HTTPConnection('127.0.0.1', sink.port, timeout=3)
+ try:
+ with contextlib.redirect_stderr(io.StringIO()) as errors:
+ client.request('POST', '/owned-alert', body=b'PRIVATE_SYNTHETIC_BODY')
+ response = client.getresponse()
+ self.assertEqual(response.status, 400)
+ self.assertEqual(response.read(), b'')
+ client.close()
+ sink.close()
+ self.assertEqual(errors.getvalue(), '')
+ finally:
+ client.close()
+ sink.close()
+ self.assertEqual(sink.error, 'JSONDecodeError')
+ self.assertEqual(sink.snapshot(), [])
+
+ def test_first_destination_failure_is_not_replaced(self):
+ sink = RUNTIME.Sink()
+ try:
+ sink.reject('TimeoutError')
+ sink.reject('Failure')
+ self.assertEqual(sink.error, 'TimeoutError')
+ finally:
+ sink.close()
+
+ def test_terminal_receipt_distinguishes_sink_failure_from_completed_cleanup(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ binary = root / 'owned-binary'
+ binary.write_bytes(b'synthetic binary identity')
+ receipt = root / 'receipt.json'
+ sink = SimpleNamespace(error='TimeoutError', close=mock.Mock())
+ journey = SimpleNamespace(run=mock.Mock(), phases=[], exits=[], requests=[],
+ active_request=None, deliveries=[], delivery_observations=[])
+ args = ['check_notification_runtime.py', '--binary', str(binary),
+ '--expected-source-sha', SHA, '--receipt', str(receipt)]
+ with mock.patch.object(sys, 'argv', args), \
+ mock.patch.object(RUNTIME, 'source_binding', return_value={'checkout_sha': SHA}), \
+ mock.patch.object(RUNTIME, 'Sink', return_value=sink), \
+ mock.patch.object(RUNTIME, 'Journey', return_value=journey), \
+ mock.patch.object(RUNTIME.subprocess, 'run', return_value=SimpleNamespace(returncode=0)), \
+ contextlib.redirect_stdout(io.StringIO()):
+ self.assertEqual(RUNTIME.main(), 1)
+ report = json.loads(receipt.read_text())
+ self.assertEqual(report['outcome'], 'failed')
+ self.assertEqual(report['sink_error'], 'TimeoutError')
+ self.assertEqual(report['sink_request_timeout_seconds'], 2)
+ self.assertTrue(report['sink_thread_joined_and_listener_closed'])
+ self.assertTrue(report['owned_temporary_data_removed'])
+ self.assertTrue(report['source_clean_after'])
+ self.assertNotIn('cleanup_error', report)
+ sink.close.assert_called_once()
+ journey.run.assert_called_once()
+
+
if __name__ == '__main__':
unittest.main(verbosity=2)