Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(telemetry): unified system event stream with correlation

Juan Denis3d3975fa3a50
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 58967dcf..44cf6dd0 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -354,6 +354,10 @@ def create_app(config_name=None):
from app.api.queue_bus import queue_bus_bp
app.register_blueprint(queue_bus_bp, url_prefix='/api/v1/queue')
+ # Register blueprints - Telemetry / System Event Stream
+ from app.api.telemetry import telemetry_bp
+ app.register_blueprint(telemetry_bp, url_prefix='/api/v1/telemetry')
+
# Register blueprints - Agent Pairing (RustDesk-style short-code flow)
from app.api.pairing import pairing_bp
app.register_blueprint(pairing_bp, url_prefix='/api/v1/pairing')
diff --git a/backend/app/api/telemetry.py b/backend/app/api/telemetry.py
new file mode 100644
index 00000000..4f425171
--- /dev/null
+++ b/backend/app/api/telemetry.py
@@ -0,0 +1,150 @@
+"""Telemetry / system event stream API."""
+from datetime import datetime, timedelta
+
+from flask import Blueprint, request, jsonify
+from flask_jwt_extended import jwt_required, get_jwt_identity
+from sqlalchemy import func
+
+from app import db
+from app.middleware.rbac import admin_required
+from app.models.system_event import SystemEvent
+from app.services.telemetry_service import TelemetryService
+
+telemetry_bp = Blueprint('telemetry', __name__)
+
+
+@telemetry_bp.route('/events', methods=['GET'])
+@jwt_required()
+def list_events():
+ """List system events with filtering and pagination."""
+ page = request.args.get('page', 1, type=int)
+ per_page = request.args.get('per_page', 50, type=int)
+ source = request.args.get('source') or None
+ event_type = request.args.get('event_type') or None
+ severity = request.args.get('severity') or None
+ resource_type = request.args.get('resource_type') or None
+ resource_id = request.args.get('resource_id') or None
+ correlation_id = request.args.get('correlation_id') or None
+ message_query = request.args.get('q') or None
+
+ start_date = _parse_iso_datetime(request.args.get('start_date'))
+ end_date = _parse_iso_datetime(request.args.get('end_date'))
+
+ pagination = TelemetryService.get_events(
+ page=page,
+ per_page=min(per_page, 200),
+ source=source,
+ event_type=event_type,
+ severity=severity,
+ resource_type=resource_type,
+ resource_id=resource_id,
+ correlation_id=correlation_id,
+ start_date=start_date,
+ end_date=end_date,
+ message_query=message_query,
+ )
+
+ return jsonify({
+ 'events': [event.to_dict(include_payload=True) for event in pagination.items],
+ 'total': pagination.total,
+ 'pages': pagination.pages,
+ 'page': pagination.page,
+ 'per_page': pagination.per_page,
+ }), 200
+
+
+@telemetry_bp.route('/events/<int:event_id>', methods=['GET'])
+@jwt_required()
+def get_event(event_id):
+ """Get a single system event by ID."""
+ event = TelemetryService.get_event_by_id(event_id)
+ if not event:
+ return jsonify({'error': 'Event not found'}), 404
+ return jsonify(event.to_dict(include_payload=True)), 200
+
+
+@telemetry_bp.route('/events/by-correlation/<correlation_id>', methods=['GET'])
+@jwt_required()
+def get_events_by_correlation(correlation_id):
+ """Get all events sharing a correlation ID."""
+ limit = request.args.get('limit', 100, type=int)
+ events = TelemetryService.get_events_by_correlation(correlation_id, limit=limit)
+ return jsonify({
+ 'correlation_id': correlation_id,
+ 'events': [event.to_dict(include_payload=True) for event in events],
+ }), 200
+
+
+@telemetry_bp.route('/stats', methods=['GET'])
+@jwt_required()
+def get_stats():
+ """Get telemetry aggregate stats."""
+ hours = request.args.get('hours', 24, type=int)
+ source = request.args.get('source') or None
+ stats = TelemetryService.get_stats(hours=hours, source=source)
+ stats['recent_event_types'] = TelemetryService.get_recent_event_types()
+ return jsonify(stats), 200
+
+
+@telemetry_bp.route('/sources', methods=['GET'])
+@jwt_required()
+def list_sources():
+ """Return all distinct event sources."""
+ sources = (db.session.query(SystemEvent.source)
+ .distinct()
+ .order_by(SystemEvent.source)
+ .all())
+ return jsonify({'sources': [s[0] for s in sources if s[0]]}), 200
+
+
+@telemetry_bp.route('/event-types', methods=['GET'])
+@jwt_required()
+def list_event_types():
+ """Return all distinct event types, optionally filtered by source."""
+ source = request.args.get('source') or None
+ query = db.session.query(SystemEvent.event_type).distinct()
+ if source:
+ query = query.filter(SystemEvent.source == source)
+ types = query.order_by(SystemEvent.event_type).all()
+ return jsonify({'event_types': [t[0] for t in types if t[0]]}), 200
+
+
+@telemetry_bp.route('/events', methods=['DELETE'])
+@jwt_required()
+@admin_required
+def cleanup_events():
+ """Delete events older than the specified number of days (admin only)."""
+ days = request.args.get('days', 90, type=int)
+ deleted = TelemetryService.cleanup_old_events(days=days)
+ return jsonify({'deleted': deleted}), 200
+
+
+@telemetry_bp.route('/events/test', methods=['POST'])
+@jwt_required()
+@admin_required
+def emit_test_event():
+ """Emit a test telemetry event (admin only)."""
+ data = request.get_json(silent=True) or {}
+ user_id = get_jwt_identity()
+ event = TelemetryService.emit(
+ source=data.get('source', 'system'),
+ event_type=data.get('event_type', 'telemetry.test'),
+ message=data.get('message', 'Test telemetry event'),
+ severity=data.get('severity', 'info'),
+ actor_user_id=user_id,
+ payload=data.get('payload', {'test': True}),
+ commit=True,
+ )
+ if event:
+ return jsonify(event.to_dict(include_payload=True)), 201
+ return jsonify({'error': 'Failed to emit test event'}), 500
+
+
+def _parse_iso_datetime(value):
+ """Parse an ISO datetime string, returning None on failure."""
+ if not value:
+ return None
+ try:
+ return datetime.fromisoformat(value.replace('Z', '+00:00')).replace(tzinfo=None)
+ except (ValueError, TypeError):
+ return None
diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py
index 5660cda9..0ad55943 100644
--- a/backend/app/models/__init__.py
+++ b/backend/app/models/__init__.py
@@ -45,6 +45,7 @@ from app.models.waf_policy import WafPolicy
from app.queue_bus.models import QueueGroup, Queue, QueueMessage
from app.notifications.models import Notification, NotificationDelivery
from app.models.email_provider import EmailProviderConnection
+from app.models.system_event import SystemEvent
__all__ = [
'User', 'Application', 'Domain', 'EnvironmentVariable', 'EnvironmentVariableHistory',
@@ -76,4 +77,5 @@ __all__ = [
'QueueGroup', 'Queue', 'QueueMessage',
'Notification', 'NotificationDelivery',
'EmailProviderConnection',
+ 'SystemEvent',
]
diff --git a/backend/app/models/deployment_job.py b/backend/app/models/deployment_job.py
index 570626db..f4268745 100644
--- a/backend/app/models/deployment_job.py
+++ b/backend/app/models/deployment_job.py
@@ -34,6 +34,9 @@ class DeploymentJob(db.Model):
result = db.Column(db.Text, default='{}')
error_message = db.Column(db.Text, nullable=True)
+ # Correlation ID for grouping this deployment with related telemetry events.
+ correlation_id = db.Column(db.String(64), nullable=True, index=True)
+
created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
started_at = db.Column(db.DateTime, nullable=True)
completed_at = db.Column(db.DateTime, nullable=True)
@@ -99,6 +102,7 @@ class DeploymentJob(db.Model):
'progress_percent': self.progress_percent,
'result': self.get_result(),
'error_message': self.error_message,
+ 'correlation_id': self.correlation_id,
'created_at': self.created_at.isoformat() if self.created_at else None,
'started_at': self.started_at.isoformat() if self.started_at else None,
'completed_at': self.completed_at.isoformat() if self.completed_at else None,
diff --git a/backend/app/models/event_subscription.py b/backend/app/models/event_subscription.py
index 9d4b98da..668574ca 100644
--- a/backend/app/models/event_subscription.py
+++ b/backend/app/models/event_subscription.py
@@ -107,6 +107,9 @@ class EventDelivery(db.Model):
created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
duration_ms = db.Column(db.Float, nullable=True)
+ # Correlation ID for grouping this webhook delivery with related telemetry events.
+ correlation_id = db.Column(db.String(64), nullable=True, index=True)
+
subscription = db.relationship('EventSubscription', back_populates='deliveries')
STATUS_PENDING = 'pending'
@@ -138,6 +141,7 @@ class EventDelivery(db.Model):
'delivered_at': self.delivered_at.isoformat() if self.delivered_at else None,
'created_at': self.created_at.isoformat() if self.created_at else None,
'duration_ms': self.duration_ms,
+ 'correlation_id': self.correlation_id,
}
def __repr__(self):
diff --git a/backend/app/models/system_event.py b/backend/app/models/system_event.py
new file mode 100644
index 00000000..498d6894
--- /dev/null
+++ b/backend/app/models/system_event.py
@@ -0,0 +1,119 @@
+"""System-wide telemetry event model.
+
+A single, queryable event stream that records what happened, when, and to which
+resource across ServerKit. Existing domain-specific tables (audit_logs,
+event_deliveries, notification_deliveries, metrics_history, etc.) remain the
+authoritative sources; SystemEvent is a correlation layer on top.
+"""
+import json
+from datetime import datetime
+
+from app import db
+
+
+class SystemEvent(db.Model):
+ """A single event in the unified telemetry stream."""
+
+ __tablename__ = 'system_events'
+
+ # Severity levels
+ SEVERITY_DEBUG = 'debug'
+ SEVERITY_INFO = 'info'
+ SEVERITY_WARNING = 'warning'
+ SEVERITY_ERROR = 'error'
+ SEVERITY_CRITICAL = 'critical'
+ VALID_SEVERITIES = [
+ SEVERITY_DEBUG,
+ SEVERITY_INFO,
+ SEVERITY_WARNING,
+ SEVERITY_ERROR,
+ SEVERITY_CRITICAL,
+ ]
+
+ # Common event sources
+ SOURCE_AUDIT = 'audit'
+ SOURCE_BACKUP = 'backup'
+ SOURCE_DEPLOYMENT = 'deployment'
+ SOURCE_MONITORING = 'monitoring'
+ SOURCE_NOTIFICATION = 'notification'
+ SOURCE_WEBHOOK = 'webhook'
+ SOURCE_WORKFLOW = 'workflow'
+ SOURCE_EVENT = 'event'
+
+ id = db.Column(db.Integer, primary_key=True)
+
+ # When the event occurred (may differ from created_at if backdated)
+ timestamp = db.Column(db.DateTime, nullable=False, default=datetime.utcnow, index=True)
+
+ # Categorization
+ source = db.Column(db.String(50), nullable=False, index=True)
+ event_type = db.Column(db.String(100), nullable=False, index=True)
+ severity = db.Column(db.String(20), nullable=False, default=SEVERITY_INFO, index=True)
+
+ # Resource scoping
+ resource_type = db.Column(db.String(50), nullable=True, index=True)
+ resource_id = db.Column(db.String(100), nullable=True, index=True)
+
+ # Actor scoping
+ actor_user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True, index=True)
+ workspace_id = db.Column(db.Integer, db.ForeignKey('workspaces.id'), nullable=True, index=True)
+
+ # Human-readable summary
+ message = db.Column(db.Text, nullable=True)
+
+ # Correlation / trace ID for grouping related events
+ correlation_id = db.Column(db.String(64), nullable=True, index=True)
+
+ # Free-form JSON payload (sanitized before storage)
+ payload_json = db.Column(db.Text, nullable=True)
+
+ created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
+
+ # Relationships
+ actor = db.relationship('User', foreign_keys=[actor_user_id])
+ workspace = db.relationship('Workspace', foreign_keys=[workspace_id])
+
+ # Composite indexes for common query patterns
+ __table_args__ = (
+ db.Index('idx_system_events_source_timestamp', 'source', 'timestamp'),
+ db.Index('idx_system_events_type_timestamp', 'event_type', 'timestamp'),
+ db.Index('idx_system_events_resource', 'resource_type', 'resource_id'),
+ db.Index('idx_system_events_severity_timestamp', 'severity', 'timestamp'),
+ )
+
+ def get_payload(self):
+ """Return parsed payload JSON."""
+ if self.payload_json:
+ try:
+ return json.loads(self.payload_json)
+ except (json.JSONDecodeError, TypeError):
+ return {}
+ return {}
+
+ def set_payload(self, payload_dict):
+ """Set payload from a dictionary."""
+ self.payload_json = json.dumps(payload_dict) if payload_dict is not None else None
+
+ def to_dict(self, include_payload=True):
+ data = {
+ 'id': self.id,
+ 'timestamp': self.timestamp.isoformat() if self.timestamp else None,
+ 'source': self.source,
+ 'event_type': self.event_type,
+ 'severity': self.severity,
+ 'resource_type': self.resource_type,
+ 'resource_id': self.resource_id,
+ 'actor_user_id': self.actor_user_id,
+ 'actor_username': self.actor.username if self.actor else None,
+ 'workspace_id': self.workspace_id,
+ 'workspace_name': self.workspace.name if self.workspace else None,
+ 'message': self.message,
+ 'correlation_id': self.correlation_id,
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
+ }
+ if include_payload:
+ data['payload'] = self.get_payload()
+ return data
+
+ def __repr__(self):
+ return f'<SystemEvent {self.source}.{self.event_type} {self.severity}>'
diff --git a/backend/app/notifications/consumer.py b/backend/app/notifications/consumer.py
index d312916a..489a56d4 100644
--- a/backend/app/notifications/consumer.py
+++ b/backend/app/notifications/consumer.py
@@ -17,6 +17,7 @@ from app.notifications.channels.base import DeliveryResult
from app.notifications.models import NotificationDelivery
from app.notifications.service import GROUP_SLUG, QUEUE_SLUG, QUEUE_CONFIG, NotificationBusService
from app.queue_bus.service import QueueBusService
+from app.services.telemetry_service import TelemetryService
logger = logging.getLogger(__name__)
@@ -108,31 +109,69 @@ def process_message(message):
_fail(message, str(exc)[:500], delivery)
return
+ correlation_id = delivery.notification.correlation_id if delivery.notification else None
+
if result.status == DeliveryResult.SENT:
delivery.status = NotificationDelivery.STATUS_SENT
delivery.sent_at = datetime.utcnow()
delivery.provider_message_id = result.message_id
delivery.error = None
db.session.commit()
+ _emit_delivery_telemetry(delivery, 'notification.delivered', correlation_id)
QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
elif result.status == DeliveryResult.SKIPPED:
delivery.status = NotificationDelivery.STATUS_SKIPPED
delivery.error = result.error
db.session.commit()
+ _emit_delivery_telemetry(delivery, 'notification.skipped', correlation_id, error=result.error)
QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
else: # FAILED — let the queue retry / dead-letter
delivery.error = result.error
db.session.commit()
- _fail(message, result.error or 'delivery failed', delivery)
+ _fail(message, result.error or 'delivery failed', delivery, correlation_id)
+
+
+def _emit_delivery_telemetry(delivery, event_type, correlation_id, error=None):
+ """Best-effort telemetry emission for a notification delivery outcome."""
+ try:
+ notification = delivery.notification
+ severity = 'info'
+ if event_type == 'notification.failed':
+ severity = 'error'
+ elif event_type == 'notification.skipped':
+ severity = 'warning'
+ elif notification and notification.severity in ('warning', 'error', 'critical'):
+ severity = notification.severity
+
+ TelemetryService.emit(
+ source='notification',
+ event_type=event_type,
+ message=f'Notification {event_type.split(".")[-1]} via {delivery.channel}',
+ severity=severity,
+ correlation_id=correlation_id,
+ payload={
+ 'notification_id': notification.id if notification else None,
+ 'delivery_id': delivery.id,
+ 'channel': delivery.channel,
+ 'target': delivery.target,
+ 'error': error,
+ },
+ commit=True,
+ )
+ except Exception:
+ pass
-def _fail(message, error_message, delivery):
+def _fail(message, error_message, delivery, correlation_id=None):
"""Fail the queue message; if the queue dead-letters it, mark the delivery
failed so the row reflects the terminal state."""
res = QueueBusService.fail(GROUP_SLUG, QUEUE_SLUG, message['id'], error_message=error_message)
if res and res.get('status') == 'dead_letter':
delivery.status = NotificationDelivery.STATUS_FAILED
db.session.commit()
+ _emit_delivery_telemetry(delivery, 'notification.failed', correlation_id, error=error_message)
+ else:
+ db.session.commit()
def start_notification_consumer(app):
diff --git a/backend/app/notifications/models.py b/backend/app/notifications/models.py
index deb19612..e8ebb67d 100644
--- a/backend/app/notifications/models.py
+++ b/backend/app/notifications/models.py
@@ -46,6 +46,9 @@ class Notification(db.Model):
# (e.g. 'admins', 'user:42', '[email protected]'). Not used for routing.
audience = db.Column(db.String(255))
+ # Correlation ID for grouping this notification with related telemetry events.
+ correlation_id = db.Column(db.String(64), nullable=True, index=True)
+
created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
deliveries = db.relationship(
@@ -73,6 +76,7 @@ class Notification(db.Model):
'title': self.title,
'body': self.body,
'audience': self.audience,
+ 'correlation_id': self.correlation_id,
'created_at': self.created_at.isoformat() if self.created_at else None,
}
if include_data:
diff --git a/backend/app/notifications/service.py b/backend/app/notifications/service.py
index a1c53936..8aeb4f8d 100644
--- a/backend/app/notifications/service.py
+++ b/backend/app/notifications/service.py
@@ -17,6 +17,7 @@ from app.models.notification_preferences import NotificationPreferences
from app.notifications import catalog
from app.notifications.models import Notification, NotificationDelivery
from app.queue_bus.service import QueueBusService
+from app.services.telemetry_service import TelemetryService, generate_correlation_id
logger = logging.getLogger(__name__)
@@ -59,11 +60,29 @@ class NotificationBusService:
title=meta['title'],
body=data.get('summary') or data.get('message'),
audience=cls._describe(to),
+ correlation_id=generate_correlation_id(),
)
notification.set_data(data)
db.session.add(notification)
db.session.flush() # assign notification.id
+ # Emit telemetry event for the notification lifecycle start.
+ TelemetryService.emit(
+ source='notification',
+ event_type='notification.queued',
+ message=f'Notification queued: {meta["title"]}',
+ severity=severity if severity in ('warning', 'error', 'critical') else 'info',
+ actor_user_id=None,
+ correlation_id=notification.correlation_id,
+ payload={
+ 'notification_id': notification.id,
+ 'event_key': event,
+ 'category': category,
+ 'audience': notification.audience,
+ },
+ commit=False,
+ )
+
seen = set()
to_enqueue = []
inapp_deliveries = []
diff --git a/backend/app/queue_bus/consumers/webhook_consumer.py b/backend/app/queue_bus/consumers/webhook_consumer.py
index 2fc7360c..c90e3b76 100644
--- a/backend/app/queue_bus/consumers/webhook_consumer.py
+++ b/backend/app/queue_bus/consumers/webhook_consumer.py
@@ -13,6 +13,7 @@ import requests as http_requests
from app import db
from app.queue_bus.service import QueueBusService
from app.models.event_subscription import EventSubscription, EventDelivery
+from app.services.telemetry_service import TelemetryService, generate_correlation_id
logger = logging.getLogger(__name__)
@@ -94,9 +95,12 @@ class WebhookConsumer:
return
subscription = delivery.subscription
+ correlation_id = delivery.correlation_id
if not subscription or not subscription.is_active:
delivery.status = EventDelivery.STATUS_FAILED
db.session.commit()
+ _emit_webhook_telemetry(delivery, 'webhook.failed', correlation_id,
+ error='subscription inactive or missing')
QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
return
@@ -142,9 +146,12 @@ class WebhookConsumer:
delivery.status = EventDelivery.STATUS_SUCCESS
delivery.delivered_at = datetime.utcnow()
db.session.commit()
+ _emit_webhook_telemetry(delivery, 'webhook.delivered', correlation_id)
QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
else:
db.session.commit()
+ _emit_webhook_telemetry(delivery, 'webhook.failed', correlation_id,
+ error=f'HTTP {resp.status_code}')
QueueBusService.fail(
GROUP_SLUG,
QUEUE_SLUG,
@@ -157,6 +164,8 @@ class WebhookConsumer:
delivery.duration_ms = round(elapsed_ms, 2)
delivery.response_body = str(e)[:1000]
db.session.commit()
+ _emit_webhook_telemetry(delivery, 'webhook.failed', correlation_id,
+ error=str(e)[:500])
QueueBusService.fail(
GROUP_SLUG,
QUEUE_SLUG,
@@ -165,6 +174,29 @@ class WebhookConsumer:
)
+def _emit_webhook_telemetry(delivery, event_type, correlation_id, error=None):
+ """Best-effort telemetry emission for a webhook delivery outcome."""
+ try:
+ TelemetryService.emit(
+ source='webhook',
+ event_type=event_type,
+ message=f'Webhook {event_type.split(".")[-1]}: {delivery.event_type}',
+ severity='error' if event_type == 'webhook.failed' else 'info',
+ correlation_id=correlation_id,
+ payload={
+ 'delivery_id': delivery.id,
+ 'subscription_id': delivery.subscription_id,
+ 'event_type': delivery.event_type,
+ 'http_status': delivery.http_status,
+ 'duration_ms': delivery.duration_ms,
+ 'error': error,
+ },
+ commit=True,
+ )
+ except Exception:
+ pass
+
+
def start_webhook_consumer(app):
"""Start the singleton webhook consumer thread.
diff --git a/backend/app/services/audit_service.py b/backend/app/services/audit_service.py
index a52cf7f6..30ed0fa5 100644
--- a/backend/app/services/audit_service.py
+++ b/backend/app/services/audit_service.py
@@ -74,6 +74,23 @@ class AuditService:
except Exception:
pass # Don't let event emission failures break audit logging
+ # Emit unified telemetry event (fire-and-forget, same transaction)
+ try:
+ from app.services.telemetry_service import TelemetryService
+ TelemetryService.emit(
+ source='audit',
+ event_type='audit.action_logged',
+ message=f'Audit action: {action}',
+ severity='info',
+ resource_type=target_type,
+ resource_id=target_id,
+ actor_user_id=user_id,
+ payload={'action': action, 'details': details or {}},
+ commit=False,
+ )
+ except Exception:
+ pass # Telemetry failures must not break audit logging
+
if commit:
db.session.commit()
diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.py
index 9056c22d..8f6094aa 100644
--- a/backend/app/services/backup_service.py
+++ b/backend/app/services/backup_service.py
@@ -17,6 +17,7 @@ from app import paths
from app.utils import backup_crypto
from app.utils.formatting import format_bytes
from app.utils.system import is_command_available
+from app.services.telemetry_service import TelemetryService, generate_correlation_id
class BackupService:
@@ -84,14 +85,28 @@ class BackupService:
@classmethod
def backup_application(cls, app_name: str, app_path: str,
- include_db: bool = False, db_config: Dict = None) -> Dict:
+ include_db: bool = False, db_config: Dict = None,
+ correlation_id: str = None) -> Dict:
"""Backup an application (files and optionally database)."""
cls.ensure_backup_dirs()
+ correlation_id = correlation_id or generate_correlation_id()
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
backup_name = f"{app_name}_{timestamp}"
backup_dir = os.path.join(cls.BACKUP_BASE_DIR, 'applications', backup_name)
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.started',
+ message=f'Application backup started: {app_name}',
+ severity='info',
+ resource_type='application',
+ resource_id=app_name,
+ correlation_id=correlation_id,
+ payload={'app_name': app_name, 'include_db': include_db},
+ commit=True,
+ )
+
try:
os.makedirs(backup_dir, exist_ok=True)
@@ -136,17 +151,46 @@ class BackupService:
# Auto-upload to remote if configured (whole directory)
cls._auto_upload(backup_dir, backup_info)
- return {
+ result = {
'success': True,
'backup': backup_info,
- 'path': backup_dir
+ 'path': backup_dir,
+ 'correlation_id': correlation_id,
}
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.completed',
+ message=f'Application backup completed: {app_name}',
+ severity='info',
+ resource_type='application',
+ resource_id=app_name,
+ correlation_id=correlation_id,
+ payload={
+ 'app_name': app_name,
+ 'backup_name': backup_name,
+ 'size': backup_info.get('size'),
+ 'include_db': include_db,
+ },
+ commit=True,
+ )
+ return result
except Exception as e:
# Cleanup on failure
if os.path.exists(backup_dir):
shutil.rmtree(backup_dir, ignore_errors=True)
- return {'success': False, 'error': str(e)}
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.failed',
+ message=f'Application backup failed: {app_name}',
+ severity='error',
+ resource_type='application',
+ resource_id=app_name,
+ correlation_id=correlation_id,
+ payload={'app_name': app_name, 'include_db': include_db, 'error': str(e)},
+ commit=True,
+ )
+ return {'success': False, 'error': str(e), 'correlation_id': correlation_id}
@classmethod
def _backup_database_internal(cls, db_type: str, db_name: str,
@@ -202,14 +246,28 @@ class BackupService:
@classmethod
def backup_database(cls, db_type: str, db_name: str,
user: str = None, password: str = None,
- host: str = 'localhost') -> Dict:
+ host: str = 'localhost',
+ correlation_id: str = None) -> Dict:
"""Backup a database."""
cls.ensure_backup_dirs()
+ correlation_id = correlation_id or generate_correlation_id()
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
backup_name = f"{db_type}_{db_name}_{timestamp}.sql.gz"
backup_path = os.path.join(cls.BACKUP_BASE_DIR, 'databases', backup_name)
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.started',
+ message=f'Database backup started: {db_name}',
+ severity='info',
+ resource_type='database',
+ resource_id=db_name,
+ correlation_id=correlation_id,
+ payload={'db_type': db_type, 'db_name': db_name},
+ commit=True,
+ )
+
config = {
'type': db_type,
'name': db_name,
@@ -242,12 +300,41 @@ class BackupService:
# Auto-upload to remote if configured
cls._auto_upload(backup_path, backup_info)
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.completed',
+ message=f'Database backup completed: {db_name}',
+ severity='info',
+ resource_type='database',
+ resource_id=db_name,
+ correlation_id=correlation_id,
+ payload={
+ 'db_type': db_type,
+ 'db_name': db_name,
+ 'backup_name': backup_name,
+ 'size': backup_info.get('size'),
+ },
+ commit=True,
+ )
+
return {
'success': True,
- 'backup': backup_info
+ 'backup': backup_info,
+ 'correlation_id': correlation_id,
}
- return result
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.failed',
+ message=f'Database backup failed: {db_name}',
+ severity='error',
+ resource_type='database',
+ resource_id=db_name,
+ correlation_id=correlation_id,
+ payload={'db_type': db_type, 'db_name': db_name, 'error': result.get('error')},
+ commit=True,
+ )
+ return {**result, 'correlation_id': correlation_id}
@classmethod
def backup_files(cls, file_paths: List[str], backup_name: str = None) -> Dict:
@@ -797,6 +884,19 @@ class BackupService:
backup_type = sched.get('backup_type', 'database')
target = sched.get('target', '')
result = None
+ correlation_id = generate_correlation_id()
+
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.scheduled_started',
+ message=f'Scheduled backup started: {sched.get("name", "Backup")}',
+ severity='info',
+ resource_type='backup_schedule',
+ resource_id=sched.get('id'),
+ correlation_id=correlation_id,
+ payload={'schedule_name': sched.get('name'), 'backup_type': backup_type, 'target': target},
+ commit=True,
+ )
try:
if backup_type == 'database':
@@ -806,13 +906,13 @@ class BackupService:
db_type, db_name = parts
else:
db_type, db_name = 'mysql', target
- result = cls.backup_database(db_type, db_name)
+ result = cls.backup_database(db_type, db_name, correlation_id=correlation_id)
elif backup_type == 'application':
from app.models import Application
app = Application.query.filter_by(name=target).first()
if app:
- result = cls.backup_application(app.name, app.root_path)
+ result = cls.backup_application(app.name, app.root_path, correlation_id=correlation_id)
else:
result = {'success': False, 'error': f'Application "{target}" not found'}
@@ -847,7 +947,8 @@ class BackupService:
cls._send_backup_notification(
sched.get('name', 'Backup'),
False,
- result.get('error', 'Unknown error')
+ result.get('error', 'Unknown error'),
+ correlation_id=correlation_id,
)
except Exception as e:
@@ -859,11 +960,23 @@ class BackupService:
s['last_status'] = 'failed'
break
cls.save_config(config)
- cls._send_backup_notification(sched.get('name', 'Backup'), False, str(e))
+ cls._send_backup_notification(sched.get('name', 'Backup'), False, str(e), correlation_id=correlation_id)
@classmethod
- def _send_backup_notification(cls, backup_name: str, success: bool, message: str) -> None:
- """Send a notification about backup status."""
+ def _send_backup_notification(cls, backup_name: str, success: bool, message: str,
+ correlation_id: str = None) -> None:
+ """Send a notification about backup status and emit telemetry."""
+ TelemetryService.emit(
+ source='backup',
+ event_type='backup.completed' if success else 'backup.failed',
+ message=f'Backup {backup_name} {"completed" if success else "failed"}',
+ severity='info' if success else 'critical',
+ resource_type='backup_schedule',
+ correlation_id=correlation_id,
+ payload={'backup_name': backup_name, 'success': success, 'message': message},
+ commit=True,
+ )
+
try:
from app.services.notification_service import NotificationService
config = cls.get_config()
diff --git a/backend/app/services/deployment_job_service.py b/backend/app/services/deployment_job_service.py
index bcb4c1d7..8493f0ab 100644
--- a/backend/app/services/deployment_job_service.py
+++ b/backend/app/services/deployment_job_service.py
@@ -14,6 +14,7 @@ from app.models.deployment_job import DeploymentJob
from app.services.deployment_runner import DeploymentPlanRunner
from app.services.docker_service import DockerService
from app.services.template_service import TemplateService
+from app.services.telemetry_service import generate_correlation_id
class DeploymentJobService:
@@ -65,6 +66,7 @@ class DeploymentJobService:
target_server_id=normalized_server_id,
requested_by=user_id,
trigger='manual',
+ correlation_id=generate_correlation_id(),
)
job.set_plan(plan_result['plan'])
db.session.add(job)
diff --git a/backend/app/services/deployment_runner.py b/backend/app/services/deployment_runner.py
index d34bf3de..7b7d12b0 100644
--- a/backend/app/services/deployment_runner.py
+++ b/backend/app/services/deployment_runner.py
@@ -14,6 +14,7 @@ from app import db
from app.models.deployment_job import DeploymentJob, DeploymentJobLog
from app.services.agent_registry import agent_registry
from app.services.docker_service import DockerService
+from app.services.telemetry_service import TelemetryService, generate_correlation_id
class DeploymentStepError(Exception):
@@ -25,6 +26,9 @@ class DeploymentPlanRunner:
def __init__(self, job: DeploymentJob):
self.job = job
+ self.correlation_id = job.correlation_id or generate_correlation_id()
+ if not job.correlation_id:
+ job.correlation_id = self.correlation_id
@property
def is_remote(self) -> bool:
@@ -41,6 +45,22 @@ class DeploymentPlanRunner:
db.session.commit()
self.log('info', f"Deployment started on {self.job.target_server_name}")
+ TelemetryService.emit(
+ source='deployment',
+ event_type='deployment.started',
+ message=f'Deployment started: {self.job.kind}',
+ severity='info',
+ resource_type='deployment_job',
+ resource_id=self.job.id,
+ correlation_id=self.correlation_id,
+ payload={
+ 'job_id': self.job.id,
+ 'kind': self.job.kind,
+ 'target_server_id': self.job.target_server_id,
+ 'total_steps': len(steps),
+ },
+ commit=False,
+ )
try:
for index, step in enumerate(steps, start=1):
@@ -50,6 +70,17 @@ class DeploymentPlanRunner:
db.session.commit()
self.log('info', name, step_index=index)
+ TelemetryService.emit(
+ source='deployment',
+ event_type='deployment.step_started',
+ message=f'Deployment step {index} started: {name}',
+ severity='info',
+ resource_type='deployment_job',
+ resource_id=self.job.id,
+ correlation_id=self.correlation_id,
+ payload={'job_id': self.job.id, 'step_index': index, 'step_name': name},
+ commit=False,
+ )
result = self._execute_step(step)
results.append({'step': index, 'name': name, 'result': result})
@@ -60,6 +91,17 @@ class DeploymentPlanRunner:
db.session.commit()
self.log('info', 'Deployment completed')
+ TelemetryService.emit(
+ source='deployment',
+ event_type='deployment.completed',
+ message=f'Deployment completed: {self.job.kind}',
+ severity='info',
+ resource_type='deployment_job',
+ resource_id=self.job.id,
+ correlation_id=self.correlation_id,
+ payload={'job_id': self.job.id, 'kind': self.job.kind, 'total_steps': len(steps)},
+ commit=False,
+ )
return {'success': True, 'steps': results}
except Exception as exc:
@@ -70,6 +112,22 @@ class DeploymentPlanRunner:
db.session.commit()
self.log('error', str(exc), step_index=self.job.current_step)
+ TelemetryService.emit(
+ source='deployment',
+ event_type='deployment.failed',
+ message=f'Deployment failed: {self.job.kind}',
+ severity='error',
+ resource_type='deployment_job',
+ resource_id=self.job.id,
+ correlation_id=self.correlation_id,
+ payload={
+ 'job_id': self.job.id,
+ 'kind': self.job.kind,
+ 'error': str(exc),
+ 'failed_step': self.job.current_step,
+ },
+ commit=False,
+ )
return {'success': False, 'error': str(exc), 'steps': results}
def log(self, level: str, message: str, data: Any = None, step_index: int = None):
diff --git a/backend/app/services/event_service.py b/backend/app/services/event_service.py
index 6ee0a4a2..5322e527 100644
--- a/backend/app/services/event_service.py
+++ b/backend/app/services/event_service.py
@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
from app import db
from app.models.event_subscription import EventSubscription, EventDelivery
from app.queue_bus.consumers.webhook_consumer import enqueue_webhook_delivery
+from app.services.telemetry_service import generate_correlation_id
logger = logging.getLogger(__name__)
@@ -97,6 +98,7 @@ class EventService:
subscription_id=sub.id,
event_type=event_type,
status=EventDelivery.STATUS_PENDING,
+ correlation_id=generate_correlation_id(),
)
delivery.set_payload(payload)
db.session.add(delivery)
diff --git a/backend/app/services/monitoring_service.py b/backend/app/services/monitoring_service.py
index 5a781230..413b60f1 100644
--- a/backend/app/services/monitoring_service.py
+++ b/backend/app/services/monitoring_service.py
@@ -15,6 +15,7 @@ logger = logging.getLogger(__name__)
from .notification_service import NotificationService
from app import paths
+from app.services.telemetry_service import TelemetryService
class MonitoringService:
@@ -283,6 +284,25 @@ class MonitoringService:
# Send to all configured notification channels (Discord, Slack, Telegram, etc.)
NotificationService.send_all(alerts_to_send)
+ # Emit unified telemetry events for each alert
+ for alert in alerts_to_send:
+ alert_key = f"{alert['type']}_{alert['severity']}"
+ TelemetryService.emit(
+ source='monitoring',
+ event_type='monitoring.threshold_breached',
+ message=f"{alert['type'].upper()} alert: {alert.get('message', '')}",
+ severity=alert.get('severity', 'warning'),
+ resource_type='server',
+ correlation_id=alert_key,
+ payload={
+ 'alert_type': alert.get('type'),
+ 'value': alert.get('value'),
+ 'threshold': alert.get('threshold'),
+ 'severity': alert.get('severity'),
+ },
+ commit=True,
+ )
+
# Emit events for workflow triggers
try:
from app.services.workflow_engine import WorkflowEventBus
diff --git a/backend/app/services/telemetry_service.py b/backend/app/services/telemetry_service.py
new file mode 100644
index 00000000..cd69aa37
--- /dev/null
+++ b/backend/app/services/telemetry_service.py
@@ -0,0 +1,317 @@
+"""Telemetry service for the unified system event stream.
+
+This service provides a single, fire-and-forget way to record events from any
+subsystem. It never raises into callers and redacts known sensitive keys from
+payloads before storage.
+"""
+import json
+import logging
+import secrets
+from datetime import datetime, timedelta
+
+from flask import g, has_request_context, request
+from sqlalchemy import func
+
+from app import db
+from app.models.system_event import SystemEvent
+
+logger = logging.getLogger(__name__)
+
+REDACTED = '[redacted]'
+SENSITIVE_KEY_PARTS = (
+ 'password',
+ 'passwd',
+ 'secret',
+ 'token',
+ 'credential',
+ 'private',
+ 'certificate',
+ 'api_key',
+ 'apikey',
+ 'authorization',
+ 'cookie',
+ 'session',
+ 'totp',
+ 'otp',
+ 'csrf',
+)
+
+MAX_STRING_LENGTH = 2000
+MAX_LIST_ITEMS = 50
+MAX_DICT_ITEMS = 50
+MAX_PAYLOAD_BYTES = 32 * 1024
+
+
+def generate_correlation_id():
+ """Generate a unique correlation ID for grouping related events."""
+ return secrets.token_hex(16)
+
+
+class TelemetryService:
+ """Service for emitting and querying system telemetry events."""
+
+ # ------------------------------------------------------------------
+ # Emit
+ # ------------------------------------------------------------------
+ @classmethod
+ def emit(cls, source, event_type, message=None, severity=SystemEvent.SEVERITY_INFO,
+ resource_type=None, resource_id=None, actor_user_id=None, workspace_id=None,
+ correlation_id=None, payload=None, commit=True):
+ """Record a telemetry event.
+
+ This method is fire-and-forget: any failure is logged but never raised,
+ so telemetry cannot break the calling operation.
+ """
+ try:
+ event = cls._create_event(
+ source=source,
+ event_type=event_type,
+ message=message,
+ severity=severity,
+ resource_type=resource_type,
+ resource_id=_stringify_resource_id(resource_id),
+ actor_user_id=actor_user_id,
+ workspace_id=workspace_id,
+ correlation_id=correlation_id,
+ payload=payload,
+ )
+ db.session.add(event)
+ if commit:
+ db.session.commit()
+ return event
+ except Exception as exc:
+ logger.warning('Telemetry emit failed for %s.%s: %s', source, event_type, exc)
+ db.session.rollback()
+ return None
+
+ @classmethod
+ def emit_for_request(cls, source, event_type, message=None, severity=SystemEvent.SEVERITY_INFO,
+ resource_type=None, resource_id=None, correlation_id=None, payload=None,
+ commit=True):
+ """Emit an event, deriving actor/workspace from the current request context."""
+ actor_user_id, workspace_id = cls._get_request_scope()
+ return cls.emit(
+ source=source,
+ event_type=event_type,
+ message=message,
+ severity=severity,
+ resource_type=resource_type,
+ resource_id=resource_id,
+ actor_user_id=actor_user_id,
+ workspace_id=workspace_id,
+ correlation_id=correlation_id,
+ payload=payload,
+ commit=commit,
+ )
+
+ @classmethod
+ def _create_event(cls, source, event_type, message, severity, resource_type,
+ resource_id, actor_user_id, workspace_id, correlation_id, payload):
+ if severity not in SystemEvent.VALID_SEVERITIES:
+ severity = SystemEvent.SEVERITY_INFO
+
+ sanitized_payload = _sanitize_payload(payload) if payload is not None else None
+
+ # Truncate very long messages
+ if message is not None and len(message) > 4000:
+ message = message[:4000] + '...[truncated]'
+
+ event = SystemEvent(
+ timestamp=datetime.utcnow(),
+ source=source,
+ event_type=event_type,
+ severity=severity,
+ resource_type=resource_type,
+ resource_id=resource_id,
+ actor_user_id=actor_user_id,
+ workspace_id=workspace_id,
+ message=message,
+ correlation_id=correlation_id,
+ )
+ event.set_payload(sanitized_payload)
+ return event
+
+ @staticmethod
+ def _get_request_scope():
+ """Return (actor_user_id, workspace_id) from request context if available."""
+ if not has_request_context():
+ return None, None
+
+ actor_user_id = None
+ workspace_id = None
+
+ # JWT or API key user
+ try:
+ from flask_jwt_extended import get_jwt_identity
+ identity = get_jwt_identity()
+ if identity is not None:
+ try:
+ actor_user_id = int(identity)
+ except (TypeError, ValueError):
+ actor_user_id = identity
+ except Exception:
+ pass
+
+ if actor_user_id is None:
+ api_key_user = getattr(g, 'api_key_user', None)
+ if api_key_user is not None:
+ actor_user_id = api_key_user.id
+
+ # Workspace from g if set by other middleware
+ workspace_id = getattr(g, 'workspace_id', None)
+
+ return actor_user_id, workspace_id
+
+ # ------------------------------------------------------------------
+ # Query
+ # ------------------------------------------------------------------
+ @classmethod
+ def get_events(cls, page=1, per_page=50, source=None, event_type=None, severity=None,
+ resource_type=None, resource_id=None, correlation_id=None,
+ start_date=None, end_date=None, message_query=None):
+ """Query system events with filters and pagination."""
+ query = SystemEvent.query.order_by(SystemEvent.timestamp.desc())
+
+ if source:
+ query = query.filter(SystemEvent.source == source)
+ if event_type:
+ query = query.filter(SystemEvent.event_type == event_type)
+ if severity:
+ query = query.filter(SystemEvent.severity == severity)
+ if resource_type:
+ query = query.filter(SystemEvent.resource_type == resource_type)
+ if resource_id is not None:
+ query = query.filter(SystemEvent.resource_id == _stringify_resource_id(resource_id))
+ if correlation_id:
+ query = query.filter(SystemEvent.correlation_id == correlation_id)
+ if start_date:
+ query = query.filter(SystemEvent.timestamp >= start_date)
+ if end_date:
+ query = query.filter(SystemEvent.timestamp <= end_date)
+ if message_query:
+ query = query.filter(SystemEvent.message.ilike(f'%{message_query}%'))
+
+ return query.paginate(page=page, per_page=per_page, error_out=False)
+
+ @classmethod
+ def get_event_by_id(cls, event_id):
+ """Get a single event by ID."""
+ return db.session.get(SystemEvent, event_id)
+
+ @classmethod
+ def get_events_by_correlation(cls, correlation_id, limit=100):
+ """Get all events sharing a correlation ID, oldest first."""
+ return (SystemEvent.query
+ .filter_by(correlation_id=correlation_id)
+ .order_by(SystemEvent.timestamp.asc())
+ .limit(limit)
+ .all())
+
+ # ------------------------------------------------------------------
+ # Stats
+ # ------------------------------------------------------------------
+ @classmethod
+ def get_stats(cls, hours=24, source=None):
+ """Return aggregate stats for the given time window."""
+ cutoff = datetime.utcnow() - timedelta(hours=hours)
+ query = SystemEvent.query.filter(SystemEvent.timestamp >= cutoff)
+ if source:
+ query = query.filter(SystemEvent.source == source)
+
+ total = query.count()
+
+ by_severity = dict(
+ query.with_entities(SystemEvent.severity, func.count(SystemEvent.id))
+ .group_by(SystemEvent.severity)
+ .all()
+ )
+
+ by_source = dict(
+ query.with_entities(SystemEvent.source, func.count(SystemEvent.id))
+ .group_by(SystemEvent.source)
+ .all()
+ )
+
+ return {
+ 'total': total,
+ 'by_severity': by_severity,
+ 'by_source': by_source,
+ 'hours': hours,
+ }
+
+ @classmethod
+ def get_recent_event_types(cls, limit=20):
+ """Return the most common event types in the last 24 hours."""
+ cutoff = datetime.utcnow() - timedelta(hours=24)
+ rows = (SystemEvent.query
+ .filter(SystemEvent.timestamp >= cutoff)
+ .with_entities(SystemEvent.event_type, func.count(SystemEvent.id))
+ .group_by(SystemEvent.event_type)
+ .order_by(func.count(SystemEvent.id).desc())
+ .limit(limit)
+ .all())
+ return [{'event_type': t, 'count': c} for t, c in rows]
+
+ # ------------------------------------------------------------------
+ # Cleanup
+ # ------------------------------------------------------------------
+ @classmethod
+ def cleanup_old_events(cls, days=90):
+ """Delete events older than the specified number of days."""
+ cutoff = datetime.utcnow() - timedelta(days=days)
+ deleted = SystemEvent.query.filter(SystemEvent.timestamp < cutoff).delete()
+ db.session.commit()
+ return deleted
+
+
+# ------------------------------------------------------------------------------
+# Helpers
+# ------------------------------------------------------------------------------
+
+def _stringify_resource_id(resource_id):
+ """Normalize resource IDs to strings to support int/UUID/mixed use."""
+ if resource_id is None:
+ return None
+ return str(resource_id)
+
+
+def _sanitize_payload(value, key=None, depth=0):
+ """Sanitize a payload dict/list, redacting sensitive keys and capping size."""
+ if key and _is_sensitive_key(key):
+ return REDACTED
+ if depth >= 4:
+ return '[truncated]'
+
+ if isinstance(value, dict):
+ result = {}
+ for item_key, item_value in list(value.items())[:MAX_DICT_ITEMS]:
+ safe_key = str(item_key)[:MAX_STRING_LENGTH]
+ result[safe_key] = _sanitize_payload(item_value, key=safe_key, depth=depth + 1)
+ return result
+
+ if isinstance(value, (list, tuple)):
+ result = [_sanitize_payload(item, depth=depth + 1) for item in value[:MAX_LIST_ITEMS]]
+ return result
+
+ if isinstance(value, str):
+ if len(value) > MAX_STRING_LENGTH:
+ return value[:MAX_STRING_LENGTH] + '...[truncated]'
+ return value
+
+ if value is None or isinstance(value, (bool, int, float)):
+ return value
+
+ return str(value)[:MAX_STRING_LENGTH]
+
+
+def _is_sensitive_key(key):
+ lowered = str(key).lower()
+ return any(part in lowered for part in SENSITIVE_KEY_PARTS)
+
+
+def _payload_too_large(payload):
+ """Check whether a payload would exceed the storage limit when serialized."""
+ try:
+ return len(json.dumps(payload).encode('utf-8')) > MAX_PAYLOAD_BYTES
+ except (TypeError, ValueError):
+ return False
diff --git a/backend/tests/test_telemetry.py b/backend/tests/test_telemetry.py
new file mode 100644
index 00000000..dc91174d
--- /dev/null
+++ b/backend/tests/test_telemetry.py
@@ -0,0 +1,200 @@
+"""Tests for the unified telemetry / system event stream."""
+from datetime import datetime, timedelta
+
+import pytest
+
+from app import db
+from app.models.system_event import SystemEvent
+from app.services.telemetry_service import TelemetryService, generate_correlation_id
+
+
+class TestTelemetryService:
+ def test_emit_event_creates_row(self, app):
+ event = TelemetryService.emit(
+ source='test',
+ event_type='test.event',
+ message='Hello telemetry',
+ severity='info',
+ resource_type='app',
+ resource_id=42,
+ payload={'hello': 'world'},
+ commit=True,
+ )
+ assert event is not None
+ assert event.id is not None
+ assert event.source == 'test'
+ assert event.event_type == 'test.event'
+ assert event.message == 'Hello telemetry'
+ assert event.resource_type == 'app'
+ assert event.resource_id == '42'
+ assert event.get_payload() == {'hello': 'world'}
+
+ def test_emit_redacts_sensitive_payload_keys(self, app):
+ event = TelemetryService.emit(
+ source='test',
+ event_type='test.secret',
+ payload={
+ 'username': 'admin',
+ 'api_key': 'super-secret',
+ 'nested': {'password': 'hunter2'},
+ },
+ commit=True,
+ )
+ payload = event.get_payload()
+ assert payload['username'] == 'admin'
+ assert payload['api_key'] == '[redacted]'
+ assert payload['nested']['password'] == '[redacted]'
+
+ def test_emit_invalid_severity_defaults_to_info(self, app):
+ event = TelemetryService.emit(
+ source='test',
+ event_type='test.event',
+ severity='not-a-severity',
+ commit=True,
+ )
+ assert event.severity == 'info'
+
+ def test_emit_does_not_raise_on_failure(self, app, monkeypatch):
+ monkeypatch.setattr(db.session, 'commit', lambda: (_ for _ in ()).throw(Exception('boom')))
+ event = TelemetryService.emit(
+ source='test',
+ event_type='test.event',
+ commit=True,
+ )
+ assert event is None
+
+ def test_get_events_filter_by_source_and_severity(self, app):
+ TelemetryService.emit(source='audit', event_type='audit.login', severity='info', commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.failed', severity='error', commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.completed', severity='info', commit=True)
+
+ result = TelemetryService.get_events(source='backup', severity='error')
+ assert result.total == 1
+ assert result.items[0].event_type == 'backup.failed'
+
+ def test_get_events_by_correlation_id(self, app):
+ correlation_id = generate_correlation_id()
+ TelemetryService.emit(source='backup', event_type='backup.started', correlation_id=correlation_id, commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.completed', correlation_id=correlation_id, commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.started', commit=True)
+
+ events = TelemetryService.get_events_by_correlation(correlation_id)
+ assert len(events) == 2
+
+ def test_get_stats(self, app):
+ TelemetryService.emit(source='backup', event_type='backup.completed', severity='info', commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.failed', severity='error', commit=True)
+ TelemetryService.emit(source='monitoring', event_type='monitoring.alert', severity='warning', commit=True)
+
+ stats = TelemetryService.get_stats(hours=24)
+ assert stats['total'] == 3
+ assert stats['by_severity']['error'] == 1
+ assert stats['by_severity']['warning'] == 1
+ assert stats['by_severity']['info'] == 1
+ assert stats['by_source']['backup'] == 2
+ assert stats['by_source']['monitoring'] == 1
+
+ def test_cleanup_old_events(self, app):
+ old = TelemetryService.emit(source='test', event_type='test.old', commit=True)
+ old.timestamp = datetime.utcnow() - timedelta(days=100)
+ db.session.commit()
+
+ TelemetryService.emit(source='test', event_type='test.new', commit=True)
+
+ deleted = TelemetryService.cleanup_old_events(days=90)
+ assert deleted == 1
+ assert SystemEvent.query.filter_by(event_type='test.old').first() is None
+ assert SystemEvent.query.filter_by(event_type='test.new').first() is not None
+
+
+class TestTelemetryAPI:
+ def test_list_events_requires_auth(self, client):
+ response = client.get('/api/v1/telemetry/events')
+ assert response.status_code == 401
+
+ def test_list_events(self, app, client, auth_headers):
+ TelemetryService.emit(source='audit', event_type='audit.login', severity='info', commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.failed', severity='error', commit=True)
+
+ response = client.get('/api/v1/telemetry/events', headers=auth_headers)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['total'] == 2
+ assert len(data['events']) == 2
+
+ def test_list_events_filter_by_source(self, app, client, auth_headers):
+ TelemetryService.emit(source='audit', event_type='audit.login', commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.failed', commit=True)
+
+ response = client.get('/api/v1/telemetry/events?source=backup', headers=auth_headers)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['total'] == 1
+ assert data['events'][0]['source'] == 'backup'
+
+ def test_get_event(self, app, client, auth_headers):
+ event = TelemetryService.emit(source='test', event_type='test.event', commit=True)
+ response = client.get(f'/api/v1/telemetry/events/{event.id}', headers=auth_headers)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['id'] == event.id
+ assert data['source'] == 'test'
+
+ def test_get_event_not_found(self, client, auth_headers):
+ response = client.get('/api/v1/telemetry/events/99999', headers=auth_headers)
+ assert response.status_code == 404
+
+ def test_get_events_by_correlation(self, app, client, auth_headers):
+ correlation_id = generate_correlation_id()
+ TelemetryService.emit(source='backup', event_type='backup.started', correlation_id=correlation_id, commit=True)
+ TelemetryService.emit(source='backup', event_type='backup.completed', correlation_id=correlation_id, commit=True)
+
+ response = client.get(f'/api/v1/telemetry/events/by-correlation/{correlation_id}', headers=auth_headers)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert len(data['events']) == 2
+
+ def test_get_stats_endpoint(self, app, client, auth_headers):
+ TelemetryService.emit(source='backup', event_type='backup.completed', commit=True)
+ response = client.get('/api/v1/telemetry/stats', headers=auth_headers)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['total'] >= 1
+ assert 'by_severity' in data
+ assert 'by_source' in data
+
+ def test_cleanup_events_admin_only(self, app, client, auth_headers):
+ response = client.delete('/api/v1/telemetry/events?days=1', headers=auth_headers)
+ assert response.status_code == 200
+
+ def test_emit_test_event_admin_only(self, app, client, auth_headers):
+ response = client.post('/api/v1/telemetry/events/test', json={'message': 'test'}, headers=auth_headers)
+ assert response.status_code == 201
+ data = response.get_json()
+ assert data['event_type'] == 'telemetry.test'
+
+
+class TestTelemetryIntegrations:
+ def test_audit_service_emits_telemetry_event(self, app):
+ from app.services.audit_service import AuditService
+ from app.models import User
+ from werkzeug.security import generate_password_hash
+
+ user = User(email='[email protected]', username='audituser', password_hash=generate_password_hash('x'))
+ db.session.add(user)
+ db.session.commit()
+
+ AuditService.log(
+ action='app.create',
+ user_id=user.id,
+ target_type='app',
+ target_id=123,
+ details={'name': 'my-app'},
+ commit=True,
+ )
+
+ event = SystemEvent.query.filter_by(source='audit').first()
+ assert event is not None
+ assert event.event_type == 'audit.action_logged'
+ assert event.actor_user_id == user.id
+ assert event.get_payload()['details']['name'] == 'my-app'
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index e7d9de0d..45c3894c 100644
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -73,6 +73,7 @@ import DynamicDns from './pages/DynamicDns';
import QueueOperations from './pages/QueueOperations';
import Notifications from './pages/Notifications';
import DeliveryLog from './pages/DeliveryLog';
+import Telemetry from './pages/Telemetry';
import useExtensionRoutes from './plugins/ExtensionRoutes';
import { useContributions } from './plugins/contributions';
@@ -126,6 +127,7 @@ const PAGE_TITLES = {
'/queue': 'Queue Bus',
'/notifications': 'Notifications',
'/admin/notifications': 'Notification Delivery Log',
+ '/telemetry': 'Telemetry',
};
function PageTitleUpdater() {
@@ -364,6 +366,7 @@ function AppRoutes() {
<Route path="queue" element={<QueueOperations />} />
<Route path="notifications" element={<Notifications />} />
<Route path="admin/notifications" element={<DeliveryLog />} />
+ <Route path="telemetry" element={<Telemetry />} />
<Route path="settings" element={<Settings />} />
<Route path="settings/:tab" element={<Settings />} />
{dashboardRoutes}
diff --git a/frontend/src/components/sidebarItems.js b/frontend/src/components/sidebarItems.js
index b0d1a7f7..e3ab009d 100644
--- a/frontend/src/components/sidebarItems.js
+++ b/frontend/src/components/sidebarItems.js
@@ -179,6 +179,13 @@ export const SIDEBAR_ITEMS = [
category: 'system',
icon: '<path d="M4 17l6-6-6-6M12 19h8"/>'
},
+ {
+ id: 'telemetry',
+ label: 'Telemetry',
+ route: '/telemetry',
+ category: 'system',
+ icon: '<path d="M22 12h-4l-3 9L9 3l-3 9H2"/>'
+ },
{
id: 'gpu',
label: 'GPU Monitor',
@@ -214,7 +221,7 @@ export const SIDEBAR_ITEMS = [
// Secrets & Webhooks vault. They're hidden by the default ("Recommended") view
// and every curated preset, but stay one click away via the "Full" view or
// Customize Sidebar — and remain fully routable (deep links, command palette).
-export const ADVANCED_ITEM_IDS = ['marketplace', 'queue', 'secrets'];
+export const ADVANCED_ITEM_IDS = ['marketplace', 'queue', 'secrets', 'telemetry'];
// Preset profiles define which items are hidden (top-level only)
export const SIDEBAR_PRESETS = {
diff --git a/frontend/src/pages/Telemetry.jsx b/frontend/src/pages/Telemetry.jsx
new file mode 100644
index 00000000..b4525d05
--- /dev/null
+++ b/frontend/src/pages/Telemetry.jsx
@@ -0,0 +1,465 @@
+import { useState, useEffect, useCallback } from 'react';
+import { useSearchParams } from 'react-router-dom';
+import {
+ Activity,
+ AlertTriangle,
+ ChevronDown,
+ ChevronRight,
+ ChevronUp,
+ Filter,
+ Info,
+ Loader2,
+ RefreshCw,
+ Search,
+ Trash2,
+ X,
+ XCircle,
+} from 'lucide-react';
+import api from '../services/api';
+import { PageTopbar } from '@/components/ds';
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { useAuth } from '../contexts/AuthContext';
+import { useToast } from '../contexts/ToastContext';
+
+const SEVERITY_ORDER = ['critical', 'error', 'warning', 'info', 'debug'];
+
+const SEVERITY_CONFIG = {
+ critical: { icon: XCircle, color: '#fb6f6f', label: 'Critical' },
+ error: { icon: XCircle, color: '#ff7b7b', label: 'Error' },
+ warning: { icon: AlertTriangle, color: '#f5b945', label: 'Warning' },
+ info: { icon: Info, color: '#6d7cff', label: 'Info' },
+ debug: { icon: Activity, color: '#9aa1b2', label: 'Debug' },
+};
+
+const PAGE_SIZE = 50;
+
+export default function Telemetry() {
+ const [searchParams, setSearchParams] = useSearchParams();
+ const { isAdmin } = useAuth();
+ const { showToast } = useToast();
+
+ const [events, setEvents] = useState([]);
+ const [loading, setLoading] = useState(true);
+ const [hasMore, setHasMore] = useState(false);
+ const [page, setPage] = useState(1);
+ const [stats, setStats] = useState(null);
+ const [sources, setSources] = useState([]);
+ const [eventTypes, setEventTypes] = useState([]);
+ const [selectedEvent, setSelectedEvent] = useState(null);
+ const [filtersExpanded, setFiltersExpanded] = useState(false);
+
+ const [filters, setFilters] = useState({
+ source: searchParams.get('source') || '',
+ event_type: searchParams.get('event_type') || '',
+ severity: searchParams.get('severity') || '',
+ resource_type: searchParams.get('resource_type') || '',
+ resource_id: searchParams.get('resource_id') || '',
+ correlation_id: searchParams.get('correlation_id') || '',
+ q: searchParams.get('q') || '',
+ start_date: searchParams.get('start_date') || '',
+ end_date: searchParams.get('end_date') || '',
+ });
+
+ const loadStats = useCallback(async () => {
+ try {
+ const data = await api.getTelemetryStats({ hours: 24 });
+ setStats(data);
+ } catch {
+ // stats are optional
+ }
+ }, []);
+
+ const loadFilters = useCallback(async () => {
+ try {
+ const [sourcesData, typesData] = await Promise.all([
+ api.getTelemetrySources(),
+ api.getTelemetryEventTypes({ source: filters.source || undefined }),
+ ]);
+ setSources(sourcesData.sources || []);
+ setEventTypes(typesData.event_types || []);
+ } catch {
+ // filters are optional
+ }
+ }, [filters.source]);
+
+ const fetchEvents = useCallback(async (nextPage = 1, replace = true) => {
+ setLoading(true);
+ try {
+ const params = { ...filters, per_page: PAGE_SIZE, page: nextPage };
+ Object.keys(params).forEach((key) => {
+ if (params[key] === '') delete params[key];
+ });
+ const data = await api.getTelemetryEvents(params);
+ const fresh = data.events || [];
+ setEvents((prev) => (replace ? fresh : [...prev, ...fresh]));
+ setHasMore(fresh.length === PAGE_SIZE);
+ setPage(nextPage);
+ } catch (err) {
+ showToast(`Failed to load telemetry: ${err.message}`, 'error');
+ setHasMore(false);
+ } finally {
+ setLoading(false);
+ }
+ }, [filters, showToast]);
+
+ useEffect(() => {
+ loadStats();
+ loadFilters();
+ fetchEvents(1, true);
+ }, [loadStats, loadFilters, fetchEvents]);
+
+ const applyFilters = () => {
+ const next = new URLSearchParams();
+ Object.entries(filters).forEach(([key, value]) => {
+ if (value) next.set(key, value);
+ });
+ setSearchParams(next);
+ fetchEvents(1, true);
+ };
+
+ const clearFilters = () => {
+ const empty = {
+ source: '',
+ event_type: '',
+ severity: '',
+ resource_type: '',
+ resource_id: '',
+ correlation_id: '',
+ q: '',
+ start_date: '',
+ end_date: '',
+ };
+ setFilters(empty);
+ setSearchParams(new URLSearchParams());
+ setFiltersExpanded(false);
+ fetchEvents(1, true);
+ };
+
+ const loadMore = () => {
+ if (!loading && hasMore) {
+ fetchEvents(page + 1, false);
+ }
+ };
+
+ const handleFilterChange = (key, value) => {
+ setFilters((prev) => ({ ...prev, [key]: value }));
+ };
+
+ const handleCorrelationClick = (correlationId) => {
+ if (!correlationId) return;
+ setFilters((prev) => ({ ...prev, correlation_id: correlationId }));
+ setSearchParams({ correlation_id: correlationId });
+ fetchEvents(1, true);
+ };
+
+ const emitTestEvent = async () => {
+ try {
+ await api.emitTestTelemetryEvent({
+ source: 'system',
+ event_type: 'telemetry.test',
+ message: 'Test event from UI',
+ severity: 'info',
+ payload: { from_ui: true },
+ });
+ showToast('Test event emitted', 'success');
+ fetchEvents(1, true);
+ loadStats();
+ } catch (err) {
+ showToast(`Failed to emit test event: ${err.message}`, 'error');
+ }
+ };
+
+ const cleanupOldEvents = async () => {
+ if (!window.confirm('Delete telemetry events older than 90 days? This cannot be undone.')) {
+ return;
+ }
+ try {
+ const data = await api.cleanupTelemetryEvents(90);
+ showToast(`Deleted ${data.deleted} old events`, 'success');
+ fetchEvents(1, true);
+ loadStats();
+ } catch (err) {
+ showToast(`Cleanup failed: ${err.message}`, 'error');
+ }
+ };
+
+ const activeFilterCount = Object.values(filters).filter(Boolean).length;
+
+ return (
+ <>
+ <PageTopbar
+ icon={<Activity size={18} />}
+ title="Telemetry"
+ meta={stats ? `${stats.total || 0} events in last 24h` : 'System event stream'}
+ actions={(
+ <>
+ {isAdmin && (
+ <Button variant="outline" size="sm" onClick={emitTestEvent}>
+ <Activity size={15} /> Test event
+ </Button>
+ )}
+ {isAdmin && (
+ <Button variant="outline" size="sm" onClick={cleanupOldEvents}>
+ <Trash2 size={15} /> Cleanup
+ </Button>
+ )}
+ <Button variant="outline" size="sm" onClick={() => fetchEvents(1, true)} disabled={loading}>
+ <RefreshCw size={15} className={loading ? 'spin' : ''} /> Refresh
+ </Button>
+ </>
+ )}
+ />
+
+ <div className="telemetry-page">
+ {stats && (
+ <div className="telemetry-stats">
+ {SEVERITY_ORDER.map((severity) => (
+ <div key={severity} className={`telemetry-stat telemetry-stat--${severity}`}>
+ <span className="telemetry-stat__count">
+ {stats.by_severity?.[severity] || 0}
+ </span>
+ <span className="telemetry-stat__label">
+ {SEVERITY_CONFIG[severity]?.label}
+ </span>
+ </div>
+ ))}
+ <div className="telemetry-stat telemetry-stat--total">
+ <span className="telemetry-stat__count">{stats.total || 0}</span>
+ <span className="telemetry-stat__label">Total</span>
+ </div>
+ </div>
+ )}
+
+ <div className="telemetry-filters">
+ <button
+ type="button"
+ className="telemetry-filters__toggle"
+ onClick={() => setFiltersExpanded((v) => !v)}
+ >
+ <Filter size={15} />
+ Filters
+ {activeFilterCount > 0 && (
+ <span className="telemetry-filters__count">{activeFilterCount}</span>
+ )}
+ {filtersExpanded ? <ChevronUp size={15} /> : <ChevronDown size={15} />}
+ </button>
+
+ {filtersExpanded && (
+ <div className="telemetry-filters__body">
+ <div className="telemetry-filters__grid">
+ <select
+ value={filters.source}
+ onChange={(e) => handleFilterChange('source', e.target.value)}
+ >
+ <option value="">All sources</option>
+ {sources.map((s) => (
+ <option key={s} value={s}>{s}</option>
+ ))}
+ </select>
+ <select
+ value={filters.event_type}
+ onChange={(e) => handleFilterChange('event_type', e.target.value)}
+ >
+ <option value="">All event types</option>
+ {eventTypes.map((t) => (
+ <option key={t} value={t}>{t}</option>
+ ))}
+ </select>
+ <select
+ value={filters.severity}
+ onChange={(e) => handleFilterChange('severity', e.target.value)}
+ >
+ <option value="">All severities</option>
+ {SEVERITY_ORDER.map((s) => (
+ <option key={s} value={s}>{SEVERITY_CONFIG[s].label}</option>
+ ))}
+ </select>
+ <Input
+ placeholder="Resource type"
+ value={filters.resource_type}
+ onChange={(e) => handleFilterChange('resource_type', e.target.value)}
+ />
+ <Input
+ placeholder="Resource ID"
+ value={filters.resource_id}
+ onChange={(e) => handleFilterChange('resource_id', e.target.value)}
+ />
+ <Input
+ placeholder="Correlation ID"
+ value={filters.correlation_id}
+ onChange={(e) => handleFilterChange('correlation_id', e.target.value)}
+ />
+ <Input
+ type="datetime-local"
+ value={filters.start_date}
+ onChange={(e) => handleFilterChange('start_date', e.target.value)}
+ />
+ <Input
+ type="datetime-local"
+ value={filters.end_date}
+ onChange={(e) => handleFilterChange('end_date', e.target.value)}
+ />
+ </div>
+ <div className="telemetry-filters__search">
+ <Search size={15} />
+ <Input
+ placeholder="Search message..."
+ value={filters.q}
+ onChange={(e) => handleFilterChange('q', e.target.value)}
+ />
+ </div>
+ <div className="telemetry-filters__actions">
+ <Button size="sm" onClick={applyFilters}>Apply</Button>
+ <Button size="sm" variant="ghost" onClick={clearFilters}>
+ <X size={15} /> Clear
+ </Button>
+ </div>
+ </div>
+ )}
+ </div>
+
+ <div className="telemetry-list">
+ {events.map((event) => {
+ const config = SEVERITY_CONFIG[event.severity] || SEVERITY_CONFIG.info;
+ const Icon = config.icon;
+ return (
+ <div
+ key={event.id}
+ className={`telemetry-item telemetry-item--${event.severity}`}
+ onClick={() => setSelectedEvent(event)}
+ >
+ <div className="telemetry-item__severity">
+ <Icon size={16} color={config.color} />
+ </div>
+ <div className="telemetry-item__content">
+ <div className="telemetry-item__header">
+ <span className="telemetry-item__source">{event.source}</span>
+ <span className="telemetry-item__type">{event.event_type}</span>
+ <span className="telemetry-item__time">
+ {new Date(event.timestamp).toLocaleString()}
+ </span>
+ </div>
+ <div className="telemetry-item__message">{event.message || event.event_type}</div>
+ <div className="telemetry-item__meta">
+ {event.resource_type && (
+ <span className="telemetry-item__badge">
+ {event.resource_type}:{event.resource_id}
+ </span>
+ )}
+ {event.actor_username && (
+ <span className="telemetry-item__badge">
+ by {event.actor_username}
+ </span>
+ )}
+ {event.correlation_id && (
+ <button
+ type="button"
+ className="telemetry-item__correlation"
+ onClick={(e) => {
+ e.stopPropagation();
+ handleCorrelationClick(event.correlation_id);
+ }}
+ title="View related events"
+ >
+ related <ChevronRight size={12} />
+ </button>
+ )}
+ </div>
+ </div>
+ </div>
+ );
+ })}
+
+ {events.length === 0 && !loading && (
+ <div className="telemetry-empty">
+ <Info size={32} />
+ <p>No telemetry events match your filters.</p>
+ </div>
+ )}
+
+ {loading && (
+ <div className="telemetry-loading">
+ <Loader2 size={24} className="spin" />
+ </div>
+ )}
+
+ {hasMore && !loading && (
+ <Button variant="outline" className="telemetry-load-more" onClick={loadMore}>
+ Load more
+ </Button>
+ )}
+ </div>
+ </div>
+
+ {selectedEvent && (
+ <div className="telemetry-drawer-backdrop" onClick={() => setSelectedEvent(null)}>
+ <div className="telemetry-drawer" onClick={(e) => e.stopPropagation()}>
+ <div className="telemetry-drawer__header">
+ <h3>Event Details</h3>
+ <Button variant="ghost" size="sm" onClick={() => setSelectedEvent(null)}>
+ <X size={16} />
+ </Button>
+ </div>
+ <div className="telemetry-drawer__body">
+ <div className="telemetry-detail__row">
+ <span>ID</span>
+ <code>{selectedEvent.id}</code>
+ </div>
+ <div className="telemetry-detail__row">
+ <span>Timestamp</span>
+ <span>{new Date(selectedEvent.timestamp).toLocaleString()}</span>
+ </div>
+ <div className="telemetry-detail__row">
+ <span>Source</span>
+ <span>{selectedEvent.source}</span>
+ </div>
+ <div className="telemetry-detail__row">
+ <span>Type</span>
+ <span>{selectedEvent.event_type}</span>
+ </div>
+ <div className="telemetry-detail__row">
+ <span>Severity</span>
+ <span className={`telemetry-detail__severity telemetry-detail__severity--${selectedEvent.severity}`}>
+ {selectedEvent.severity}
+ </span>
+ </div>
+ <div className="telemetry-detail__row">
+ <span>Message</span>
+ <span>{selectedEvent.message || '-'}</span>
+ </div>
+ {selectedEvent.resource_type && (
+ <div className="telemetry-detail__row">
+ <span>Resource</span>
+ <span>{selectedEvent.resource_type}:{selectedEvent.resource_id}</span>
+ </div>
+ )}
+ {selectedEvent.actor_username && (
+ <div className="telemetry-detail__row">
+ <span>Actor</span>
+ <span>{selectedEvent.actor_username}</span>
+ </div>
+ )}
+ {selectedEvent.correlation_id && (
+ <div className="telemetry-detail__row">
+ <span>Correlation</span>
+ <button
+ type="button"
+ className="telemetry-item__correlation"
+ onClick={() => handleCorrelationClick(selectedEvent.correlation_id)}
+ >
+ {selectedEvent.correlation_id} <ChevronRight size={12} />
+ </button>
+ </div>
+ )}
+ <div className="telemetry-detail__payload">
+ <span>Payload</span>
+ <pre>{JSON.stringify(selectedEvent.payload || {}, null, 2)}</pre>
+ </div>
+ </div>
+ </div>
+ </div>
+ )}
+ </>
+ );
+}
diff --git a/frontend/src/services/api/index.js b/frontend/src/services/api/index.js
index 9949c3c0..36e39a06 100644
--- a/frontend/src/services/api/index.js
+++ b/frontend/src/services/api/index.js
@@ -20,6 +20,7 @@ import * as secretsWebhooksMethods from './secretsWebhooks.js';
import * as containerOpsMethods from './containerOps.js';
import * as queueBusMethods from './queueBus.js';
import * as notificationMethods from './notifications.js';
+import * as telemetryMethods from './telemetry.js';
class ApiService extends ApiClient {
constructor() {
@@ -47,6 +48,7 @@ class ApiService extends ApiClient {
containerOpsMethods,
queueBusMethods,
notificationMethods,
+ telemetryMethods,
];
for (const mod of modules) {
for (const [key, fn] of Object.entries(mod)) {
diff --git a/frontend/src/services/api/telemetry.js b/frontend/src/services/api/telemetry.js
new file mode 100644
index 00000000..3f423bec
--- /dev/null
+++ b/frontend/src/services/api/telemetry.js
@@ -0,0 +1,59 @@
+// Telemetry / system event stream API methods
+
+export async function getTelemetryEvents(params = {}) {
+ const query = new URLSearchParams();
+ if (params.page) query.append('page', params.page);
+ if (params.per_page) query.append('per_page', params.per_page);
+ if (params.source) query.append('source', params.source);
+ if (params.event_type) query.append('event_type', params.event_type);
+ if (params.severity) query.append('severity', params.severity);
+ if (params.resource_type) query.append('resource_type', params.resource_type);
+ if (params.resource_id) query.append('resource_id', params.resource_id);
+ if (params.correlation_id) query.append('correlation_id', params.correlation_id);
+ if (params.start_date) query.append('start_date', params.start_date);
+ if (params.end_date) query.append('end_date', params.end_date);
+ if (params.q) query.append('q', params.q);
+ const suffix = query.toString() ? `?${query}` : '';
+ return this.request(`/telemetry/events${suffix}`);
+}
+
+export async function getTelemetryEvent(eventId) {
+ return this.request(`/telemetry/events/${eventId}`);
+}
+
+export async function getTelemetryEventsByCorrelation(correlationId, params = {}) {
+ const query = new URLSearchParams();
+ if (params.limit) query.append('limit', params.limit);
+ const suffix = query.toString() ? `?${query}` : '';
+ return this.request(`/telemetry/events/by-correlation/${correlationId}${suffix}`);
+}
+
+export async function getTelemetryStats(params = {}) {
+ const query = new URLSearchParams();
+ if (params.hours) query.append('hours', params.hours);
+ if (params.source) query.append('source', params.source);
+ const suffix = query.toString() ? `?${query}` : '';
+ return this.request(`/telemetry/stats${suffix}`);
+}
+
+export async function getTelemetrySources() {
+ return this.request('/telemetry/sources');
+}
+
+export async function getTelemetryEventTypes(params = {}) {
+ const query = new URLSearchParams();
+ if (params.source) query.append('source', params.source);
+ const suffix = query.toString() ? `?${query}` : '';
+ return this.request(`/telemetry/event-types${suffix}`);
+}
+
+export async function cleanupTelemetryEvents(days = 90) {
+ return this.request(`/telemetry/events?days=${days}`, { method: 'DELETE' });
+}
+
+export async function emitTestTelemetryEvent(data = {}) {
+ return this.request('/telemetry/events/test', {
+ method: 'POST',
+ body: JSON.stringify(data),
+ });
+}
diff --git a/frontend/src/styles/main.scss b/frontend/src/styles/main.scss
index d8c1a55d..08904e81 100644
--- a/frontend/src/styles/main.scss
+++ b/frontend/src/styles/main.scss
@@ -119,3 +119,4 @@
@import 'pages/_gpu';
@import 'pages/_dynamic-dns';
@import 'pages/_queue-operations';
+@import 'pages/_telemetry';
diff --git a/frontend/src/styles/pages/_telemetry.scss b/frontend/src/styles/pages/_telemetry.scss
new file mode 100644
index 00000000..0dc893ca
--- /dev/null
+++ b/frontend/src/styles/pages/_telemetry.scss
@@ -0,0 +1,359 @@
+.telemetry-page {
+ padding: $spacing-md;
+}
+
+.telemetry-stats {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
+ gap: $spacing-md;
+ margin-bottom: $spacing-lg;
+}
+
+.telemetry-stat {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ padding: $spacing-md;
+ border-radius: $radius-md;
+ background: $bg-card;
+ border: 1px solid $border-default;
+
+ &__count {
+ font-size: 1.75rem;
+ font-weight: 700;
+ color: $text-primary;
+ }
+
+ &__label {
+ font-size: 0.75rem;
+ text-transform: uppercase;
+ letter-spacing: 0.05em;
+ color: $text-tertiary;
+ margin-top: $spacing-xs;
+ }
+
+ &--critical { border-top: 3px solid $danger; }
+ &--error { border-top: 3px solid $danger-hover; }
+ &--warning { border-top: 3px solid $warning; }
+ &--info { border-top: 3px solid $info; }
+ &--debug { border-top: 3px solid $text-tertiary; }
+ &--total { border-top: 3px solid $accent-primary; }
+}
+
+.telemetry-filters {
+ background: $bg-card;
+ border: 1px solid $border-default;
+ border-radius: $radius-md;
+ margin-bottom: $spacing-lg;
+ overflow: hidden;
+
+ &__toggle {
+ display: flex;
+ align-items: center;
+ gap: $spacing-sm;
+ width: 100%;
+ padding: $spacing-md;
+ background: transparent;
+ border: none;
+ color: $text-primary;
+ font-weight: 500;
+ cursor: pointer;
+
+ &:hover {
+ background: $bg-hover;
+ }
+ }
+
+ &__count {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ min-width: 20px;
+ height: 20px;
+ padding: 0 6px;
+ border-radius: $radius-full;
+ background: $accent-primary;
+ color: white;
+ font-size: 0.75rem;
+ font-weight: 600;
+ }
+
+ &__body {
+ padding: $spacing-md;
+ border-top: 1px solid $border-default;
+ }
+
+ &__grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+ gap: $spacing-md;
+ margin-bottom: $spacing-md;
+
+ select,
+ input {
+ width: 100%;
+ padding: $spacing-sm;
+ border-radius: $radius-sm;
+ border: 1px solid $border-default;
+ background: $bg-card;
+ color: $text-primary;
+ font-size: 0.875rem;
+ }
+ }
+
+ &__search {
+ position: relative;
+ display: flex;
+ align-items: center;
+ margin-bottom: $spacing-md;
+
+ svg {
+ position: absolute;
+ left: $spacing-sm;
+ color: $text-tertiary;
+ }
+
+ input {
+ width: 100%;
+ padding-left: 2rem;
+ }
+ }
+
+ &__actions {
+ display: flex;
+ gap: $spacing-sm;
+ }
+}
+
+.telemetry-list {
+ display: flex;
+ flex-direction: column;
+ gap: $spacing-sm;
+}
+
+.telemetry-item {
+ display: flex;
+ align-items: flex-start;
+ gap: $spacing-md;
+ padding: $spacing-md;
+ background: $bg-card;
+ border: 1px solid $border-default;
+ border-radius: $radius-md;
+ cursor: pointer;
+ transition: background $transition-fast, border-color $transition-fast;
+
+ &:hover {
+ background: $bg-hover;
+ border-color: $border-hover;
+ }
+
+ &__severity {
+ flex-shrink: 0;
+ margin-top: 2px;
+ }
+
+ &__content {
+ flex: 1;
+ min-width: 0;
+ }
+
+ &__header {
+ display: flex;
+ align-items: center;
+ gap: $spacing-sm;
+ flex-wrap: wrap;
+ margin-bottom: $spacing-xs;
+ }
+
+ &__source {
+ font-size: 0.75rem;
+ font-weight: 600;
+ text-transform: uppercase;
+ letter-spacing: 0.05em;
+ color: $text-tertiary;
+ }
+
+ &__type {
+ font-size: 0.75rem;
+ font-weight: 500;
+ color: $accent-primary;
+ background: $accent-bg-soft;
+ padding: 2px 8px;
+ border-radius: $radius-sm;
+ }
+
+ &__time {
+ font-size: 0.75rem;
+ color: $text-tertiary;
+ margin-left: auto;
+ }
+
+ &__message {
+ font-size: 0.875rem;
+ color: $text-primary;
+ margin-bottom: $spacing-xs;
+ word-break: break-word;
+ }
+
+ &__meta {
+ display: flex;
+ align-items: center;
+ gap: $spacing-sm;
+ flex-wrap: wrap;
+ }
+
+ &__badge {
+ font-size: 0.75rem;
+ color: $text-tertiary;
+ background: $bg-hover;
+ padding: 2px 8px;
+ border-radius: $radius-sm;
+ }
+
+ &__correlation {
+ display: inline-flex;
+ align-items: center;
+ gap: 2px;
+ font-size: 0.75rem;
+ color: $accent-primary;
+ background: transparent;
+ border: none;
+ padding: 0;
+ cursor: pointer;
+
+ &:hover {
+ text-decoration: underline;
+ }
+ }
+
+ &--critical { border-left: 3px solid $danger; }
+ &--error { border-left: 3px solid $danger-hover; }
+ &--warning { border-left: 3px solid $warning; }
+ &--info { border-left: 3px solid $info; }
+ &--debug { border-left: 3px solid $text-tertiary; }
+}
+
+.telemetry-empty,
+.telemetry-loading {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ padding: $spacing-xl;
+ color: $text-tertiary;
+ gap: $spacing-md;
+}
+
+.telemetry-load-more {
+ align-self: center;
+ margin-top: $spacing-md;
+}
+
+.telemetry-drawer-backdrop {
+ position: fixed;
+ inset: 0;
+ background: rgba(0, 0, 0, 0.5);
+ z-index: $z-modal;
+ display: flex;
+ justify-content: flex-end;
+}
+
+.telemetry-drawer {
+ width: 100%;
+ max-width: 560px;
+ height: 100%;
+ background: $bg-card;
+ border-left: 1px solid $border-default;
+ display: flex;
+ flex-direction: column;
+ overflow: hidden;
+
+ &__header {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ padding: $spacing-md;
+ border-bottom: 1px solid $border-default;
+
+ h3 {
+ margin: 0;
+ font-size: 1rem;
+ font-weight: 600;
+ }
+ }
+
+ &__body {
+ flex: 1;
+ overflow-y: auto;
+ padding: $spacing-md;
+ }
+}
+
+.telemetry-detail {
+ &__row {
+ display: flex;
+ justify-content: space-between;
+ align-items: flex-start;
+ gap: $spacing-md;
+ padding: $spacing-sm 0;
+ border-bottom: 1px solid $border-default;
+ font-size: 0.875rem;
+
+ span:first-child {
+ color: $text-tertiary;
+ flex-shrink: 0;
+ }
+
+ span:last-child,
+ code {
+ color: $text-primary;
+ word-break: break-word;
+ text-align: right;
+ }
+ }
+
+ &__severity {
+ text-transform: uppercase;
+ font-size: 0.75rem;
+ font-weight: 600;
+ padding: 2px 8px;
+ border-radius: $radius-sm;
+
+ &--critical { background: $danger-bg; color: $danger; }
+ &--error { background: rgba($danger-hover, 0.15); color: $danger-hover; }
+ &--warning { background: $warning-bg; color: $warning; }
+ &--info { background: $info-bg; color: $info; }
+ &--debug { background: rgba($text-tertiary-raw, 0.15); color: $text-tertiary; }
+ }
+
+ &__payload {
+ margin-top: $spacing-md;
+
+ span {
+ display: block;
+ font-size: 0.875rem;
+ color: $text-tertiary;
+ margin-bottom: $spacing-sm;
+ }
+
+ pre {
+ background: $bg-tertiary;
+ border: 1px solid $border-default;
+ border-radius: $radius-md;
+ padding: $spacing-md;
+ overflow-x: auto;
+ font-size: 0.8125rem;
+ color: $text-primary;
+ }
+ }
+}
+
+.spin {
+ animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+ from { transform: rotate(0deg); }
+ to { transform: rotate(360deg); }
+}