Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import { apiFetchJSON, type APIContextOptions } from '@/utils/apiClient';
import {
arrayOrEmpty,
finiteNumberOrUndefined,
strictBoolean,
strictString,
stringArray,
} from './responseUtils';
export interface EmailProvider {
id?: string;
name: string;
smtpHost: string;
smtpPort: number;
tls: boolean;
startTLS: boolean;
authRequired: boolean;
instructions: string;
server?: string;
port?: number;
security?: 'none' | 'tls' | 'starttls';
}
export interface WebhookTemplate {
id?: string;
service: string;
label?: string;
mentionPlaceholder?: string;
mentionHelp?: string;
name: string;
urlPattern: string;
method: string;
headers: Record<string, string>;
payloadTemplate: string;
instructions: string;
description?: string;
template?: {
url?: string;
method?: string;
headers?: Record<string, string>;
body?: string;
};
}
export type NotificationMinimumSeverity = 'all' | 'warning' | 'critical';
export interface EmailConfig {
enabled: boolean;
provider: string;
server: string;
port: number;
username: string;
password?: string;
from: string;
to: string[];
tls: boolean;
startTLS: boolean;
rateLimit?: number;
tagFilter?: string[];
tagFilterMode?: 'all' | 'any';
minimumSeverity?: NotificationMinimumSeverity;
}
export interface Webhook {
id: string;
name: string;
url: string;
method: string;
headers: Record<string, string>;
template?: string;
enabled: boolean;
service?: string; // Added to support Discord, Slack, etc.
customFields?: Record<string, string>;
mention?: string; // Platform-specific mention (e.g., @everyone, @channel, <@USER_ID>)
tagFilter?: string[];
tagFilterMode?: 'all' | 'any';
minimumSeverity?: NotificationMinimumSeverity;
}
export interface AppriseConfig {
enabled: boolean;
mode?: 'cli' | 'http';
targets?: string[];
cliPath?: string;
timeoutSeconds?: number;
serverUrl?: string;
configKey?: string;
apiKey?: string;
apiKeyHeader?: string;
skipTlsVerify?: boolean;
minimumSeverity?: NotificationMinimumSeverity;
// Responses never include the stored apiKey; this reports whether one is saved.
hasApiKey?: boolean;
}
export interface NotificationTestRequest {
type: 'email' | 'webhook' | 'apprise';
config?: Record<string, unknown> | AppriseConfig; // Backend expects different format than frontend types
webhookId?: string;
}
export type NotificationQueueHealthStatus = 'healthy' | 'degraded' | 'unavailable';
export type NotificationFailureClass =
| 'authentication'
| 'rate_limited'
| 'connectivity'
| 'tls'
| 'configuration'
| 'rejected'
| 'server_error'
| 'unknown';
export type NotificationFailureClassCounts = Record<NotificationFailureClass, number>;
export interface NotificationQueueHealth {
pending: number;
sending: number;
sent: number;
failed: number;
deadLetter: number;
healthy: boolean;
status: NotificationQueueHealthStatus;
attentionRequired: number;
reasonCodes: string[];
completedRetentionDays: number;
deadLetterRetentionDays: number;
countsAreRetentionBounded: boolean;
retryAttemptsAffectHealth: boolean;
terminalFailuresAffectHealth: boolean;
failureClasses7d: NotificationFailureClassCounts;
failureClassesAvailable: boolean;
failureClassWindowDays: number;
}
export interface NotificationHealth {
overallHealthy: boolean;
queue: NotificationQueueHealth;
}
export interface NotificationTerminalFailureActionResult {
success: boolean;
affected: number;
}
export type NotificationDeliveryOutcome = 'sent' | 'retry' | 'failed' | 'dead_letter' | 'cancelled';
export interface NotificationDeliveryLogEntry {
notificationId: string;
type: string;
method?: string;
destinationId?: string;
outcome: NotificationDeliveryOutcome;
alertIds: string[];
alertCount: number;
attempts: number;
success: boolean;
errorMessage?: string;
failureClass?: NotificationFailureClass;
timestamp: string;
}
export interface NotificationDeliveryLog {
entries: NotificationDeliveryLogEntry[];
windowDays: number;
completedRetentionDays: number;
deadLetterRetentionDays: number;
}
export interface NotificationTestResult {
status?: string;
message?: string;
// True when the test was delivered but real alert delivery is paused by the
// activation gate, so a successful test must not be read as proof that live
// alerts are getting through.
deliveryPaused?: boolean;
}
function apiRecord(value: unknown): Record<string, unknown> {
return value && typeof value === 'object' ? (value as Record<string, unknown>) : {};
}
function normalizeQueueHealthStatus(value: unknown): NotificationQueueHealthStatus {
return value === 'healthy' || value === 'degraded' ? value : 'unavailable';
}
function nonNegativeCount(value: unknown): number | undefined {
const normalized = finiteNumberOrUndefined(value);
return normalized !== undefined && Number.isInteger(normalized) && normalized >= 0
? normalized
: undefined;
}
const notificationFailureClasses: NotificationFailureClass[] = [
'authentication',
'rate_limited',
'connectivity',
'tls',
'configuration',
'rejected',
'server_error',
'unknown',
];
const notificationDeliveryOutcomes: NotificationDeliveryOutcome[] = [
'sent',
'retry',
'failed',
'dead_letter',
'cancelled',
];
function normalizeDeliveryLogEntry(value: unknown): NotificationDeliveryLogEntry | undefined {
const record = apiRecord(value);
const notificationId = strictString(record.notificationId);
const outcome = record.outcome;
const timestamp = strictString(record.timestamp);
if (
!notificationId ||
!timestamp ||
!notificationDeliveryOutcomes.includes(outcome as NotificationDeliveryOutcome)
) {
return undefined;
}
const failureClass = record.failureClass;
const entry: NotificationDeliveryLogEntry = {
notificationId,
type: strictString(record.type),
outcome: outcome as NotificationDeliveryOutcome,
alertIds: stringArray(record.alertIds),
alertCount: nonNegativeCount(record.alertCount) ?? 0,
attempts: nonNegativeCount(record.attempts) ?? 0,
success: strictBoolean(record.success),
timestamp,
};
const method = strictString(record.method);
if (method) entry.method = method;
const destinationId = strictString(record.destinationId);
if (destinationId) entry.destinationId = destinationId;
const errorMessage = strictString(record.errorMessage);
if (errorMessage) entry.errorMessage = errorMessage;
if (notificationFailureClasses.includes(failureClass as NotificationFailureClass)) {
entry.failureClass = failureClass as NotificationFailureClass;
}
return entry;
}
function normalizeFailureClassCounts(value: unknown): NotificationFailureClassCounts | undefined {
const record = apiRecord(value);
const counts = {} as NotificationFailureClassCounts;
for (const failureClass of notificationFailureClasses) {
const count = nonNegativeCount(record[failureClass]);
if (count === undefined) return undefined;
counts[failureClass] = count;
}
return counts;
}
export class NotificationsAPI {
private static baseUrl = '/api/notifications';
static async getAppriseConfig(options?: APIContextOptions): Promise<AppriseConfig> {
return apiFetchJSON(`${this.baseUrl}/apprise`, options);
}
static async getHealth(options?: APIContextOptions): Promise<NotificationHealth> {
const payload = await apiFetchJSON<Record<string, unknown>>(`${this.baseUrl}/health`, options);
const queue = apiRecord(payload.queue);
const pending = nonNegativeCount(queue.pending);
const sending = nonNegativeCount(queue.sending);
const sent = nonNegativeCount(queue.sent);
const failed = nonNegativeCount(queue.failed);
const deadLetter = nonNegativeCount(queue.dlq);
const attentionRequired = nonNegativeCount(queue.attention_required);
const completedRetentionDays = nonNegativeCount(queue.completed_retention_days);
const deadLetterRetentionDays = nonNegativeCount(queue.dead_letter_retention_days);
const failureClasses7d = normalizeFailureClassCounts(queue.failure_classes_7d);
const failureClassWindowDays = nonNegativeCount(queue.failure_class_window_days);
const failureClassesAvailable = strictBoolean(queue.failure_classes_available);
const reasonCodesAreValid =
Array.isArray(queue.reason_codes) &&
queue.reason_codes.every((reasonCode) => typeof reasonCode === 'string');
const semanticsAreValid =
queue.counts_are_retention_bounded === true &&
queue.retry_attempts_affect_health === false &&
queue.terminal_failures_affect_health === true;
const availableFieldsAreValid =
pending !== undefined &&
sending !== undefined &&
sent !== undefined &&
attentionRequired !== undefined &&
completedRetentionDays !== undefined &&
completedRetentionDays > 0 &&
deadLetterRetentionDays !== undefined &&
deadLetterRetentionDays > 0 &&
failureClasses7d !== undefined &&
failureClassWindowDays !== undefined &&
failureClassWindowDays > 0 &&
reasonCodesAreValid &&
semanticsAreValid;
const rawStatus = normalizeQueueHealthStatus(queue.status);
const rawHealthy = typeof queue.healthy === 'boolean' ? queue.healthy : undefined;
const terminalFailureCount =
failed !== undefined && deadLetter !== undefined ? failed + deadLetter : undefined;
const statusIsConsistent =
rawStatus === 'unavailable' ||
(rawStatus === 'healthy' &&
availableFieldsAreValid &&
rawHealthy === true &&
terminalFailureCount === 0 &&
attentionRequired === 0) ||
(rawStatus === 'degraded' &&
availableFieldsAreValid &&
rawHealthy === false &&
terminalFailureCount !== undefined &&
terminalFailureCount > 0 &&
attentionRequired === terminalFailureCount);
const status = statusIsConsistent ? rawStatus : 'unavailable';
return {
overallHealthy: status === 'healthy' && strictBoolean(payload.overall_healthy),
queue: {
pending: pending ?? 0,
sending: sending ?? 0,
sent: sent ?? 0,
failed: failed ?? 0,
deadLetter: deadLetter ?? 0,
healthy: status === 'healthy',
status,
attentionRequired: attentionRequired ?? 0,
reasonCodes: stringArray(queue.reason_codes),
completedRetentionDays: completedRetentionDays ?? 7,
deadLetterRetentionDays: deadLetterRetentionDays ?? 30,
countsAreRetentionBounded: strictBoolean(queue.counts_are_retention_bounded),
retryAttemptsAffectHealth: strictBoolean(queue.retry_attempts_affect_health),
terminalFailuresAffectHealth: strictBoolean(queue.terminal_failures_affect_health, true),
failureClasses7d:
failureClasses7d ??
({
authentication: 0,
rate_limited: 0,
connectivity: 0,
tls: 0,
configuration: 0,
rejected: 0,
server_error: 0,
unknown: 0,
} satisfies NotificationFailureClassCounts),
failureClassesAvailable,
failureClassWindowDays: failureClassWindowDays ?? 7,
},
};
}
static async getDeliveryLog(
limit?: number,
options?: APIContextOptions,
): Promise<NotificationDeliveryLog> {
const query = limit && limit > 0 ? `?limit=${limit}` : '';
const payload = await apiFetchJSON<Record<string, unknown>>(
`${this.baseUrl}/delivery-log${query}`,
options,
);
const rawEntries = arrayOrEmpty<unknown>(payload.entries);
const entries: NotificationDeliveryLogEntry[] = [];
for (const rawEntry of rawEntries) {
const entry = normalizeDeliveryLogEntry(rawEntry);
if (entry) entries.push(entry);
}
const windowDays = nonNegativeCount(payload.window_days);
const normalizedWindowDays = windowDays && windowDays > 0 ? windowDays : 7;
const completedRetentionDays = nonNegativeCount(payload.completed_retention_days);
const deadLetterRetentionDays = nonNegativeCount(payload.dead_letter_retention_days);
return {
entries,
windowDays: normalizedWindowDays,
// Older servers exposed only window_days plus the dead-letter window.
// Preserve an honest seven-day completed-row fallback while accepting
// their explicit 30-day dead-letter retention field.
completedRetentionDays:
completedRetentionDays && completedRetentionDays > 0
? completedRetentionDays
: Math.min(normalizedWindowDays, 7),
deadLetterRetentionDays:
deadLetterRetentionDays && deadLetterRetentionDays > 0
? deadLetterRetentionDays
: normalizedWindowDays,
};
}
static async retryTerminalFailures(
options?: APIContextOptions,
): Promise<NotificationTerminalFailureActionResult> {
const payload = await apiFetchJSON<Record<string, unknown>>(
`${this.baseUrl}/terminal-failures/retry`,
{ ...options, method: 'POST' },
);
return {
success: strictBoolean(payload.success),
affected: nonNegativeCount(payload.affected) ?? 0,
};
}
static async dismissTerminalFailures(
options?: APIContextOptions,
): Promise<NotificationTerminalFailureActionResult> {
const payload = await apiFetchJSON<Record<string, unknown>>(
`${this.baseUrl}/terminal-failures/dismiss`,
{ ...options, method: 'POST' },
);
return {
success: strictBoolean(payload.success),
affected: nonNegativeCount(payload.affected) ?? 0,
};
}
static async updateAppriseConfig(
config: AppriseConfig,
options?: APIContextOptions,
): Promise<AppriseConfig> {
return apiFetchJSON(`${this.baseUrl}/apprise`, {
...options,
method: 'PUT',
body: JSON.stringify(config),
});
}
// Email configuration
static async getEmailConfig(options?: APIContextOptions): Promise<EmailConfig> {
const backendConfig = await apiFetchJSON<Record<string, unknown>>(
`${this.baseUrl}/email`,
options,
);
const port = finiteNumberOrUndefined(backendConfig.port);
// Backend already returns fields with correct names (server, port)
const config: EmailConfig = {
enabled: strictBoolean(backendConfig.enabled),
provider: strictString(backendConfig.provider),
server: strictString(backendConfig.server),
port: port ?? 587,
username: strictString(backendConfig.username),
password: strictString(backendConfig.password),
from: strictString(backendConfig.from),
to: stringArray(backendConfig.to),
tls: strictBoolean(backendConfig.tls),
startTLS: strictBoolean(backendConfig.startTLS),
rateLimit: finiteNumberOrUndefined(backendConfig.rateLimit),
};
if (Array.isArray(backendConfig.tagFilter)) {
config.tagFilter = stringArray(backendConfig.tagFilter);
}
if (typeof backendConfig.tagFilterMode === 'string') {
config.tagFilterMode = backendConfig.tagFilterMode === 'any' ? 'any' : 'all';
}
config.minimumSeverity =
backendConfig.minimumSeverity === 'critical' || backendConfig.minimumSeverity === 'warning'
? backendConfig.minimumSeverity
: 'all';
return config;
}
static async updateEmailConfig(
config: EmailConfig,
options?: APIContextOptions,
): Promise<{ success: boolean }> {
// Backend expects fields with these names (server, port)
const backendConfig: Record<string, unknown> = {
enabled: config.enabled,
server: config.server,
port: config.port,
username: config.username,
password: config.password,
from: config.from,
to: config.to,
tls: config.tls,
startTLS: config.startTLS,
provider: config.provider,
};
// Only include rateLimit if it's explicitly set
if (config.rateLimit !== undefined) {
backendConfig.rateLimit = config.rateLimit;
}
if (config.tagFilter !== undefined) {
backendConfig.tagFilter = config.tagFilter;
}
if (config.tagFilterMode !== undefined) {
backendConfig.tagFilterMode = config.tagFilterMode === 'any' ? 'any' : 'all';
}
if (config.minimumSeverity !== undefined) {
backendConfig.minimumSeverity =
config.minimumSeverity === 'critical' || config.minimumSeverity === 'warning'
? config.minimumSeverity
: 'all';
}
return apiFetchJSON(`${this.baseUrl}/email`, {
...options,
method: 'PUT',
body: JSON.stringify(backendConfig),
});
}
// Webhook management
static async getWebhooks(options?: APIContextOptions): Promise<Webhook[]> {
const data = await apiFetchJSON<Webhook[] | null>(`${this.baseUrl}/webhooks`, options);
return arrayOrEmpty<Webhook>(data);
}
static async createWebhook(
webhook: Omit<Webhook, 'id'>,
options?: APIContextOptions,
): Promise<Webhook> {
return apiFetchJSON(`${this.baseUrl}/webhooks`, {
...options,
method: 'POST',
body: JSON.stringify(webhook),
});
}
static async updateWebhook(
id: string,
webhook: Partial<Webhook>,
options?: APIContextOptions,
): Promise<Webhook> {
return apiFetchJSON(`${this.baseUrl}/webhooks/${encodeURIComponent(id)}`, {
...options,
method: 'PUT',
body: JSON.stringify(webhook),
});
}
static async deleteWebhook(
id: string,
options?: APIContextOptions,
): Promise<{ success: boolean }> {
return apiFetchJSON(`${this.baseUrl}/webhooks/${encodeURIComponent(id)}`, {
...options,
method: 'DELETE',
});
}
// Templates and providers
static async getEmailProviders(): Promise<EmailProvider[]> {
return apiFetchJSON(`${this.baseUrl}/email-providers`);
}
static async getWebhookTemplates(): Promise<WebhookTemplate[]> {
return apiFetchJSON(`${this.baseUrl}/webhook-templates`);
}
// Testing
static async testNotification(
request: NotificationTestRequest,
options?: APIContextOptions,
): Promise<NotificationTestResult> {
const body: {
method: string;
config?: Record<string, unknown> | AppriseConfig;
webhookId?: string;
} = {
method: request.type,
};
// Include config if provided for testing without saving
if (request.config) {
body.config = request.config;
}
// Include webhookId for webhook testing
if (request.webhookId) {
body.webhookId = request.webhookId;
}
return apiFetchJSON(`${this.baseUrl}/test`, {
...options,
method: 'POST',
body: JSON.stringify(body),
});
}
static async testWebhook(
webhook: Omit<Webhook, 'id'>,
options?: APIContextOptions,
): Promise<{ success?: boolean; message?: string; deliveryPaused?: boolean }> {
return apiFetchJSON(`${this.baseUrl}/webhooks/test`, {
...options,
method: 'POST',
body: JSON.stringify(webhook),
});
}
}