Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Fix the systemic silent failures: empty log, dead WS, locked-out agents

Juan Denis5541504ef8a8
diff --git a/agent/internal/logger/logger.go b/agent/internal/logger/logger.go
index 7f65bc80..79d1eb56 100644
--- a/agent/internal/logger/logger.go
+++ b/agent/internal/logger/logger.go
@@ -1,15 +1,43 @@
package logger
import (
+ "fmt"
"io"
"os"
"path/filepath"
+ "sync"
"github.com/serverkit/agent/internal/config"
"gopkg.in/natefinch/lumberjack.v2"
"log/slog"
)
+// resilientWriter wraps multiple io.Writers and writes to all of them
+// independently, swallowing per-writer errors. The standard library's
+// io.MultiWriter aborts the entire write chain on the first error,
+// which is a real bug on Windows services: a single transient stdout
+// write failure (closed handle, ERROR_NO_DATA, etc.) blocks the
+// lumberjack file writer from ever getting subsequent log records.
+// That's how agent.log ended up at 0 bytes while events.json worked
+// fine — events.json uses a direct os.File handle, not slog's
+// handler chain.
+type resilientWriter struct {
+ mu sync.Mutex
+ writers []io.Writer
+}
+
+func (r *resilientWriter) Write(p []byte) (int, error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for _, w := range r.writers {
+ // Best-effort: write to each writer, but never let one writer's
+ // failure cascade to another. Returns len(p), nil unconditionally
+ // so slog never sees a write error and disables itself.
+ _, _ = w.Write(p)
+ }
+ return len(p), nil
+}
+
// Logger wraps slog.Logger with additional context
type Logger struct {
*slog.Logger
@@ -41,34 +69,60 @@ func New(cfg config.LoggingConfig) *Logger {
}
var writers []io.Writer
+ // Capture init errors so we can write them somewhere visible after
+ // the logger is constructed — silently swallowing them was how we
+ // got 0-byte agent.log files in the first place.
+ var initErrs []string
- // Always write to stdout
+ // Always write to stdout (services on Windows: still safe even if
+ // the write goes nowhere — resilientWriter swallows per-writer
+ // errors).
writers = append(writers, os.Stdout)
- // Also write to file if configured
+ // Also write to file if configured.
var rotator *lumberjack.Logger
if cfg.File != "" {
- // Ensure log directory exists
dir := filepath.Dir(cfg.File)
- if err := os.MkdirAll(dir, 0755); err == nil {
- // Use lumberjack for log rotation
- rotator = &lumberjack.Logger{
- Filename: cfg.File,
- MaxSize: cfg.MaxSize, // megabytes
- MaxBackups: cfg.MaxBackups,
- MaxAge: cfg.MaxAge, // days
- Compress: cfg.Compress,
+ if err := os.MkdirAll(dir, 0755); err != nil {
+ initErrs = append(initErrs, fmt.Sprintf("MkdirAll(%q): %v", dir, err))
+ } else {
+ // Probe file writability before handing the path to lumberjack
+ // so a permission error gets surfaced at boot instead of
+ // silently disabling file logging.
+ if f, perr := os.OpenFile(cfg.File, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); perr != nil {
+ initErrs = append(initErrs, fmt.Sprintf("OpenFile(%q): %v", cfg.File, perr))
+ } else {
+ _ = f.Close()
+ rotator = &lumberjack.Logger{
+ Filename: cfg.File,
+ MaxSize: cfg.MaxSize, // megabytes
+ MaxBackups: cfg.MaxBackups,
+ MaxAge: cfg.MaxAge, // days
+ Compress: cfg.Compress,
+ }
+ writers = append(writers, rotator)
}
- writers = append(writers, rotator)
}
}
- // Create multi-writer
- multiWriter := io.MultiWriter(writers...)
-
- handler := slog.NewJSONHandler(multiWriter, opts)
+ multi := &resilientWriter{writers: writers}
+ handler := slog.NewJSONHandler(multi, opts)
logger := slog.New(handler)
+ // If rotator wired up successfully, drop a startup marker so the
+ // file is never confused with "logger broken" — empty file is the
+ // failure mode we just fixed, so always emit one explicit line.
+ if rotator != nil {
+ logger.Info("logger initialized",
+ "file", cfg.File, "level", cfg.Level,
+ "max_size_mb", cfg.MaxSize, "max_backups", cfg.MaxBackups)
+ }
+ for _, e := range initErrs {
+ // Visible on stdout AND via slog (which still has rotator==nil
+ // in this branch — best-effort visibility either way).
+ logger.Warn("logger init issue", "error", e)
+ }
+
return &Logger{Logger: logger, rotator: rotator}
}
diff --git a/agent/internal/ws/client.go b/agent/internal/ws/client.go
index 22617775..1cf84c5b 100644
--- a/agent/internal/ws/client.go
+++ b/agent/internal/ws/client.go
@@ -482,6 +482,27 @@ func (c *Client) Run(ctx context.Context) error {
// readLoop reads Socket.IO messages from the WebSocket
func (c *Client) readLoop(ctx context.Context) error {
+ // Read deadline = pingInterval * 2.5 + 5s grace. Any inbound message
+ // (server ping, app frame, WS-level pong) resets the deadline. If
+ // nothing arrives within this window, the connection is presumed
+ // dead at the network layer and the read errors out, which is what
+ // the Manager's flap detector relies on. Without this, CF/ngrok
+ // killing the connection at the network layer (without sending a
+ // WS close frame) makes ReadMessage hang for minutes — exactly the
+ // "ws_disconnected every 2-3 min" pattern in the user's events.json.
+ readWait := c.pingInterval*5/2 + 5*time.Second
+ if readWait < 30*time.Second {
+ readWait = 30 * time.Second
+ }
+ resetDeadline := func() {
+ _ = c.conn.SetReadDeadline(time.Now().Add(readWait))
+ }
+ resetDeadline()
+ c.conn.SetPongHandler(func(string) error {
+ resetDeadline()
+ return nil
+ })
+
for {
select {
case <-ctx.Done():
@@ -493,6 +514,8 @@ func (c *Client) readLoop(ctx context.Context) error {
if err != nil {
return fmt.Errorf("read error: %w", err)
}
+ // Any successful read = liveness signal. Reset the window.
+ resetDeadline()
msgStr := string(msg)
diff --git a/backend/app/api/servers.py b/backend/app/api/servers.py
index 4aec6149..1d0222a6 100644
--- a/backend/app/api/servers.py
+++ b/backend/app/api/servers.py
@@ -240,11 +240,17 @@ def create_server():
# Generate registration token
registration_token = Server.generate_registration_token()
- # Get permissions from profile or custom list
- permissions = data.get('permissions', [])
+ # Get permissions from profile or custom list. Default is `['*']`
+ # (full access) — ServerKit is single-tenant by default and the
+ # previous default of `[]` left every new server 403'ing on every
+ # action. Users who want a locked-down ACL pass an explicit list
+ # or `permission_profile`.
+ permissions = data.get('permissions')
profile = data.get('permission_profile')
if profile and profile in PERMISSION_PROFILES:
permissions = PERMISSION_PROFILES[profile]['permissions']
+ elif permissions is None:
+ permissions = ['*']
server = Server(
name=data['name'],
diff --git a/backend/migrations/versions/006_default_full_permissions.py b/backend/migrations/versions/006_default_full_permissions.py
new file mode 100644
index 00000000..41641147
--- /dev/null
+++ b/backend/migrations/versions/006_default_full_permissions.py
@@ -0,0 +1,56 @@
+"""Grant ['*'] to servers stuck with empty permissions.
+
+Single-tenant deployments registered servers with the prior default
+of `permissions = []`, which made every action 403. The new default
+in api/servers.py is `['*']`. This migration retroactively fixes
+already-registered servers — but only when their permissions list
+is empty / NULL, so admin-defined ACLs are left untouched.
+
+Revision ID: 006_default_full_permissions
+Revises: 005_server_capabilities_cache
+Create Date: 2026-05-01
+"""
+
+from alembic import op
+import sqlalchemy as sa
+import json
+
+revision = '006_default_full_permissions'
+down_revision = '005_server_capabilities_cache'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ if 'servers' not in inspector.get_table_names():
+ return
+ cols = {c['name'] for c in inspector.get_columns('servers')}
+ if 'permissions' not in cols:
+ return
+
+ # Iterate rows; SQLAlchemy's JSON column round-trips as text in some
+ # backends. Normalize to a list, check emptiness, write back.
+ rows = conn.execute(sa.text("SELECT id, permissions FROM servers")).fetchall()
+ for row in rows:
+ raw = row.permissions
+ try:
+ current = json.loads(raw) if isinstance(raw, str) else (raw or [])
+ except (ValueError, TypeError):
+ current = []
+ if not isinstance(current, list):
+ current = []
+ if len(current) == 0:
+ # Stuck row — grant full access. Admin-curated lists with
+ # any entry are untouched.
+ conn.execute(
+ sa.text("UPDATE servers SET permissions = :p WHERE id = :id"),
+ {'p': json.dumps(['*']), 'id': row.id},
+ )
+
+
+def downgrade():
+ # Intentionally a no-op — we can't tell which `['*']` rows were
+ # originally empty vs. explicitly set. Revert manually if needed.
+ pass