Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

8.8 KB
Raw Blame
package api

// Regression coverage for issue #1640: POST /api/ai/patrol/readiness runs up
// to four sequential provider calls (~45s and more on slow local hardware)
// and previously wrote nothing to the response until the evaluation finished,
// so any reverse proxy with a ~30-second read timeout severed the request.
// The handler now commits a 200 up front and streams JSON-whitespace
// keepalives while the evaluation runs; these tests pin that transport shape.

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// firstByte reports the first byte of a body in a form that is safe to print
// when the body is empty, so a transport regression fails the assertion
// instead of panicking inside it.
func firstByte(body string) string {
	if body == "" {
		return "<empty body>"
	}
	return body[:1]
}

func passingReadinessResult(summary string) ai.PatrolModelReadinessResult {
	out := ai.PatrolModelReadinessResult{}
	out.ProbeVersion = ai.PatrolModelReadinessProbeVersion
	out.Status = ai.PatrolModelReadinessPass
	out.Summary = summary
	return out
}

func TestIssue1640KeepalivesFlowWhileEvaluationRuns(t *testing.T) {
	t.Parallel()

	rec := httptest.NewRecorder()
	result := streamPatrolModelReadinessKeepalives(rec, 5*time.Millisecond, func() ai.PatrolModelReadinessResult {
		time.Sleep(60 * time.Millisecond)
		return passingReadinessResult("verified")
	})
	response := patrolModelReadinessSnapshot(&result, time.Now())
	require.NoError(t, utils.WriteJSONResponse(rec, response))

	body := rec.Body.String()
	// Bytes were on the wire before the evaluation completed, and they were
	// flushed so intermediaries actually observed them.
	assert.True(t, strings.HasPrefix(body, "\n"), "expected keepalive padding before the payload, got %q", firstByte(body))
	assert.GreaterOrEqual(t, strings.Count(body, "\n")-strings.Count(strings.TrimLeft(body, "\n"), "\n"), 2,
		"expected multiple keepalives during a slow evaluation")
	assert.True(t, rec.Flushed, "keepalives must be flushed, not buffered")

	// The padding is insignificant JSON whitespace: the final payload still
	// parses as a single ordinary JSON document.
	var snapshot PatrolModelReadinessSnapshot
	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &snapshot))
	require.NotNil(t, snapshot.PatrolModelReadinessResult)
	assert.Equal(t, "verified", snapshot.Summary)
}

func TestIssue1640FastEvaluationEmitsNoPadding(t *testing.T) {
	t.Parallel()

	rec := httptest.NewRecorder()
	result := streamPatrolModelReadinessKeepalives(rec, time.Hour, func() ai.PatrolModelReadinessResult {
		out := ai.PatrolModelReadinessResult{}
		out.ProbeVersion = ai.PatrolModelReadinessProbeVersion
		return out
	})
	response := patrolModelReadinessSnapshot(&result, time.Now())
	require.NoError(t, utils.WriteJSONResponse(rec, response))

	assert.True(t, strings.HasPrefix(rec.Body.String(), "{"),
		"a fast evaluation should produce a plain JSON body")
}

// The transport must survive a real HTTP connection, not just a recorder: the
// headers and keepalive bytes must reach the client while evaluation is still
// running, rather than remaining buffered until the final JSON response.
func TestIssue1640ResponseStartsBeforeEvaluationCompletes(t *testing.T) {
	t.Parallel()

	// This is a deadlock guard, not a latency SLO. Completion is controlled
	// by the client observing flushed headers and a keepalive, not by a sleep.
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	releaseEvaluation := make(chan struct{})
	var releaseOnce sync.Once
	release := func() { releaseOnce.Do(func() { close(releaseEvaluation) }) }
	started := make(chan struct{})
	completed := make(chan struct{})

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		result := streamPatrolModelReadinessKeepalives(w, 50*time.Millisecond, func() ai.PatrolModelReadinessResult {
			close(started)
			<-releaseEvaluation
			close(completed)
			return passingReadinessResult("verified over the wire")
		})
		response := patrolModelReadinessSnapshot(&result, time.Now())
		if err := utils.WriteJSONResponse(w, response); err != nil {
			t.Errorf("write readiness response: %v", err)
		}
	}))
	defer server.Close()
	// Release before Close even on a failed assertion, so a transport
	// regression cannot strand the evaluator or hang server cleanup.
	defer release()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
	require.NoError(t, err)
	resp, err := server.Client().Do(req)
	require.NoError(t, err)
	defer func() { _ = resp.Body.Close() }()

	// Response headers alone prove the status line was committed early: the
	// client's Get returns as soon as they arrive.
	assert.Equal(t, http.StatusOK, resp.StatusCode)
	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
	assert.Equal(t, "no", resp.Header.Get("X-Accel-Buffering"))
	select {
	case <-started:
	case <-ctx.Done():
		t.Fatal("evaluation did not start")
	}
	select {
	case <-completed:
		t.Fatal("evaluation completed before the client released it")
	default:
	}

	// The evaluator cannot finish until this actual socket read succeeds.
	// An unstarted response or buffered keepalives instead reach the request
	// deadline; neither can pass just because a sleeping evaluator woke late.
	one := make([]byte, 1)
	_, err = io.ReadFull(resp.Body, one)
	require.NoError(t, err, "keepalive must reach the client while evaluation is blocked")
	assert.Equal(t, "\n", string(one), "the first body byte should be a keepalive newline")
	select {
	case <-completed:
		t.Fatal("evaluation completed before the keepalive was observed")
	default:
	}
	release()
	select {
	case <-completed:
	case <-ctx.Done():
		t.Fatal("evaluation did not complete after release")
	}

	rest, err := io.ReadAll(resp.Body)
	require.NoError(t, err)

	// The padded body is still one ordinary JSON document.
	body := string(one) + string(rest)
	assert.True(t, strings.HasPrefix(body, "\n"), "expected leading keepalive whitespace, got %q", firstByte(body))
	var snapshot PatrolModelReadinessSnapshot
	require.NoError(t, json.Unmarshal([]byte(body), &snapshot))
	require.NotNil(t, snapshot.PatrolModelReadinessResult)
	assert.Equal(t, "verified over the wire", snapshot.Summary)
	assert.Equal(t, ai.PatrolModelReadinessPass, snapshot.Status)
}

// A panic inside provider streaming or validation runs on the evaluation
// goroutine, outside the server's per-request recovery. Unrecovered it takes
// the whole Pulse process down; it must become a failed readiness result.
func TestIssue1640EvaluationPanicBecomesFailedResult(t *testing.T) {
	t.Parallel()

	rec := httptest.NewRecorder()
	result := streamPatrolModelReadinessKeepalives(rec, 5*time.Millisecond, func() ai.PatrolModelReadinessResult {
		panic("provider stream decoder exploded")
	})

	assert.Equal(t, ai.PatrolModelReadinessFail, result.Status)
	assert.Equal(t, ai.PatrolFailureCauseInternalError, result.Cause)
	assert.NotEmpty(t, result.Summary)
	assert.False(t, result.Success)
	assert.False(t, result.PatrolCapable)
	// Nothing was measured, so nothing may be reported as measured.
	assert.Equal(t, ai.PatrolModelReadinessNotAssessed, result.Dimensions.Connectivity.Status)
	assert.Equal(t, ai.PatrolModeNotAssessed, result.Modes.Monitor.Status)

	// The response still completes as a normal 200 JSON document.
	response := patrolModelReadinessSnapshot(&result, time.Now())
	require.NoError(t, utils.WriteJSONResponse(rec, response))
	assert.Equal(t, http.StatusOK, rec.Code)
	var snapshot PatrolModelReadinessSnapshot
	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &snapshot))
	require.NotNil(t, snapshot.PatrolModelReadinessResult)
	assert.Equal(t, ai.PatrolModelReadinessFail, snapshot.Status)
}

// A writer that cannot flush must not take the readiness run down with it: the
// keepalives degrade to buffered writes, but the response still completes.
func TestIssue1640NonFlushableWriterStillCompletes(t *testing.T) {
	t.Parallel()

	w := newNonFlushingResponseWriter()
	if _, ok := http.ResponseWriter(w).(http.Flusher); ok {
		t.Fatal("test writer must not implement http.Flusher")
	}

	result := streamPatrolModelReadinessKeepalives(w, 5*time.Millisecond, func() ai.PatrolModelReadinessResult {
		time.Sleep(20 * time.Millisecond)
		return passingReadinessResult("verified without a flusher")
	})
	response := patrolModelReadinessSnapshot(&result, time.Now())
	require.NoError(t, utils.WriteJSONResponse(w, response))

	assert.Equal(t, http.StatusOK, w.statusCode)
	var snapshot PatrolModelReadinessSnapshot
	require.NoError(t, json.Unmarshal([]byte(w.body.String()), &snapshot))
	require.NotNil(t, snapshot.PatrolModelReadinessResult)
	assert.Equal(t, "verified without a flusher", snapshot.Summary)
}