Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

51.4 KB
Raw Blame
package pbs

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"slices"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type roundTripFunc func(*http.Request) (*http.Response, error)

func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
	return f(req)
}

func newRootNodeNameClient(t *testing.T, host string) *Client {
	t.Helper()
	client, err := NewClient(ClientConfig{
		Host:    host,
		User:    "root@pam",
		Timeout: 2 * time.Second,
	})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	return client
}

func TestNewClient_TokenAuth_SetsAuthorizationHeader(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/version" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		if got := r.Header.Get("Authorization"); got != "PBSAPIToken=root@pam!pulse-token:secret" {
			t.Fatalf("Authorization = %q", got)
		}
		_ = json.NewEncoder(w).Encode(map[string]any{
			"data": Version{Version: "3.0", Release: "1", Repoid: "abc"},
		})
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{
		Host:       server.URL,
		TokenName:  "root@pam!pulse-token",
		TokenValue: "secret",
		Timeout:    2 * time.Second,
	})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}

	v, err := client.GetVersion(context.Background())
	if err != nil {
		t.Fatalf("GetVersion: %v", err)
	}
	if v.Version != "3.0" {
		t.Fatalf("unexpected version: %+v", v)
	}
}

func TestClient_ListRunningDataTasks(t *testing.T) {
	var requests []string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/nodes/localhost/tasks" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		if got := r.URL.Query().Get("running"); got != "true" {
			t.Fatalf("running query = %q, want true", got)
		}
		if got := r.URL.Query().Get("limit"); got != "200" {
			t.Fatalf("limit query = %q, want 200", got)
		}
		workerType := r.URL.Query().Get("typefilter")
		requests = append(requests, workerType)
		data := []map[string]any{}
		switch workerType {
		case "backup":
			data = append(data,
				map[string]any{"worker_type": "backup", "worker_id": "main:vm/117", "starttime": 1800000000},
				map[string]any{"worker_type": "verificationjob", "worker_id": "verify-main", "starttime": 1800000200},
			)
		case "syncjob":
			data = append(data, map[string]any{"worker-type": "syncjob", "worker-id": "offsite", "start-time": 1800000100})
		default:
			t.Fatalf("unexpected typefilter: %q", workerType)
		}
		_ = json.NewEncoder(w).Encode(map[string]any{"data": data})
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	tasks, err := client.ListRunningDataTasks(context.Background())
	if err != nil {
		t.Fatalf("ListRunningDataTasks: %v", err)
	}
	if len(tasks) != 2 {
		t.Fatalf("running data tasks = %#v, want backup and sync only", tasks)
	}
	if !slices.Equal(requests, []string{"backup", "syncjob"}) {
		t.Fatalf("type-filtered requests = %v, want backup then syncjob", requests)
	}
	if tasks[0].WorkerType != "backup" || tasks[0].WorkerID != "main:vm/117" || tasks[0].StartTime != 1800000000 {
		t.Fatalf("backup task = %#v", tasks[0])
	}
	if tasks[1].WorkerType != "syncjob" || tasks[1].WorkerID != "offsite" || tasks[1].StartTime != 1800000100 {
		t.Fatalf("sync task = %#v", tasks[1])
	}
}

func TestClient_ListRunningDataTasksRejectsTruncatedWriterSet(t *testing.T) {
	oldLimit, oldMaxPages := pbsRunningTaskPageLimit, pbsRunningTaskMaxPages
	pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = 2, 1
	t.Cleanup(func() {
		pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = oldLimit, oldMaxPages
	})

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{
			{"worker_type": "backup", "worker_id": "main:vm/117"},
			{"worker_type": "backup", "worker_id": "main:vm/118"},
		}})
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	tasks, err := client.ListRunningDataTasks(context.Background())
	if err == nil || !strings.Contains(err.Error(), "bounded cap") {
		t.Fatalf("ListRunningDataTasks() tasks = %#v, err = %v; want bounded-cap error", tasks, err)
	}
}

func TestClient_ListRunningDataTasksRequiresReadableInventory(t *testing.T) {
	for _, family := range []string{"backup", "syncjob"} {
		for _, body := range []string{
			`null`, `{}`, `{"data":null}`, `{"data":{}}`, `{"data":false}`, `{"data":`,
			`{"data":[null]}`, `{"data":[{}]}`, `{"data":[{"worker_type":null}]}`,
			`{"data":[{"worker_type":123}]}`, `{"data":[{"worker_type":" "}]}`,
			`{"data":[],"errors":{"private":"provider detail"}}`, `{"data":[]} {"data":[]}`,
		} {
			t.Run(family+"/"+body, func(t *testing.T) {
				reads := 0
				client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
					reads++
					if r.Method != http.MethodGet || r.URL.Query().Get("running") != "true" {
						t.Errorf("unexpected writer read: %s %s", r.Method, r.URL)
					}
					if r.URL.Query().Get("typefilter") == family {
						_, _ = w.Write([]byte(body))
					} else {
						_, _ = w.Write([]byte(`{"data":[{"worker_type":"backup","worker_id":"archive:vm/100"}]}`))
					}
				})
				defer server.Close()
				tasks, err := client.ListRunningDataTasks(context.Background())
				if err == nil || tasks != nil {
					t.Fatalf("unread inventory accepted or returned partial writers: tasks=%v err=%v", tasks, err)
				}
				if strings.Contains(err.Error(), "provider detail") {
					t.Fatal("provider body leaked into format diagnostic")
				}
				wantReads := 1
				if family == "syncjob" {
					wantReads = 2
				}
				if reads != wantReads {
					t.Fatalf("reads=%d want=%d; invalid inventories must not retry", reads, wantReads)
				}
			})
		}
	}
	for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`} {
		t.Run("explicit-empty/"+body, func(t *testing.T) {
			client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })
			defer server.Close()
			tasks, err := client.ListRunningDataTasks(context.Background())
			if err != nil || len(tasks) != 0 {
				t.Fatalf("explicit no-writer inventory rejected: tasks=%v err=%v", tasks, err)
			}
		})
	}
}

func TestClient_ListRunningDataTasksRejectsUnreadLaterPage(t *testing.T) {
	client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Query().Get("start") != "" {
			_, _ = w.Write([]byte(`{"data":null}`))
			return
		}
		rows := make([]map[string]string, pbsRunningTaskPageLimit)
		for i := range rows {
			rows[i] = map[string]string{"worker_type": "backup", "worker_id": "archive:vm/100"}
		}
		_ = json.NewEncoder(w).Encode(map[string]any{"data": rows})
	})
	defer server.Close()
	tasks, err := client.ListRunningDataTasks(context.Background())
	if err == nil || tasks != nil {
		t.Fatalf("unread later page accepted as complete writers: tasks=%v err=%v", tasks, err)
	}
}

type runningTasksBody struct {
	io.Reader
	closeErr error
	closed   bool
}

func (b *runningTasksBody) Close() error { b.closed = true; return b.closeErr }

func TestClient_ListRunningDataTasksRequiresCompleteResponse(t *testing.T) {
	for _, failure := range []string{"read", "close", "size"} {
		t.Run(failure, func(t *testing.T) {
			body := &runningTasksBody{Reader: strings.NewReader(`{"data":[]}`)}
			switch failure {
			case "read":
				body.Reader = io.MultiReader(body.Reader, &failReader{})
			case "close":
				body.closeErr = errors.New("fixture close failure")
			case "size":
				body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))
			}
			client, err := NewClient(ClientConfig{Host: "https://pbs.invalid", TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
			if err != nil {
				t.Fatal(err)
			}
			client.httpClient.Transport = roundTripFunc(func(*http.Request) (*http.Response, error) {
				return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: body}, nil
			})
			tasks, err := client.ListRunningDataTasks(context.Background())
			if err == nil || tasks != nil || !body.closed {
				t.Fatalf("incomplete response accepted/unclosed: tasks=%v err=%v closed=%t", tasks, err, body.closed)
			}
		})
	}
}

func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/api2/json/nodes/localhost/tasks":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{
				{
					"upid":        "UPID:sync:1",
					"worker-type": "syncjob",
					"worker-id":   "sync-remote-a",
					"status":      "OK",
					"starttime":   1700000000,
					"endtime":     1700000060,
				},
				{
					"upid":        "UPID:backup:1",
					"worker-type": "backup",
					"worker-id":   "vm/100",
					"status":      "OK",
					"endtime":     1700000100,
				},
			}})
		case "/api2/json/admin/sync":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
				"id":               "sync-remote-a",
				"store":            "fast",
				"remote":           "remote-a",
				"schedule":         "hourly",
				"last-run-state":   "OK",
				"last-run-upid":    "UPID:sync:1",
				"last-run-endtime": 1700000060,
				"next-run":         1700003600,
			}}})
		case "/api2/json/admin/verify":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
				"id":             "verify-fast",
				"store":          "fast",
				"last-run-state": "OK",
			}}})
		case "/api2/json/admin/prune":
			w.WriteHeader(http.StatusForbidden)
			_, _ = w.Write([]byte("permission denied"))
		case "/api2/json/admin/datastore/fast/gc":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{
				"schedule":         "daily",
				"last-run-state":   "OK",
				"last-run-endtime": 1700000200,
			}})
		default:
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}

	evidence, err := client.GetJobHealthEvidence(context.Background(), []string{"fast"}, JobHealthOptions{
		MonitorBackups:     true,
		MonitorSyncJobs:    true,
		MonitorVerifyJobs:  true,
		MonitorPruneJobs:   true,
		MonitorGarbageJobs: true,
	})
	if err != nil {
		t.Fatalf("GetJobHealthEvidence: %v", err)
	}

	byID := make(map[string]JobHealthEvidence)
	for _, item := range evidence {
		byID[item.ID] = item
	}
	if got := byID["sync-remote-a"]; got.Confidence != "direct-task-match" || got.EvidenceSource != JobEvidenceSourcePBSJobConfig || got.EvidenceScope != JobEvidenceScopeConfiguredJob || got.UPID != "UPID:sync:1" || got.LastRunState != "OK" || got.NextRun != 1700003600 {
		t.Fatalf("expected direct sync evidence with raw last-run fields, got %+v", got)
	}
	if got := byID["verify-fast"]; got.Confidence != "direct-config-last-run" || got.LastRunState != "OK" {
		t.Fatalf("expected config last-run verify evidence, got %+v", got)
	}
	if got := byID["prune:partial"]; got.Confidence != "partial-permission" || got.EvidenceScope != JobEvidenceScopePartialRead || got.Error == "" {
		t.Fatalf("expected partial permission prune evidence, got %+v", got)
	}
	if got := byID["backup:vm/100"]; got.Confidence != JobEvidenceConfidenceObservedBackupTask || got.EvidenceSource != JobEvidenceSourcePBSTaskHistory || got.EvidenceScope != JobEvidenceScopeObservedTask || got.Schedule != "" || got.NextRun != 0 || got.TaskStatus != "OK" {
		t.Fatalf("expected observed backup task evidence without scheduled compliance, got %+v", got)
	}
	if got := byID["garbage:fast"]; got.Confidence != "direct-config-last-run" || got.Store != "fast" {
		t.Fatalf("expected garbage config evidence, got %+v", got)
	}
}

// A real PBS returns run results only from /admin/{family}; /config/{family}
// lists the same jobs with schedules and retention but no last-run fields.
// Reading /config made every successful prune, verify and sync job look as if
// it had never run.
func TestClient_GetJobHealthEvidence_ReadsRunStatusFromAdminEndpoints(t *testing.T) {
	var configReads []string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/api2/json/nodes/localhost/tasks":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})
		case "/api2/json/admin/prune":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
				"id":               "prune-main-delly",
				"store":            "main",
				"ns":               "delly",
				"schedule":         "07:10",
				"keep-daily":       3,
				"last-run-state":   "OK",
				"last-run-upid":    "UPID:pbs:prunejob:main:delly",
				"last-run-endtime": 1793603400,
				"next-run":         1793689800,
			}}})
		case "/api2/json/admin/verify":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
				"id":               "verify-main",
				"store":            "main",
				"schedule":         "sun 05:00",
				"last-run-state":   "OK",
				"last-run-endtime": 1793509200,
				"next-run":         1794114000,
			}}})
		case "/api2/json/admin/sync":
			// An older PBS without the status endpoint falls back to /config.
			w.WriteHeader(http.StatusNotFound)
			_, _ = w.Write([]byte("Path '/api2/json/admin/sync' not found."))
		case "/api2/json/config/sync":
			configReads = append(configReads, r.URL.Path)
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
				"id":       "sync-offsite",
				"store":    "main",
				"schedule": "hourly",
			}}})
		case "/api2/json/config/prune", "/api2/json/config/verify":
			configReads = append(configReads, r.URL.Path)
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})
		default:
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	evidence, err := client.GetJobHealthEvidence(context.Background(), []string{"main"}, JobHealthOptions{
		MonitorSyncJobs:   true,
		MonitorVerifyJobs: true,
		MonitorPruneJobs:  true,
	})
	if err != nil {
		t.Fatalf("GetJobHealthEvidence: %v", err)
	}
	byID := make(map[string]JobHealthEvidence)
	for _, item := range evidence {
		byID[item.ID] = item
	}
	if got := byID["prune-main-delly"]; got.LastRunState != "OK" || got.LastRunEndtime != 1793603400 || got.NextRun != 1793689800 || got.Confidence != JobEvidenceConfidenceConfigLastRun {
		t.Fatalf("expected prune run status from /admin/prune, got %+v", got)
	}
	if got := byID["verify-main"]; got.LastRunState != "OK" || got.Confidence != JobEvidenceConfidenceConfigLastRun {
		t.Fatalf("expected verify run status from /admin/verify, got %+v", got)
	}
	if got := byID["sync-offsite"]; got.Schedule != "hourly" || got.LastRunState != "" || got.Confidence != JobEvidenceConfidenceConfigOnly {
		t.Fatalf("expected config-only sync evidence after /admin/sync was not found, got %+v", got)
	}
	if len(configReads) != 1 || configReads[0] != "/api2/json/config/sync" {
		t.Fatalf("expected /config to be read only for the family without /admin, got %v", configReads)
	}
}

func TestPBSJobTaskHistoryQueries_UsesWorkerTypeFilters(t *testing.T) {
	queries := buildTaskHistoryQueries([]string{"fast", "fast", "slow"}, JobHealthOptions{
		MonitorBackups:     true,
		MonitorSyncJobs:    true,
		MonitorVerifyJobs:  true,
		MonitorPruneJobs:   true,
		MonitorGarbageJobs: true,
	}, 1700000000, 1700003600)

	got := make(map[string][]taskHistoryQuery)
	for _, query := range queries {
		got[query.Family] = append(got[query.Family], query)
		if query.Since != 1700000000 || query.Until != 1700003600 {
			t.Fatalf("query %s bounds = (%d, %d), want (1700000000, 1700003600)", query.Family, query.Since, query.Until)
		}
	}

	backupQueries := got["backup"]
	if len(backupQueries) != 2 {
		t.Fatalf("backup query count = %d, want 2: %#v", len(backupQueries), backupQueries)
	}
	backupStores := map[string]bool{}
	for _, query := range backupQueries {
		if query.TypeFilter != "backup" {
			t.Fatalf("backup typefilter = %q, want backup", query.TypeFilter)
		}
		backupStores[query.Store] = true
	}
	if !backupStores["fast"] || !backupStores["slow"] {
		t.Fatalf("backup stores = %#v, want fast and slow", backupStores)
	}

	for family, want := range map[string]string{
		"sync":    "syncjob",
		"verify":  "verificationjob",
		"prune":   "prunejob",
		"garbage": "garbage_collection",
	} {
		familyQueries := got[family]
		if len(familyQueries) != 1 {
			t.Fatalf("%s query count = %d, want 1: %#v", family, len(familyQueries), familyQueries)
		}
		query := familyQueries[0]
		if query.TypeFilter != want {
			t.Fatalf("%s typefilter = %q, want %q", family, query.TypeFilter, want)
		}
		if query.Store != "" {
			t.Fatalf("%s store filter = %q, want empty", family, query.Store)
		}
	}
}

func TestClient_GetJobHealthEvidence_UsesBoundedFilteredTaskHistory(t *testing.T) {
	oldLimit := pbsTaskHistoryPageLimit
	oldPages := pbsTaskHistoryMaxPages
	oldLookback := pbsTaskHistoryLookback
	pbsTaskHistoryPageLimit = 2
	pbsTaskHistoryMaxPages = 2
	pbsTaskHistoryLookback = time.Hour
	t.Cleanup(func() {
		pbsTaskHistoryPageLimit = oldLimit
		pbsTaskHistoryMaxPages = oldPages
		pbsTaskHistoryLookback = oldLookback
	})

	var queries []url.Values
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/nodes/localhost/tasks" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		q := r.URL.Query()
		queries = append(queries, q)
		if q.Get("typefilter") != "backup" {
			t.Fatalf("typefilter = %q, want backup", q.Get("typefilter"))
		}
		if q.Get("store") != "fast" {
			t.Fatalf("store = %q, want fast", q.Get("store"))
		}
		if q.Get("since") == "" || q.Get("until") == "" {
			t.Fatalf("expected bounded since/until filters, got %s", r.URL.RawQuery)
		}
		if q.Get("limit") != "2" {
			t.Fatalf("limit = %q, want 2", q.Get("limit"))
		}

		statusFilters := q["statusfilter"]
		if slices.Contains(statusFilters, "error") {
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
				"upid":      "UPID:backup:error",
				"type":      "backup",
				"id":        "vm/failed",
				"status":    "error",
				"starttime": 1700000400,
				"endtime":   1700000500,
			}}})
			return
		}
		if slices.Contains(statusFilters, "warning") {
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})
			return
		}

		switch q.Get("start") {
		case "":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{
				{"upid": "UPID:backup:1", "type": "backup", "id": "vm/100", "status": "OK", "starttime": 1700000000, "endtime": 1700000100},
				{"upid": "UPID:backup:2", "type": "backup", "id": "vm/101", "status": "OK", "starttime": 1700000200, "endtime": 1700000300},
			}})
		case "2":
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{
				{"upid": "UPID:backup:3", "type": "backup", "id": "vm/102", "status": "OK", "starttime": 1700000400, "endtime": 1700000500},
				{"upid": "UPID:backup:4", "type": "backup", "id": "vm/103", "status": "OK", "starttime": 1700000600, "endtime": 1700000700},
			}})
		default:
			t.Fatalf("unexpected start=%q", q.Get("start"))
		}
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}

	evidence, err := client.GetJobHealthEvidence(context.Background(), []string{"fast"}, JobHealthOptions{MonitorBackups: true})
	if err != nil {
		t.Fatalf("GetJobHealthEvidence: %v", err)
	}

	byID := make(map[string]JobHealthEvidence)
	for _, item := range evidence {
		byID[item.ID] = item
	}
	if got := byID["backup:vm/failed"]; got.Confidence != JobEvidenceConfidenceObservedBackupTask || got.TaskStatus != "error" {
		t.Fatalf("expected error statusfilter task to be preserved as observed backup evidence, got %+v", got)
	}
	if got := byID["backup:task-history:fast:truncated"]; got.Confidence != JobEvidenceConfidenceHistoryTruncated || got.EvidenceScope != JobEvidenceScopePartialRead || got.Error == "" {
		t.Fatalf("expected visible truncation evidence, got %+v", got)
	}

	var sawSecondPage, sawErrorStatusFilter bool
	for _, q := range queries {
		if q.Get("start") == "2" {
			sawSecondPage = true
		}
		if slices.Contains(q["statusfilter"], "error") {
			sawErrorStatusFilter = true
		}
	}
	if !sawSecondPage || !sawErrorStatusFilter {
		t.Fatalf("expected pagination and statusfilter fallback queries, got %#v", queries)
	}
}

func TestNewClient_PasswordAuth_FallsBackToFormOnUnsupportedMediaType(t *testing.T) {
	var calls int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/access/ticket" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}

		switch atomic.AddInt32(&calls, 1) {
		case 1:
			if ct := r.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
				t.Fatalf("expected json content-type, got %q", ct)
			}
			w.WriteHeader(http.StatusUnsupportedMediaType)
			_, _ = w.Write([]byte("unsupported"))
		case 2:
			if ct := r.Header.Get("Content-Type"); !strings.Contains(ct, "application/x-www-form-urlencoded") {
				t.Fatalf("expected form content-type, got %q", ct)
			}
			_ = json.NewEncoder(w).Encode(map[string]any{
				"data": map[string]any{
					"ticket":              "ticket123",
					"CSRFPreventionToken": "csrf456",
				},
			})
		default:
			t.Fatalf("unexpected call count")
		}
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{
		Host:     server.URL,
		User:     "root@pam",
		Password: "password",
		Timeout:  2 * time.Second,
	})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}
	if client.auth.ticket != "ticket123" || client.auth.csrfToken != "csrf456" {
		t.Fatalf("expected auth fields to be set, got ticket=%q csrf=%q", client.auth.ticket, client.auth.csrfToken)
	}
}

func TestClient_request_SendsTicketAndCSRFToken(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/test" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		if got := r.Header.Get("Cookie"); got != "PBSAuthCookie=ticket123" {
			t.Fatalf("Cookie = %q", got)
		}
		if got := r.Header.Get("CSRFPreventionToken"); got != "csrf456" {
			t.Fatalf("CSRFPreventionToken = %q", got)
		}
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{
		Host:    server.URL,
		User:    "root@pam",
		Timeout: 2 * time.Second,
	})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}

	client.auth.ticket = "ticket123"
	client.auth.csrfToken = "csrf456"
	client.auth.expiresAt = time.Now().Add(time.Hour)

	resp, err := client.request(context.Background(), http.MethodPost, "/test", url.Values{"a": {"b"}})
	if err != nil {
		t.Fatalf("request: %v", err)
	}
	_ = resp.Body.Close()
}

func TestClient_GetNodeStatus_PermissionDeniedReturnsNil(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/nodes/localhost/status" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		w.WriteHeader(http.StatusForbidden)
		_, _ = w.Write([]byte("permission denied"))
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{
		Host:       server.URL,
		TokenName:  "root@pam!pulse-token",
		TokenValue: "secret",
		Timeout:    2 * time.Second,
	})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}

	status, err := client.GetNodeStatus(context.Background())
	if err != nil {
		t.Fatalf("GetNodeStatus: %v", err)
	}
	if status != nil {
		t.Fatalf("expected nil status on permission error, got: %+v", status)
	}
}

func TestClient_GetDatastores_HTMLResponseOnHTTP(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/admin/datastore" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("<html><body>error</body></html>"))
	}))
	defer server.Close()

	client, err := NewClient(ClientConfig{
		Host:       server.URL, // http://...
		TokenName:  "root@pam!pulse-token",
		TokenValue: "secret",
		Timeout:    2 * time.Second,
	})
	if err != nil {
		t.Fatalf("NewClient: %v", err)
	}

	_, err = client.GetDatastores(context.Background())
	if err == nil {
		t.Fatalf("expected error")
	}
	if !strings.Contains(err.Error(), "Try changing your URL") {
		t.Fatalf("unexpected error: %v", err)
	}
}

func TestClient_GetNodeName_CachesResult(t *testing.T) {
	var nodesCalls atomic.Int64
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/nodes" {
			t.Fatalf("unexpected path: %s", r.URL.Path)
		}
		nodesCalls.Add(1)
		_ = json.NewEncoder(w).Encode(map[string]any{
			"data": []map[string]any{{"node": "pbs-main"}},
		})
	}))
	defer server.Close()

	client := newRootNodeNameClient(t, server.URL)

	for i := 0; i < 3; i++ {
		name, err := client.GetNodeName(context.Background())
		if err != nil {
			t.Fatalf("GetNodeName call %d: %v", i, err)
		}
		if name != "pbs-main" {
			t.Fatalf("GetNodeName call %d = %q, want pbs-main", i, name)
		}
	}
	if got := nodesCalls.Load(); got != 1 {
		t.Fatalf("/nodes hit %d times, want 1 (cached after first success)", got)
	}
}

func TestClient_GetNodeName_SuperuserPermissionFailureRetries(t *testing.T) {
	for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} {
		t.Run(http.StatusText(status), func(t *testing.T) {
			var nodesCalls atomic.Int64
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Path != "/api2/json/nodes" {
					t.Fatalf("unexpected path: %s", r.URL.Path)
				}
				if nodesCalls.Add(1) == 1 {
					w.WriteHeader(status)
					_, _ = w.Write([]byte(`{"message":"access denied"}`))
					return
				}
				_ = json.NewEncoder(w).Encode(map[string]any{
					"data": []map[string]any{{"node": "pbs-recovered"}},
				})
			}))
			defer server.Close()

			client := newRootNodeNameClient(t, server.URL)

			_, firstErr := client.GetNodeName(context.Background())
			if firstErr == nil {
				t.Fatal("first GetNodeName: expected error")
			}
			if got, ok := HTTPStatus(firstErr); !ok || got != status {
				t.Fatalf("first GetNodeName status = (%d, %v), want (%d, true): %v", got, ok, status, firstErr)
			}
			name, err := client.GetNodeName(context.Background())
			if err != nil {
				t.Fatalf("second GetNodeName: %v", err)
			}
			if name != "pbs-recovered" {
				t.Fatalf("second GetNodeName = %q, want pbs-recovered", name)
			}
			if got := nodesCalls.Load(); got != 2 {
				t.Fatalf("/nodes hit %d times, want 2 (%d is retryable for direct root session)", got, status)
			}
		})
	}
}

func TestClient_GetNodeName_TransientHTTPFailuresRetryAndRecover(t *testing.T) {
	tests := []struct {
		name   string
		status int
		body   string
	}{
		{name: "rate limited", status: http.StatusTooManyRequests, body: `{"message":"slow down"}`},
		{name: "internal server error", status: http.StatusInternalServerError, body: `{"message":"temporary failure"}`},
		{name: "service unavailable body mentions permission", status: http.StatusServiceUnavailable, body: `{"message":"permission service unavailable"}`},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			var nodesCalls atomic.Int64
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Path != "/api2/json/nodes" {
					t.Fatalf("unexpected path: %s", r.URL.Path)
				}
				if nodesCalls.Add(1) == 1 {
					w.WriteHeader(tc.status)
					_, _ = w.Write([]byte(tc.body))
					return
				}
				_ = json.NewEncoder(w).Encode(map[string]any{
					"data": []map[string]any{{"node": "pbs-recovered"}},
				})
			}))
			defer server.Close()

			client := newRootNodeNameClient(t, server.URL)

			_, firstErr := client.GetNodeName(context.Background())
			if firstErr == nil {
				t.Fatal("first GetNodeName: expected error")
			}
			if got, ok := HTTPStatus(firstErr); !ok || got != tc.status {
				t.Fatalf("first GetNodeName status = (%d, %v), want (%d, true): %v", got, ok, tc.status, firstErr)
			}
			name, err := client.GetNodeName(context.Background())
			if err != nil {
				t.Fatalf("recovery GetNodeName: %v", err)
			}
			if name != "pbs-recovered" {
				t.Fatalf("recovery GetNodeName = %q, want pbs-recovered", name)
			}
			if got := nodesCalls.Load(); got != 2 {
				t.Fatalf("/nodes hit %d times, want 2 (transient failure then recovery)", got)
			}
		})
	}
}

func TestClient_GetNodeName_NetworkFailureRetriesAndRecovers(t *testing.T) {
	client := newRootNodeNameClient(t, "http://pbs.example.test")

	var calls atomic.Int64
	client.httpClient.Transport = roundTripFunc(func(req *http.Request) (*http.Response, error) {
		if calls.Add(1) == 1 {
			return nil, errors.New("network permission proxy failure")
		}
		return &http.Response{
			StatusCode: http.StatusOK,
			Header:     make(http.Header),
			Body:       io.NopCloser(strings.NewReader(`{"data":[{"node":"pbs-network-recovered"}]}`)),
			Request:    req,
		}, nil
	})

	if _, err := client.GetNodeName(context.Background()); err == nil {
		t.Fatal("first GetNodeName: expected network error")
	}

	name, err := client.GetNodeName(context.Background())
	if err != nil {
		t.Fatalf("recovery GetNodeName: %v", err)
	}
	if name != "pbs-network-recovered" {
		t.Fatalf("recovery GetNodeName = %q, want pbs-network-recovered", name)
	}
	if got := calls.Load(); got != 2 {
		t.Fatalf("transport called %d times, want 2", got)
	}
}

func TestClient_GetNodeName_UnsupportedAuthNeverCallsNodes(t *testing.T) {
	tests := []struct {
		name   string
		config ClientConfig
	}{
		{
			name: "API token including root owner",
			config: ClientConfig{
				TokenName:  "root@pam!pulse-token",
				TokenValue: "secret",
			},
		},
		{
			name: "non-superuser password identity",
			config: ClientConfig{
				User: "admin@pbs",
			},
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			var nodesCalls atomic.Int64
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				nodesCalls.Add(1)
				t.Fatalf("unsupported auth requested %s", r.URL.Path)
			}))
			defer server.Close()

			tc.config.Host = server.URL
			tc.config.Timeout = 2 * time.Second
			client, err := NewClient(tc.config)
			if err != nil {
				t.Fatalf("NewClient: %v", err)
			}

			for i := 0; i < 3; i++ {
				name, err := client.GetNodeName(context.Background())
				if name != "" {
					t.Fatalf("GetNodeName call %d = %q, want empty", i, name)
				}
				if !errors.Is(err, ErrNodeNameUnavailableForAuth) {
					t.Fatalf("GetNodeName call %d error = %v, want ErrNodeNameUnavailableForAuth", i, err)
				}
			}
			if got := nodesCalls.Load(); got != 0 {
				t.Fatalf("/nodes hit %d times, want 0", got)
			}
		})
	}
}

func TestClient_GetNodeName_ConcurrentTransientFailureIsSingleFlight(t *testing.T) {
	const callers = 16

	var nodesCalls atomic.Int64
	requestStarted := make(chan struct{})
	releaseFirstRequest := make(chan struct{})
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if nodesCalls.Add(1) == 1 {
			close(requestStarted)
			<-releaseFirstRequest
			w.WriteHeader(http.StatusServiceUnavailable)
			_, _ = w.Write([]byte(`{"message":"temporary permission backend outage"}`))
			return
		}
		_ = json.NewEncoder(w).Encode(map[string]any{
			"data": []map[string]any{{"node": "pbs-concurrent-recovery"}},
		})
	}))
	defer server.Close()

	client := newRootNodeNameClient(t, server.URL)

	start := make(chan struct{})
	errs := make(chan error, callers)
	var wg sync.WaitGroup
	for i := 0; i < callers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-start
			_, err := client.GetNodeName(context.Background())
			errs <- err
		}()
	}
	close(start)
	<-requestStarted

	deadline := time.Now().Add(2 * time.Second)
	for {
		client.nodeNameMu.Lock()
		joined := 0
		if client.nodeNameAttempt != nil {
			joined = client.nodeNameAttempt.joined
		}
		client.nodeNameMu.Unlock()
		if joined == callers-1 {
			break
		}
		if time.Now().After(deadline) {
			close(releaseFirstRequest)
			t.Fatalf("only %d of %d concurrent callers joined the in-flight request", joined, callers-1)
		}
		time.Sleep(time.Millisecond)
	}
	close(releaseFirstRequest)
	wg.Wait()
	close(errs)

	for err := range errs {
		if err == nil {
			t.Fatal("concurrent GetNodeName: expected transient error")
		}
		if got, ok := HTTPStatus(err); !ok || got != http.StatusServiceUnavailable {
			t.Fatalf("concurrent GetNodeName status = (%d, %v), want (503, true): %v", got, ok, err)
		}
	}
	if got := nodesCalls.Load(); got != 1 {
		t.Fatalf("/nodes hit %d times for concurrent transient failure, want 1", got)
	}

	name, err := client.GetNodeName(context.Background())
	if err != nil {
		t.Fatalf("recovery GetNodeName: %v", err)
	}
	if name != "pbs-concurrent-recovery" {
		t.Fatalf("recovery GetNodeName = %q, want pbs-concurrent-recovery", name)
	}
	if _, err := client.GetNodeName(context.Background()); err != nil {
		t.Fatalf("cached GetNodeName: %v", err)
	}
	if got := nodesCalls.Load(); got != 2 {
		t.Fatalf("/nodes hit %d times after recovery and cache read, want 2", got)
	}
}

// Restricted node metrics must not turn a later outage into a permission
// fallback, nor prevent recovery on the same long-lived polling client.
func TestClient_GetNodeStatus_PermissionOutageRecovery(t *testing.T) {
	var responseCode atomic.Int32
	responseCode.Store(http.StatusForbidden)
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/api2/json/nodes/localhost/status" {
			t.Errorf("unexpected path: %s", r.URL.Path)
			http.NotFound(w, r)
			return
		}
		code := int(responseCode.Load())
		w.WriteHeader(code)
		if code != http.StatusOK {
			// Deliberately misleading body: classification must use HTTP status.
			_, _ = w.Write([]byte("permission denied: upstream authentication error"))
			return
		}
		_, _ = w.Write([]byte(`{"data":{"cpu":0.25,"memory":{"used":500,"total":1000}}}`))
	}))
	defer server.Close()
	client, err := NewClient(ClientConfig{
		Host: server.URL, TokenName: "monitor@pbs!pulse",
		TokenValue: "test-only", Timeout: 2 * time.Second,
	})
	if err != nil {
		t.Fatal(err)
	}
	status, err := client.GetNodeStatus(context.Background())
	if err != nil || status != nil {
		t.Fatalf("restricted metrics = (%+v, %v), want (nil, nil)", status, err)
	}
	for _, code := range []int{http.StatusServiceUnavailable, http.StatusBadGateway} {
		responseCode.Store(int32(code))
		status, err = client.GetNodeStatus(context.Background())
		if status != nil || err == nil {
			t.Fatalf("outage %d = (%+v, %v), want nil status and error", code, status, err)
		}
		if got, ok := HTTPStatus(err); !ok || got != code {
			t.Fatalf("outage status = (%d, %v), want %d", got, ok, code)
		}
	}
	responseCode.Store(http.StatusOK)
	status, err = client.GetNodeStatus(context.Background())
	if err != nil || status == nil {
		t.Fatalf("recovery = (%+v, %v), want metrics", status, err)
	}
	if status.CPU != 0.25 {
		t.Fatalf("recovered CPU = %v, want 0.25", status.CPU)
	}
}

func TestClient_GetNodeStatus_MissingData(t *testing.T) {
	for _, body := range []string{`{"data":null}`, `{}`, `null`} {
		t.Run(body, func(t *testing.T) {
			client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				_, _ = w.Write([]byte(body))
			})
			defer server.Close()
			status, err := client.GetNodeStatus(context.Background())
			if status != nil || err == nil {
				t.Fatalf("absent metrics = (%+v, %v), want nil status and error", status, err)
			}
		})
	}
}

// Missing measurements must not be indistinguishable from measured zero usage.
func TestClient_GetNodeStatus_MetricCompleteness(t *testing.T) {
	for _, tc := range []struct {
		name, data string
		valid      bool
	}{
		{"empty", `{}`, false},
		{"cpu only", `{"cpu":0}`, false},
		{"missing cpu", `{"memory":{"used":0,"total":1000}}`, false},
		{"null cpu", `{"cpu":null,"memory":{"used":0,"total":1000}}`, false},
		{"null memory", `{"cpu":0,"memory":null}`, false},
		{"missing used", `{"cpu":0,"memory":{"total":1000}}`, false},
		{"null used", `{"cpu":0,"memory":{"used":null,"total":1000}}`, false},
		{"missing total", `{"cpu":0,"memory":{"used":0}}`, false},
		{"null total", `{"cpu":0,"memory":{"used":0,"total":null}}`, false},
		{"zero total", `{"cpu":0,"memory":{"used":0,"total":0}}`, false},
		{"negative total", `{"cpu":0,"memory":{"used":0,"total":-1}}`, false},
		{"negative used", `{"cpu":0,"memory":{"used":-1,"total":1000}}`, false},
		{"negative cpu", `{"cpu":-1,"memory":{"used":0,"total":1000}}`, false},
		{"wrong type", `{"cpu":0,"memory":{"used":"0","total":1000}}`, false},
		{"measured zero", `{"cpu":0,"memory":{"used":0,"total":1000}}`, true},
		{"extra fields", `{"cpu":0.2,"memory":{"used":500,"total":1000,"free":500},"future":true}`, true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
				_, _ = w.Write([]byte(`{"data":` + tc.data + `}`))
			})
			defer server.Close()
			status, err := client.GetNodeStatus(context.Background())
			if tc.valid {
				if err != nil || status == nil {
					t.Fatalf("valid measurements = (%+v, %v)", status, err)
				}
			} else if err == nil || status != nil {
				t.Fatalf("incomplete/invalid measurements = (%+v, %v), want nil status and error", status, err)
			}
		})
	}
}

// All three listings carry inventory authority: an accepted empty list may
// remove previous backup evidence. A decodable prefix is not a complete read.
func backupInventoryReaders() []struct {
	name string
	read func(*Client) (bool, error)
} {
	return []struct {
		name string
		read func(*Client) (bool, error)
	}{
		{"namespaces", func(c *Client) (bool, error) {
			rows, err := c.ListNamespaces(context.Background(), "archive", "tenant", 2)
			return rows == nil, err
		}},
		{"groups", func(c *Client) (bool, error) {
			rows, err := c.ListBackupGroups(context.Background(), "archive", "tenant")
			return rows == nil, err
		}},
		{"snapshots", func(c *Client) (bool, error) {
			rows, err := c.ListBackupSnapshots(context.Background(), "archive", "tenant", "vm", "101")
			return rows == nil, err
		}},
	}
}

func TestBackupInventoryRequiresWholeResponse(t *testing.T) {
	bodies := map[string]string{
		"second value":           `{"data":[]} {"data":[]}`,
		"trailing garbage":       `{"data":[]} private-provider-detail`,
		"reported error":         `{"data":[],"errors":{"private":"private-provider-detail"}}`,
		"empty errors object":    `{"data":[],"errors":{}}`,
		"false errors":           `{"data":[],"errors":false}`,
		"duplicate data":         `{"data":null,"data":[]}`,
		"escaped duplicate data": `{"data":null,"d\u0061ta":[]}`,
		"duplicate errors":       `{"errors":{"private":"private-provider-detail"},"data":[],"errors":null}`,
		"case data":              `{"data":null,"Data":[]}`,
		"case errors":            `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,
		"invalid utf8":           "{\"data\":[],\"metadata\":\"\xff\"}",
	}
	for _, reader := range backupInventoryReaders() {
		for name, body := range bodies {
			t.Run(reader.name+"/"+name, func(t *testing.T) {
				reads := 0
				client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
					reads++
					if r.Method != http.MethodGet {
						t.Errorf("inventory attempted %s", r.Method)
					}
					_, _ = w.Write([]byte(body))
				})
				defer server.Close()
				isNil, err := reader.read(client)
				if err == nil || !isNil {
					t.Fatalf("unread inventory accepted: nil=%t err=%v", isNil, err)
				}
				if strings.Contains(err.Error(), "private-provider-detail") {
					t.Fatal("provider contents leaked in format diagnostic")
				}
				if reads != 1 {
					t.Fatalf("malformed inventory caused %d requests", reads)
				}
			})
		}
	}
}

type inventoryResponseBody struct {
	io.Reader
	closeErr error
	closed   bool
	bytes    int
}

func (b *inventoryResponseBody) Read(p []byte) (int, error) {
	n, err := b.Reader.Read(p)
	b.bytes += n
	return n, err
}
func (b *inventoryResponseBody) Close() error { b.closed = true; return b.closeErr }

type inventoryReadFailure struct{}

func (inventoryReadFailure) Read([]byte) (int, error) { return 0, errors.New("fixture read failure") }

func TestBackupInventoryRequiresCompleteBody(t *testing.T) {
	for _, reader := range backupInventoryReaders() {
		for _, failure := range []string{"read after JSON", "close", "size", "partial status"} {
			t.Run(reader.name+"/"+failure, func(t *testing.T) {
				body := &inventoryResponseBody{Reader: strings.NewReader(`{"data":[]}`)}
				status := http.StatusOK
				switch failure {
				case "read after JSON":
					body.Reader = io.MultiReader(body.Reader, inventoryReadFailure{})
				case "close":
					body.closeErr = errors.New("fixture close failure")
				case "size":
					body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))
				case "partial status":
					status = http.StatusPartialContent
				}
				client, err := NewClient(ClientConfig{Host: "https://fixture.invalid", TokenName: "fixture@pbs!read", TokenValue: "fixture-only"})
				if err != nil {
					t.Fatal(err)
				}
				reads := 0
				client.httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
					reads++
					return &http.Response{StatusCode: status, Body: body, Header: make(http.Header)}, nil
				})}
				isNil, err := reader.read(client)
				if err == nil || !isNil {
					t.Errorf("incomplete inventory accepted: nil=%t err=%v", isNil, err)
				}
				if reads != 1 || !body.closed {
					t.Errorf("reads=%d closed=%t", reads, body.closed)
				}
				if int64(body.bytes) > maxResponseBodyBytes+1 {
					t.Errorf("read %d bytes past body bound", body.bytes)
				}
			})
		}
	}
}

func TestBackupInventoryAcceptsCompleteEmptyResponse(t *testing.T) {
	for _, reader := range backupInventoryReaders() {
		for _, body := range []string{`{"data":[]}`, ` {"data":[],"errors":null,"total":0,"metadata":{"future":"温度"}} `} {
			t.Run(reader.name+"/"+body, func(t *testing.T) {
				client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })
				defer server.Close()
				isNil, err := reader.read(client)
				if err != nil || isNil {
					t.Fatalf("complete empty read rejected: nil=%t err=%v", isNil, err)
				}
			})
		}
	}
}

// No-writer evidence must not depend on which duplicate/aliased field a
// decoder picks. Reject the entire set, including earlier accepted pages or
// families, rather than letting an ambiguous empty result end a backup.
func TestClient_ListRunningDataTasksRequiresUnambiguousResponse(t *testing.T) {
	oldLimit, oldMaxPages := pbsRunningTaskPageLimit, pbsRunningTaskMaxPages
	pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = 2, 2
	t.Cleanup(func() {
		pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = oldLimit, oldMaxPages
	})
	for _, family := range []string{"backup", "syncjob"} {
		for _, laterPage := range []bool{false, true} {
			for name, body := range map[string]string{
				"duplicate-data":     `{"data":null,"data":[]}`,
				"discarded-writer":   `{"data":[{"worker_type":"backup","worker_id":"private-provider-detail"}],"data":[]}`,
				"case-data":          `{"data":null,"DATA":[]}`,
				"duplicate-errors":   `{"data":[],"errors":{"private":"private-provider-detail"},"errors":null}`,
				"case-errors":        `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,
				"duplicate-metadata": `{"data":[],"total":1,"total":0}`,
				"escaped-duplicate":  `{"data":null,"d\u0061ta":[]}`,
				"invalid-utf8":       "{\"data\":[],\"metadata\":\"\xff\"}",
			} {
				t.Run(fmt.Sprintf("%s/later=%t/%s", family, laterPage, name), func(t *testing.T) {
					reads := 0
					client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
						reads++
						q := r.URL.Query()
						if r.Method != http.MethodGet || r.URL.Path != "/api2/json/nodes/localhost/tasks" || q.Get("running") != "true" || q.Get("limit") != "2" {
							t.Errorf("writer query changed: %s %s", r.Method, r.URL)
						}
						if q.Get("typefilter") == family && (!laterPage || q.Get("start") == "2") {
							_, _ = w.Write([]byte(body))
							return
						}
						rows := []map[string]any{{"worker_type": q.Get("typefilter"), "worker_id": "archive:vm/100"}}
						if q.Get("typefilter") == family && laterPage {
							rows = append(rows, map[string]any{"worker_type": family, "worker_id": "archive:vm/101"})
						}
						_ = json.NewEncoder(w).Encode(map[string]any{"data": rows})
					})
					defer server.Close()
					tasks, err := client.ListRunningDataTasks(context.Background())
					if err == nil || tasks != nil {
						t.Fatalf("ambiguous inventory accepted/partial writers returned: tasks=%v err=%v", tasks, err)
					}
					if strings.Contains(err.Error(), "private-provider-detail") {
						t.Fatal("provider payload leaked into format diagnostic")
					}
					wantReads := 1
					if family == "syncjob" {
						wantReads++
					}
					if laterPage {
						wantReads++
					}
					if reads != wantReads {
						t.Fatalf("reads=%d want=%d; no ambiguous inventory retry", reads, wantReads)
					}
				})
			}
		}
	}
}

func TestClient_ListRunningDataTasksRequiresHTTP200(t *testing.T) {
	for _, family := range []string{"backup", "syncjob"} {
		for _, status := range []int{http.StatusCreated, http.StatusAccepted, http.StatusNonAuthoritativeInfo, http.StatusNoContent, http.StatusPartialContent} {
			t.Run(fmt.Sprintf("%s/%d", family, status), func(t *testing.T) {
				client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
					if r.URL.Query().Get("typefilter") == family {
						w.WriteHeader(status)
					}
					_, _ = w.Write([]byte(`{"data":[]}`))
				})
				defer server.Close()
				tasks, err := client.ListRunningDataTasks(context.Background())
				if err == nil || tasks != nil {
					t.Fatalf("non-200 writer set admitted: tasks=%v err=%v", tasks, err)
				}
			})
		}
	}
}

// Datastore names establish the outer scope used to reconcile every backup.
// Validate the complete envelope before applying exclusions or reading children.
func TestDatastoreInventoryRequiresWholeResponse(t *testing.T) {
	for name, body := range map[string]string{
		"second value":        `{"data":[]} {"data":[]}`,
		"trailing garbage":    `{"data":[]} private-provider-detail`,
		"reported error":      `{"data":[],"errors":{"private":"private-provider-detail"}}`,
		"empty errors":        `{"data":[],"errors":{}}`,
		"duplicate data":      `{"data":null,"data":[]}`,
		"discarded datastore": `{"data":[{"store":"archive"}],"data":[]}`,
		"escaped duplicate":   `{"data":null,"d\u0061ta":[]}`,
		"duplicate errors":    `{"data":[],"errors":{"private":"private-provider-detail"},"errors":null}`,
		"case data":           `{"data":null,"DATA":[]}`,
		"case errors":         `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,
		"duplicate metadata":  `{"data":[],"total":1,"total":0}`,
		"invalid utf8":        "{\"data\":[],\"metadata\":\"\xff\"}",
	} {
		for _, excluded := range []bool{false, true} {
			suffix := "/included"
			if excluded {
				suffix = "/excluded"
			}
			t.Run(name+suffix, func(t *testing.T) {
				reads, filtered := 0, 0
				client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
					reads++
					if r.Method != http.MethodGet || r.URL.Path != "/api2/json/admin/datastore" {
						t.Errorf("unread discovery caused a child or mutation request: %s %s", r.Method, r.URL)
					}
					_, _ = w.Write([]byte(body))
				})
				defer server.Close()
				rows, err := client.GetDatastoresFiltered(context.Background(), func(string) bool { filtered++; return !excluded })
				if err == nil || rows != nil {
					t.Fatalf("unread datastore scope accepted: rows=%+v err=%v", rows, err)
				}
				if strings.Contains(err.Error(), "private-provider-detail") {
					t.Error("provider text leaked into format error")
				}
				if reads != 1 || filtered != 0 {
					t.Errorf("discovery failed after side effects: reads=%d filters=%d", reads, filtered)
				}
			})
		}
	}
}

func TestDatastoreInventoryRequiresCompleteHTTP200Body(t *testing.T) {
	for _, failure := range []string{"read after JSON", "close", "size", "created", "accepted", "partial"} {
		t.Run(failure, func(t *testing.T) {
			body := &inventoryResponseBody{Reader: strings.NewReader(`{"data":[]}`)}
			status := http.StatusOK
			switch failure {
			case "read after JSON":
				body.Reader = io.MultiReader(body.Reader, inventoryReadFailure{})
			case "close":
				body.closeErr = errors.New("fixture close failure")
			case "size":
				body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))
			case "created":
				status = http.StatusCreated
			case "accepted":
				status = http.StatusAccepted
			case "partial":
				status = http.StatusPartialContent
			}
			client, err := NewClient(ClientConfig{Host: "https://fixture.invalid", TokenName: "fixture@pbs!read", TokenValue: "fixture-only"})
			if err != nil {
				t.Fatal(err)
			}
			reads := 0
			client.httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
				reads++
				return &http.Response{StatusCode: status, Body: body, Header: make(http.Header)}, nil
			})}
			rows, err := client.GetDatastores(context.Background())
			if err == nil || rows != nil {
				t.Errorf("incomplete datastore response accepted: rows=%+v err=%v", rows, err)
			}
			if reads != 1 || !body.closed || int64(body.bytes) > maxResponseBodyBytes+1 {
				t.Errorf("reads=%d closed=%t bytes=%d", reads, body.closed, body.bytes)
			}
		})
	}
}

func TestDatastoreInventoryAcceptsCompleteScopeBeforeFiltering(t *testing.T) {
	for _, empty := range []bool{false, true} {
		t.Run(map[bool]string{false: "included-and-excluded", true: "empty"}[empty], func(t *testing.T) {
			requests := []string{}
			client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
				requests = append(requests, r.URL.RequestURI())
				switch r.URL.Path {
				case "/api2/json/admin/datastore":
					data := []map[string]string{}
					if !empty {
						data = []map[string]string{{"store": "archive", "comment": "literal <not markup>"}, {"store": "excluded"}}
					}
					_ = json.NewEncoder(w).Encode(map[string]any{"data": data, "errors": nil, "metadata": map[string]any{"future": "温度"}})
				case "/api2/json/admin/datastore/archive/rrd":
					_, _ = w.Write([]byte(`{"data":[]}`))
				case "/api2/json/admin/datastore/archive/status":
					_, _ = w.Write([]byte(`{"data":{"total":100,"used":40,"avail":60,"deduplication-factor":2}}`))
				default:
					t.Errorf("unexpected child read: %s", r.URL)
					http.NotFound(w, r)
				}
			})
			defer server.Close()
			rows, err := client.GetDatastoresFiltered(context.Background(), func(name string) bool { return name != "excluded" })
			if err != nil {
				t.Fatal(err)
			}
			if empty {
				if len(rows) != 0 || len(requests) != 1 {
					t.Fatalf("empty response rows=%+v requests=%v", rows, requests)
				}
				return
			}
			if len(rows) != 1 || rows[0].Store != "archive" || rows[0].Status != "available" || rows[0].Total != 100 || rows[0].Used != 40 || rows[0].Avail != 60 || rows[0].DeduplicationFactor != 2 || len(requests) != 3 {
				t.Fatalf("ordinary read/filter changed: rows=%+v requests=%v", rows, requests)
			}
		})
	}
}