Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reject malformed Proxmox task logs before guest synthesis

pulse-triage[bot]edea3324a633
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 2c02e8fd88..b7436c8322 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,36 @@
## Purpose
+### Backup guest results require an unambiguous task log
+
+The task-log reader requests the prefix from line one and accepts only a whole,
+bounded UTF-8 HTTP 200 response with an explicit array. Missing/null data,
+contradictory errors, duplicate/case-aliased fields, trailing JSON/text, unread
+suffixes and invalid rows reject the entire read. Each row needs a native
+integer line number and string text; numbers must be contiguous from one in
+response order. Do not sort, fill gaps, discard bad rows or silently cap an
+over-limit reply. Empty text, unique provider metadata, explicit empty arrays
+and contiguous prefixes up to the existing 20,000-line limit remain valid.
+
+A malformed success is unread, not empty or a permission denial. The ordinary
+collector fallback retains accepted guest results and their original log
+observation, never renewing alert deferral or inventing a guest completion.
+An ended parent still withdraws running intent without lending its result to
+a guest. Independent nodes/connections progress; an ordinary valid complete
+log recovers and uses the existing immutable cache. Explicit empty reads do
+not resurrect older guest data on a later failed read. Existing attribution,
+terminal-marker, request-budget, auth, scope and guest-operation rules stay
+unchanged. Format diagnostics never reproduce log payloads.
+
+`pkg/proxmox/task_log_response_test.go` exercises the real HTTP reader, body
+bounds/closure, line ordering, malformed fields and accepted capped prefixes.
+`internal/monitoring/vzdump_task_log_response_test.go` crosses HTTP, polling,
+accepted cache, durable task outcomes and backup-intent expiry in cold/warm,
+running/ended and ordinary recovery cycles. Actual produced frames are consumed
+by desktop/narrow Backups Coverage and task disclosures. These are read-only
+source correctness proofs, not native recovery or a restored guest.
+
+
### Unfinished backup logs cannot monopolise the read allowance
Multi-guest vzdump log collection keeps its 25-request cap per poll. Pending
@@ -4042,6 +4072,7 @@ cleanup so readers cannot retain orphaned runtime or alert projections.
21b. `pkg/proxmox/client.go`
21c. `pkg/proxmox/io_counters.go`
21d. `pkg/proxmox/backup_history_response.go`
+21e. `pkg/proxmox/task_log_response.go`
22. `pkg/proxmox/zfs.go`
22a. `pkg/pbs/client.go`
23. `internal/monitoring/guest_memory_sources.go`
@@ -4818,7 +4849,10 @@ named and default organizations. The custom event envelope stays unchanged.
`TestResolveBackupIntentContextAcceptsSynthesizedJobGuestTask` in
`internal/monitoring/monitor_alert_intent_test.go`, and
`TestClusterClient_GetTaskLog` in
- `pkg/proxmox/cluster_client_api_test.go`.
+ `pkg/proxmox/cluster_client_api_test.go`. Task log response admission and
+ no-renewal/recovery behaviour are also checked by
+ `pkg/proxmox/task_log_response_test.go` and
+ `internal/monitoring/vzdump_task_log_response_test.go`.
14. A TrueNAS app container reported as `EXITED` is a completed workload and
must raise no incident. TrueNAS classifies container exits before Pulse
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 2f06c9eb7a..619ace2bc5 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6416,6 +6416,7 @@
"pkg/proxmox/client.go",
"pkg/proxmox/cluster_client.go",
"pkg/proxmox/io_counters.go",
+ "pkg/proxmox/task_log_response.go",
"pkg/proxmox/zfs.go",
"pkg/tlsutil/certificate.go"
],
@@ -6425,6 +6426,20 @@
"exact_files": [],
"require_explicit_path_policy_coverage": true,
"path_policies": [
+ {
+ "id": "task-log-response",
+ "label": "bounded ordered task log admission and guest evidence continuity proof",
+ "match_prefixes": [],
+ "match_files": [
+ "pkg/proxmox/task_log_response.go"
+ ],
+ "allow_same_subsystem_tests": false,
+ "test_prefixes": [],
+ "exact_files": [
+ "internal/monitoring/vzdump_task_log_response_test.go",
+ "pkg/proxmox/task_log_response_test.go"
+ ]
+ },
{
"id": "backup-history-response",
"label": "bounded backup inventory and completion evidence proof",
@@ -6747,7 +6762,7 @@
},
{
"id": "proxmox-guest-counter-runtime",
- "label": "Proxmox guest counter presence, receipt-time and memory integrity proof",
+ "label": "Proxmox guest counter, memory integrity and task log admission proof",
"match_prefixes": [],
"match_files": [
"pkg/proxmox/client.go",
@@ -6763,7 +6778,8 @@
"internal/monitoring/ratetracker_test.go",
"pkg/proxmox/host_diagnostics_regression_test.go",
"pkg/proxmox/io_counters_test.go",
- "pkg/proxmox/meminfo_integrity_test.go"
+ "pkg/proxmox/meminfo_integrity_test.go",
+ "pkg/proxmox/task_log_response_test.go"
]
},
{
diff --git a/internal/monitoring/vzdump_task_log_response_test.go b/internal/monitoring/vzdump_task_log_response_test.go
new file mode 100644
index 0000000000..e64ba8b1d1
--- /dev/null
+++ b/internal/monitoring/vzdump_task_log_response_test.go
@@ -0,0 +1,233 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/require"
+)
+
+// Actual HTTP reader -> ordinary polling -> live/cache/durable guest results.
+// No guest-agent endpoint, task operation or external provider is involved.
+func TestVzdumpTaskLogMalformedReadRetainsAcceptedEvidence(t *testing.T) {
+ for _, seeded := range []bool{false, true} {
+ for _, malformed := range []string{"null", "trailing", "unfinished", "gap", "reordered", "null-text", "conflicting-error"} {
+ t.Run(fmt.Sprintf("seeded=%t/%s", seeded, malformed), func(t *testing.T) {
+ ctx := context.Background()
+ job, prefix := vzdumpGuestResultFixture()
+ var phase, reads atomic.Int32
+ completed := append(append([]proxmox.TaskLogLine{}, prefix...),
+ proxmox.TaskLogLine{LineNumber: 6, Text: "INFO: Finished Backup of VM 103 (00:02:00)"},
+ proxmox.TaskLogLine{LineNumber: 7, Text: "TASK ERROR: job errors"})
+ badLines := append([]proxmox.TaskLogLine{}, completed...)
+ switch malformed {
+ case "gap":
+ badLines[5].LineNumber++
+ case "reordered":
+ badLines[4], badLines[5] = badLines[5], badLines[4]
+ }
+ bad, err := json.Marshal(map[string]any{"data": badLines})
+ require.NoError(t, err)
+ switch malformed {
+ case "null":
+ bad = []byte(`{"data":null}`)
+ case "trailing":
+ bad = append(bad, []byte(` {"data":[]}`)...)
+ case "null-text":
+ bad = []byte(`{"data":[{"n":1,"t":null},{"n":2,"t":"INFO: Finished Backup of VM 103 (00:02:00)"}]}`)
+ case "conflicting-error":
+ bad = append(bad[:len(bad)-1], []byte(`,"errors":{"message":"fixture-private-detail"}}`)...)
+ }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ p := phase.Load()
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"pve1","status":"online"},{"node":"pve2","status":"online"}]}`)
+ case "/api2/json/access/permissions":
+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})
+ case "/api2/json/nodes/pve1/tasks":
+ task := proxmox.Task{UPID: testJobUPID, Node: "pve1", Type: "vzdump", StartTime: job.StartTime.Unix()}
+ if p >= 2 {
+ task.EndTime, task.Status = job.EndTime.Unix(), "job errors"
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{task}})
+ case "/api2/json/nodes/pve2/tasks":
+ status := "OK"
+ if p > 0 {
+ status = "ERROR: independent node failure"
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{{UPID: "independent", Node: "pve2", Type: "vzdump", ID: "104", StartTime: job.StartTime.Unix(), EndTime: job.EndTime.Unix(), Status: status}}})
+ default:
+ require.Equal(t, "/api2/json/nodes/pve1/tasks/"+testJobUPID+"/log", r.URL.Path)
+ require.Equal(t, http.MethodGet, r.Method)
+ reads.Add(1)
+ if p == 0 {
+ json.NewEncoder(w).Encode(map[string]any{"data": prefix})
+ } else if p == 3 {
+ json.NewEncoder(w).Encode(map[string]any{"data": completed})
+ } else {
+ if malformed == "unfinished" {
+ w.Header().Set("Content-Length", fmt.Sprint(len(bad)+100))
+ }
+ w.Write(bad)
+ }
+ }
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})
+ require.NoError(t, err)
+ manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
+ m := &Monitor{state: models.NewState(), recoveryManager: manager}
+ store, err := manager.StoreForOrg("default")
+ require.NoError(t, err)
+ defer store.Close()
+ peer := models.BackupTask{ID: "peer-same-job-vm103", Instance: "other", Node: "pve1", VMID: 103, Type: "vzdump", Status: "error", Error: "independent connection failure", StartTime: job.StartTime, EndTime: job.EndTime, ObservedAt: job.StartTime}
+ m.state.UpdateBackupTasksForInstance(peer.Instance, []models.BackupTask{peer})
+ frames := map[string]any{}
+ var accepted []models.BackupTask
+ key := "site|" + testJobUPID
+ observed := time.Now().UTC().Add(-2 * backupIntentEvidenceMaxAge)
+ first := 1
+ if seeded {
+ first = 0
+ }
+ for p := first; p <= 4; p++ {
+ phase.Store(int32(p))
+ // Model separate polls beyond the persisted millisecond watermark.
+ coverage, err := store.BackupHistoryCoverage(ctx, job.StartTime, time.Now().UTC())
+ require.NoError(t, err)
+ for _, source := range coverage.Sources {
+ if wait := time.Until(source.LastObservedAt.Add(time.Millisecond)); wait > 0 {
+ time.Sleep(wait)
+ }
+ }
+ m.pollBackupTasks(ctx, "site", client)
+ snapshot := m.state.GetSnapshot()
+ var guests []models.BackupTask
+ for _, task := range snapshot.PVEBackups.BackupTasks {
+ if task.Instance == "other" {
+ require.Equal(t, peer, task)
+ } else if task.Node == "pve2" {
+ require.Equal(t, 104, task.VMID)
+ if p > 0 {
+ require.Equal(t, "ERROR: independent node failure", task.Status)
+ }
+ } else if task.VMID > 0 {
+ guests = append(guests, task)
+ }
+ }
+ if p == 0 {
+ assertVzdumpGuestResults(t, guests, "running")
+ _, active := m.resolveBackupIntentContext("", "site", "pve1", 103, time.Now())
+ require.True(t, active, "accepted fresh log establishes guest backup intent")
+ entry := m.vzdumpJobTaskCache[key]
+ for i := range entry.tasks {
+ entry.tasks[i].ObservedAt = observed
+ }
+ m.vzdumpJobTaskCache[key] = entry
+ accepted = append([]models.BackupTask{}, entry.tasks...)
+ } else if p < 3 && !seeded {
+ require.Empty(t, guests, "malformed cold reads cannot invent guest results")
+ } else if p == 1 {
+ require.Equal(t, accepted, guests, "unread log retains exact accepted evidence without renewal")
+ } else if p == 2 {
+ assertVzdumpGuestResults(t, guests, "unknown")
+ for _, guest := range guests {
+ require.Equal(t, observed, guest.ObservedAt)
+ require.False(t, guest.EndTime.IsZero())
+ }
+ } else {
+ assertVzdumpGuestResults(t, guests, "OK")
+ require.True(t, m.vzdumpJobTaskCache[key].final, "ordinary complete read recovers and finalises")
+ }
+ if p > 0 && p < 3 {
+ require.False(t, m.vzdumpJobTaskCache[key].final)
+ _, active := m.resolveBackupIntentContext("", "site", "pve1", 103, time.Now())
+ require.False(t, active, "malformed success cannot renew a guest's alert deferral")
+ }
+ points, _, err := store.ListPoints(ctx, recovery.ListPointsOptions{Kind: recovery.KindOther})
+ require.NoError(t, err)
+ byID := map[string]recovery.RecoveryPoint{}
+ for _, point := range points {
+ byID[point.ID] = point
+ }
+ parentID := "pve-task:site-" + testJobUPID
+ require.Equal(t, p >= 3, byID[parentID].Details["guestTaskDetailsComplete"], "malformed reads cannot qualify all-guest report coverage")
+ last, present := byID[parentID+"-vm103"]
+ if !seeded && p < 3 {
+ require.False(t, present)
+ } else {
+ require.True(t, present)
+ want := recovery.OutcomeRunning
+ if p == 2 {
+ want = recovery.OutcomeUnknown
+ } else if p >= 3 {
+ want = recovery.OutcomeSuccess
+ }
+ require.Equal(t, want, last.Outcome)
+ }
+ frames[[]string{"accepted", "unread-running", "unread-ended", "recovered", "cached"}[p]] = map[string]any{"pve": map[string]any{"data": snapshot.PVEBackups}}
+ }
+ require.EqualValues(t, 4-first, reads.Load(), "complete cached log is not refetched; each malformed poll spends one request")
+ if seeded && malformed == "trailing" && os.Getenv("PULSE_TASK_LOG_FRAMES") == "1" {
+ b, err := json.Marshal(frames)
+ require.NoError(t, err)
+ t.Log("TASK_LOG_FRAMES=" + string(b))
+ }
+ })
+ }
+ }
+}
+
+func TestVzdumpTaskLogExplicitEmptyDoesNotResurrectGuests(t *testing.T) {
+ job, lines := vzdumpGuestResultFixture()
+ var phase atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"pve1","status":"online"}]}`)
+ case "/api2/json/access/permissions":
+ fmt.Fprint(w, `{"data":{"/nodes/pve1":{"Sys.Audit":1}}}`)
+ case "/api2/json/nodes/pve1/tasks":
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{{UPID: testJobUPID, Node: "pve1", Type: "vzdump", StartTime: job.StartTime.Unix()}}})
+ default:
+ require.True(t, strings.HasSuffix(r.URL.Path, "/log"))
+ switch phase.Load() {
+ case 0, 3:
+ json.NewEncoder(w).Encode(map[string]any{"data": lines})
+ case 1:
+ fmt.Fprint(w, `{"data":[]}`)
+ case 2:
+ fmt.Fprint(w, `{"data":null}`)
+ }
+ }
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})
+ require.NoError(t, err)
+ m := &Monitor{state: models.NewState()}
+ for p := range 4 {
+ phase.Store(int32(p))
+ m.pollBackupTasks(context.Background(), "site", client)
+ count := len(m.state.GetBackupTasks())
+ if p == 0 || p == 3 {
+ require.Equal(t, 4, count)
+ } else {
+ require.Equal(t, 1, count, "valid empty clears guest prefix; later unread cannot resurrect older cache data")
+ }
+ require.False(t, m.vzdumpJobTaskCache["site|"+testJobUPID].final)
+ }
+}
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go
index 82f2438cb8..df52196825 100644
--- a/pkg/proxmox/client.go
+++ b/pkg/proxmox/client.go
@@ -1476,25 +1476,7 @@ func (c *Client) GetTaskLog(ctx context.Context, node, upid string) ([]TaskLogLi
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, err := readResponseBodyLimited(resp.Body)
- if err != nil {
- return nil, err
- }
- return nil, fmt.Errorf("failed to get task log (status %d): %s", resp.StatusCode, strings.TrimSpace(string(body)))
- }
-
- var result struct {
- Data []TaskLogLine `json:"data"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, err
- }
-
- return result.Data, nil
+ return readTaskLogResponse(resp)
}
// GetContainerInterfaces returns the network interfaces (with IPs) for a container.
diff --git a/pkg/proxmox/task_log_response.go b/pkg/proxmox/task_log_response.go
new file mode 100644
index 0000000000..dcbfdb6cec
--- /dev/null
+++ b/pkg/proxmox/task_log_response.go
@@ -0,0 +1,62 @@
+package proxmox
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "unicode/utf8"
+)
+
+var errInvalidTaskLogResponse = errors.New("invalid Proxmox task log response")
+
+// Guest results depend on log order. Accept only a complete bounded response
+// containing the requested prefix, not a decodable success before a read error
+// or reordered/ambiguous lines. Never copy log text into format diagnostics.
+func readTaskLogResponse(resp *http.Response) ([]TaskLogLine, error) {
+ body, readErr := readResponseBodyLimited(resp.Body)
+ closeErr := resp.Body.Close()
+ if readErr != nil || closeErr != nil {
+ return nil, errors.Join(readErr, closeErr)
+ }
+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {
+ return nil, errInvalidTaskLogResponse
+ }
+ envelope, valid := inventoryResponseObject(body, "data", "errors")
+ if !valid {
+ return nil, errInvalidTaskLogResponse
+ }
+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {
+ return nil, errInvalidTaskLogResponse
+ }
+ data := bytes.TrimSpace(envelope["data"])
+ if len(data) == 0 || data[0] != '[' {
+ return nil, errInvalidTaskLogResponse
+ }
+ var rows []json.RawMessage
+ if json.Unmarshal(data, &rows) != nil || len(rows) > taskLogLineLimit {
+ return nil, errInvalidTaskLogResponse
+ }
+ lines := make([]TaskLogLine, 0, len(rows))
+ for i, row := range rows {
+ fields, valid := inventoryResponseObject(row, "n", "t")
+ if !valid {
+ return nil, errInvalidTaskLogResponse
+ }
+ for _, name := range []string{"n", "t"} {
+ value := bytes.TrimSpace(fields[name])
+ if len(value) == 0 || bytes.Equal(value, []byte("null")) {
+ return nil, errInvalidTaskLogResponse
+ }
+ }
+ var line TaskLogLine
+ if json.Unmarshal(row, &line) != nil || line.LineNumber != int64(i+1) {
+ return nil, errInvalidTaskLogResponse
+ }
+ lines = append(lines, line)
+ }
+ // An explicit empty log and a full capped prefix remain usable reads, but
+ // neither proves that all guests/results have been observed. The collector
+ // keeps its existing terminal-marker and attribution checks for finality.
+ return lines, nil
+}
diff --git a/pkg/proxmox/task_log_response_test.go b/pkg/proxmox/task_log_response_test.go
new file mode 100644
index 0000000000..dd6589524e
--- /dev/null
+++ b/pkg/proxmox/task_log_response_test.go
@@ -0,0 +1,134 @@
+package proxmox
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestTaskLogRejectsUnverifiedResponse(t *testing.T) {
+ const row = `{"n":1,"t":"INFO: Starting Backup of VM 101 (qemu)"}`
+ const response = `{"data":[` + row + `]}`
+ for _, tc := range []struct {
+ name, body string
+ unfinished bool
+ }{
+ {"missing-data", `{}`, false},
+ {"null-data", `{"data":null}`, false},
+ {"null-envelope", `null`, false},
+ {"not-array", `{"data":{}}`, false},
+ {"error-with-data", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, false},
+ {"case-data", `{"Data":[]}`, false},
+ {"duplicate-data", `{"data":[` + row + `],"data":[]}`, false},
+ {"escaped-duplicate-data", `{"data":[],"\u0064ata":[]}`, false},
+ {"case-overwrite", `{"data":[` + row + `],"Data":[]}`, false},
+ {"case-error", `{"data":[],"Errors":null}`, false},
+ {"trailing-value", response + ` {"data":[]}`, false},
+ {"trailing-text", response + `fixture-private-detail`, false},
+ {"unfinished-body", response, true},
+ {"oversized-body", response + strings.Repeat(" ", int(maxResponseBodyBytes)), false},
+ {"invalid-utf8", `{"data":[` + row + `],"metadata":"` + string([]byte{0xff}) + `"}`, false},
+ {"null-row", `{"data":[` + row + `,null]}`, false},
+ {"missing-number", `{"data":[{"t":"fixture-private-detail"}]}`, false},
+ {"null-number", `{"data":[{"n":null,"t":"text"}]}`, false},
+ {"string-number", `{"data":[{"n":"1","t":"text"}]}`, false},
+ {"fractional-number", `{"data":[{"n":1.5,"t":"text"}]}`, false},
+ {"zero-number", `{"data":[{"n":0,"t":"text"}]}`, false},
+ {"negative-number", `{"data":[{"n":-1,"t":"text"}]}`, false},
+ {"missing-first-line", `{"data":[{"n":2,"t":"text"}]}`, false},
+ {"missing-text", `{"data":[{"n":1}]}`, false},
+ {"null-text", `{"data":[{"n":1,"t":null}]}`, false},
+ {"non-string-text", `{"data":[{"n":1,"t":42}]}`, false},
+ {"duplicate-number", `{"data":[{"n":2,"n":1,"t":"text"}]}`, false},
+ {"duplicate-text", `{"data":[{"n":1,"t":"first","\u0074":"second"}]}`, false},
+ {"case-number", `{"data":[{"n":1,"N":2,"t":"text"}]}`, false},
+ {"case-text", `{"data":[{"n":1,"t":"first","T":"second"}]}`, false},
+ {"repeated-line", `{"data":[` + row + `,` + row + `]}`, false},
+ {"gap-after-good-line", `{"data":[` + row + `,{"n":3,"t":"INFO: Finished Backup of VM 101 (00:01:00)"}]}`, false},
+ {"reordered-lines", `{"data":[{"n":2,"t":"INFO: Finished Backup of VM 101 (00:01:00)"},` + row + `]}`, false},
+ {"invalid-later-row", `{"data":[` + row + `,{"n":2,"t":false}]}`, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var reads atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reads.Add(1)
+ require.Equal(t, "/api2/json/nodes/member/tasks/job/log", r.URL.Path)
+ require.Equal(t, "0", r.URL.Query().Get("start"))
+ require.Equal(t, fmt.Sprint(taskLogLineLimit), r.URL.Query().Get("limit"))
+ if tc.unfinished {
+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))
+ } else {
+ w.(http.Flusher).Flush() // whole-body bound, not a Content-Length shortcut
+ }
+ fmt.Fprint(w, tc.body)
+ }))
+ defer server.Close()
+ lines, err := backupTestClient(t, server.URL).GetTaskLog(context.Background(), "member", "job")
+ require.Error(t, err)
+ require.Nil(t, lines, "a good prefix cannot publish partial guest evidence")
+ require.NotContains(t, err.Error(), "fixture-private-detail")
+ require.EqualValues(t, 1, reads.Load(), "invalid successful reads are not retried")
+ _, typed := APIErrorStatus(err)
+ require.False(t, typed, "malformed success is not a permission denial")
+ })
+ }
+}
+
+func TestTaskLogAcceptsExplicitPrefixAndExtensions(t *testing.T) {
+ for _, tc := range []struct {
+ name, body string
+ want []TaskLogLine
+ }{
+ {"empty", `{"data":[]}`, []TaskLogLine{}},
+ {"empty-with-metadata", `{"data":[],"errors":null,"total":0}`, []TaskLogLine{}},
+ {"blank-and-unicode", `{"data":[{"n":1,"t":"","provider-metadata":true},{"n":2,"t":"INFO: café 日本語"}],"errors":null}`, []TaskLogLine{{1, ""}, {2, "INFO: café 日本語"}}},
+ {"running-prefix", `{"data":[{"n":1,"t":"INFO: Starting Backup of VM 101 (qemu)"}]}`, []TaskLogLine{{1, "INFO: Starting Backup of VM 101 (qemu)"}}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, tc.body) }))
+ defer server.Close()
+ lines, err := backupTestClient(t, server.URL).GetTaskLog(context.Background(), "member", "job")
+ require.NoError(t, err)
+ require.Equal(t, tc.want, lines)
+ })
+ }
+ for _, count := range []int{taskLogLineLimit, taskLogLineLimit + 1} {
+ t.Run(fmt.Sprint(count), func(t *testing.T) {
+ rows := make([]TaskLogLine, count)
+ for i := range rows {
+ rows[i] = TaskLogLine{LineNumber: int64(i + 1), Text: "INFO: progress"}
+ }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]any{"data": rows}) }))
+ defer server.Close()
+ lines, err := backupTestClient(t, server.URL).GetTaskLog(context.Background(), "member", "job")
+ if count == taskLogLineLimit {
+ require.NoError(t, err)
+ require.Equal(t, rows, lines, "the existing capped prefix stays usable, not complete")
+ } else {
+ require.Error(t, err)
+ require.Nil(t, lines, "never silently truncate an over-limit reply")
+ }
+ })
+ }
+}
+
+func TestTaskLogReadClosureAndStatus(t *testing.T) {
+ for _, status := range []int{http.StatusNoContent, http.StatusPartialContent, http.StatusInternalServerError} {
+ lines, err := readTaskLogResponse(&http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(`{"data":[]}`))})
+ require.Error(t, err)
+ require.Nil(t, lines)
+ }
+ body := &snapshotCloseFailure{Reader: strings.NewReader(`{"data":[{"n":1,"t":"accepted prefix"}]}`)}
+ lines, err := readTaskLogResponse(&http.Response{StatusCode: http.StatusOK, Body: body})
+ require.Error(t, err)
+ require.Nil(t, lines)
+ require.True(t, body.closed)
+}