Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep unread PBS namespaces in the live backup inventory

pulse-triage[bot]84921736d369
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 31c9177da0..fcd463b53a 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,32 @@
## Purpose
+### PBS unread namespaces retain only their own last accepted live files
+
+When a PBS group-list request fails transiently, its previous live snapshot
+rows remain inspectable even if a sibling namespace succeeds. A transient
+namespace-discovery failure similarly retains known but unqueried namespaces,
+without guessing or issuing hidden-namespace requests. Accepted rows take
+priority over carried rows at the existing live-state bound. Carried dates,
+identities and snapshot-cache times are unchanged, and carried rows never enter
+the observed-group batch or renew recovery-point evidence. Provider history
+stays partial/unavailable rather than claiming protection.
+
+An actual access rejection still withdraws that namespace's raw rows, including
+when every namespace failed and another failure was transient. Failed discovery
+cannot override a namespace request actually made: neither a rejection nor an
+accepted empty listing can be undone by carry-forward. Durable historical facts
+remain qualified by collection/access evidence until a complete accepted scan
+can reconcile absence. Ordinary successful reads restore the inventory. Public
+JSON, identifiers, bounds, policy and native backup/recovery operations are
+unchanged. `pbs_namespace_continuity_test.go` exercises VM/container scopes,
+repeated transient/malformed reads, discovery loss, mixed denial, accepted empty
+siblings, connection isolation and complete recovery/removal. Its optional
+lifecycle frames drive the production Backups consumer and API hooks in
+desktop/narrow browser proof. This is source continuity, not archive availability,
+installed guest recovery or #3248's diagnosed stale-date cause.
+
+
### Same-named local backup disks retain independent read origins
An accepted storage listing on one PVE node cannot erase last-known files on
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 0331468b6a..78a3f733cb 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,32 @@
## Purpose
+### PBS unread namespaces retain only their own last accepted live files
+
+When a PBS group-list request fails transiently, its previous live snapshot
+rows remain inspectable even if a sibling namespace succeeds. A transient
+namespace-discovery failure similarly retains known but unqueried namespaces,
+without guessing or issuing hidden-namespace requests. Accepted rows take
+priority over carried rows at the existing live-state bound. Carried dates,
+identities and snapshot-cache times are unchanged, and carried rows never enter
+the observed-group batch or renew recovery-point evidence. Provider history
+stays partial/unavailable rather than claiming protection.
+
+An actual access rejection still withdraws that namespace's raw rows, including
+when every namespace failed and another failure was transient. Failed discovery
+cannot override a namespace request actually made: neither a rejection nor an
+accepted empty listing can be undone by carry-forward. Durable historical facts
+remain qualified by collection/access evidence until a complete accepted scan
+can reconcile absence. Ordinary successful reads restore the inventory. Public
+JSON, identifiers, bounds, policy and native backup/recovery operations are
+unchanged. `pbs_namespace_continuity_test.go` exercises VM/container scopes,
+repeated transient/malformed reads, discovery loss, mixed denial, accepted empty
+siblings, connection isolation and complete recovery/removal. Its optional
+lifecycle frames drive the production Backups consumer and API hooks in
+desktop/narrow browser proof. This is source continuity, not archive availability,
+installed guest recovery or #3248's diagnosed stale-date cause.
+
+
### Backup activity ranges follow local calendar dates
The existing 7/30/90/365-day Proxmox backup activity windows contain exactly
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 3f23b7cdfe..02c15a1ea0 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -1671,6 +1671,11 @@ type pbsBackupGroupKey struct {
backupID string
}
+type pbsBackupNamespaceKey struct {
+ datastore string
+ namespace string
+}
+
type cachedPBSGroup struct {
snapshots []models.PBSBackup
latest time.Time
@@ -1694,6 +1699,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
observedGroups := make(map[pbsBackupGroupKey]struct{})
retainedGroups := make(map[pbsBackupGroupKey]struct{}, len(existingGroups))
refreshedGroups := make(map[pbsBackupGroupKey]time.Time)
+ queriedNamespaces := make(map[pbsBackupNamespaceKey]struct{})
+ transientNamespaces := make(map[pbsBackupNamespaceKey]struct{})
+ transientDiscovery := make(map[string]struct{})
datastoreCount := len(datastores) // Number of datastores to query
datastoreFetches := 0 // Number of successful datastore fetches
datastoreErrors := 0 // Datastores with incomplete history
@@ -1728,6 +1736,8 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
datastoreNamespaceErrors++
if !shouldReuseCachedPBSBackups(err) {
datastoreTerminalNamespaceErrors++
+ } else {
+ transientDiscovery[ds.Name] = struct{}{}
}
}
groupsReused := 0
@@ -1742,10 +1752,14 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
}
groups, err := client.ListBackupGroups(ctx, ds.Name, namespace)
+ namespaceKey := pbsBackupNamespaceKey{datastore: ds.Name, namespace: namespace}
+ queriedNamespaces[namespaceKey] = struct{}{}
if err != nil {
datastoreNamespaceErrors++
if !shouldReuseCachedPBSBackups(err) {
datastoreTerminalNamespaceErrors++
+ } else {
+ transientNamespaces[namespaceKey] = struct{}{}
}
log.Error().
Err(err).
@@ -1889,13 +1903,6 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
Str("instance", instanceName).
Str("datastore", ds.Name).
Msg("No namespaces succeeded for PBS datastore; using cached backups")
- for key, entry := range existingGroups {
- if key.datastore != ds.Name || len(entry.snapshots) == 0 {
- continue
- }
- allBackups = appendPBSBackupsWithinLimit(allBackups, entry.snapshots)
- retainedGroups[key] = struct{}{}
- }
}
}
}
@@ -1905,6 +1912,26 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
return
}
+ // A readable sibling namespace cannot establish absence in an unread one.
+ // Carry only the transiently unread scope, after accepted rows so old
+ // inventory cannot displace fresh results at the existing live-state bound.
+ // Discovery failure may hide old namespaces, but never overrides a request
+ // we actually made (notably an access rejection or a successful empty list).
+ var carriedBackups []models.PBSBackup
+ for key, entry := range existingGroups {
+ namespaceKey := pbsBackupNamespaceKey{datastore: key.datastore, namespace: key.namespace}
+ _, unread := transientNamespaces[namespaceKey]
+ _, undiscovered := transientDiscovery[key.datastore]
+ _, queried := queriedNamespaces[namespaceKey]
+ if !unread && !(undiscovered && !queried) {
+ continue
+ }
+ carriedBackups = append(carriedBackups, entry.snapshots...)
+ retainedGroups[key] = struct{}{}
+ }
+ sortPBSBackupsByLatest(carriedBackups)
+ allBackups = appendPBSBackupsWithinLimit(allBackups, carriedBackups)
+
log.Info().
Str("instance", instanceName).
Int("count", len(allBackups)).
@@ -1928,7 +1955,8 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
}
// Decide whether to keep existing backups when all queries failed
- if shouldPreservePBSBackupsWithTerminal(datastoreCount, datastoreFetches, datastoreTerminalFailures) {
+ if collectionTerminalFailures == 0 &&
+ shouldPreservePBSBackupsWithTerminal(datastoreCount, datastoreFetches, datastoreTerminalFailures) {
if protectionObservationErr == nil {
m.ingestProtectionProviderObservationsAsync(
[]recovery.ProtectionProviderObservation{protectionObservation},
diff --git a/internal/monitoring/pbs_namespace_continuity_test.go b/internal/monitoring/pbs_namespace_continuity_test.go
new file mode 100644
index 0000000000..294fc5e7cf
--- /dev/null
+++ b/internal/monitoring/pbs_namespace_continuity_test.go
@@ -0,0 +1,362 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "reflect"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+)
+
+type pbsNamespaceContinuityFixture struct {
+ *pbsPosturePollFixture
+ rootStatus, tenantStatus atomic.Int32
+ rootEmpty, tenantEmpty atomic.Bool
+ rootAt, tenantAt atomic.Int64
+ rootReads, tenantReads atomic.Int32
+}
+
+func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceContinuityFixture {
+ t.Helper()
+ f := &pbsNamespaceContinuityFixture{pbsPosturePollFixture: newPBSPosturePollFixture(t, 2, kind)}
+ f.rootAt.Store(f.backupAt)
+ f.tenantAt.Store(f.backupAt)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ id, at, status, empty := "100", f.rootAt.Load(), f.rootStatus.Load(), f.rootEmpty.Load()
+ if r.URL.Query().Get("ns") == "tenant" {
+ id, at, status, empty = "101", f.tenantAt.Load(), f.tenantStatus.Load(), f.tenantEmpty.Load()
+ f.tenantReads.Add(1)
+ } else {
+ f.rootReads.Add(1)
+ }
+ var data any = []any{}
+ switch r.URL.Path {
+ case "/api2/json/admin/datastore/archive/groups":
+ if status == http.StatusOK {
+ _, _ = w.Write([]byte(`{"data":`))
+ return
+ }
+ if status != 0 {
+ http.Error(w, "fixture unavailable", int(status))
+ return
+ }
+ if !empty {
+ data = []map[string]any{{"backup-type": kind, "backup-id": id, "last-backup": at, "backup-count": 1}}
+ }
+ case "/api2/json/admin/datastore/archive/snapshots":
+ data = []map[string]any{{"backup-type": kind, "backup-id": id, "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}
+ default:
+ http.NotFound(w, r)
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": data})
+ }))
+ t.Cleanup(server.Close)
+ var err error
+ f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return f
+}
+
+func (f *pbsNamespaceContinuityFixture) collect(t *testing.T, discovered bool, discoveryErr error) {
+ t.Helper()
+ ds := models.PBSDatastore{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}}}
+ if discovered {
+ ds.Namespaces = append(ds.Namespaces, models.PBSNamespace{Path: "tenant"})
+ }
+ errs := map[string]error{}
+ if discoveryErr != nil {
+ errs["archive"] = discoveryErr
+ }
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, []models.PBSDatastore{ds}, errs)
+ f.wait(t)
+}
+
+func (f *pbsNamespaceContinuityFixture) live() map[string]models.PBSBackup {
+ out := map[string]models.PBSBackup{}
+ for _, b := range f.m.state.GetSnapshot().PBSBackups {
+ if b.Instance == "pbs-main" {
+ out[b.VMID] = b
+ }
+ }
+ return out
+}
+
+func TestPBSNamespaceTransientGroupReadKeepsLiveInventory(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, ns := range []string{"", "tenant"} {
+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusOK} {
+ t.Run(fmt.Sprintf("%s/ns=%s/status=%d", kind, ns, status), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ f.collect(t, true, nil)
+ before, points := f.live(), f.points(t, "pbs-main")
+ if len(before) != 2 || len(points) != 2 {
+ t.Fatal("healthy control did not list both namespaces")
+ }
+ failed, accepted := "100", "101"
+ if ns == "tenant" {
+ failed, accepted = "101", "100"
+ f.tenantStatus.Store(int32(status))
+ f.rootAt.Add(15 * 60)
+ } else {
+ f.rootStatus.Store(int32(status))
+ f.tenantAt.Add(15 * 60)
+ }
+ key := pbsBackupGroupKey{datastore: "archive", namespace: ns, backupType: kind, backupID: failed}
+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
+ for range 2 {
+ f.collect(t, true, nil)
+ got := f.live()
+ if len(got) != 2 || !reflect.DeepEqual(got[failed], before[failed]) {
+ t.Errorf("transient namespace failure evicted or changed accepted live history: got=%+v", got)
+ }
+ if got[accepted].BackupTime.Unix() != before[accepted].BackupTime.Add(15*time.Minute).Unix() {
+ t.Error("accepted sibling did not advance")
+ }
+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
+ t.Error("unread namespace cache time changed")
+ }
+ for _, prior := range points {
+ if prior.Details["namespace"] == ns {
+ found := false
+ for _, now := range f.points(t, "pbs-main") {
+ if now.ID == prior.ID {
+ found = reflect.DeepEqual(prior, now)
+ }
+ }
+ if !found {
+ t.Error("carried namespace renewed or lost recovery evidence")
+ }
+ }
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown {
+ t.Errorf("incomplete history claimed current protection: %+v", p)
+ }
+ }
+ }
+ f.rootStatus.Store(0)
+ f.tenantStatus.Store(0)
+ f.collect(t, true, nil)
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Errorf("ordinary read failed to recover: %+v", p)
+ }
+ }
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ f.collect(t, true, nil)
+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
+ t.Fatal("complete empty read failed to clear confirmed absence")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestPBSNamespaceDiscoveryContinuityAndAccessWithdrawal(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, denied := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/denied=%t", kind, denied), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ f.collect(t, true, nil)
+ prior := f.live()["101"]
+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
+ childReads := f.tenantReads.Load()
+ f.rootAt.Add(15 * 60)
+ discoveryErr := fmt.Errorf("fixture namespace discovery unavailable")
+ if denied {
+ discoveryErr = fmt.Errorf("api error 403: fixture denied")
+ }
+ for range 2 {
+ f.collect(t, false, discoveryErr)
+ got := f.live()
+ if denied {
+ if len(got) != 1 {
+ t.Errorf("discovery denial restored unread raw inventory: %+v", got)
+ }
+ } else if len(got) != 2 || !reflect.DeepEqual(got["101"], prior) {
+ t.Errorf("transient discovery erased or changed unread namespace: %+v", got)
+ }
+ if !denied && !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
+ t.Error("failed discovery renewed snapshot reuse")
+ }
+ if f.tenantReads.Load() != childReads {
+ t.Error("discovery failure guessed a hidden namespace read")
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected {
+ t.Errorf("failed discovery claimed protection: %+v", p)
+ }
+ }
+ }
+ f.collect(t, true, nil)
+ if len(f.live()) != 2 {
+ t.Fatal("ordinary discovery did not recover inventory")
+ }
+ // A complete discovery that removes the child is authoritative absence.
+ f.collect(t, false, nil)
+ if len(f.live()) != 1 || len(f.points(t, "pbs-main")) != 1 {
+ t.Fatal("complete namespace removal retained absent history")
+ }
+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).IsZero() {
+ t.Error("removed namespace kept snapshot cache time")
+ }
+ })
+ }
+ }
+}
+
+func TestPBSNamespaceMixedFailuresDoNotResurrectDeniedScope(t *testing.T) {
+ for _, rootDenied := range []bool{false, true} {
+ for _, discoveryFailed := range []bool{false, true} {
+ t.Run(fmt.Sprintf("root-denied=%t/discovery-failed=%t", rootDenied, discoveryFailed), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, "vm")
+ f.collect(t, true, nil)
+ before := f.live()
+ other := before["100"]
+ other.ID = "other-instance-backup"
+ other.Instance = "pbs-other"
+ f.m.state.UpdatePBSBackups("pbs-other", []models.PBSBackup{other})
+ denied, retained := "101", "100"
+ f.rootStatus.Store(http.StatusServiceUnavailable)
+ f.tenantStatus.Store(http.StatusForbidden)
+ if rootDenied {
+ denied, retained = "100", "101"
+ f.rootStatus.Store(http.StatusUnauthorized)
+ f.tenantStatus.Store(http.StatusServiceUnavailable)
+ }
+ var discoveryErr error
+ if discoveryFailed {
+ discoveryErr = fmt.Errorf("fixture transient discovery failure")
+ }
+ for range 2 {
+ f.collect(t, true, discoveryErr)
+ got := f.live()
+ if len(got) != 1 || !reflect.DeepEqual(got[retained], before[retained]) {
+ t.Errorf("mixed failure lost transient scope or retained denied scope %s: %+v", denied, got)
+ }
+ if len(f.points(t, "pbs-main")) != 2 {
+ t.Error("mixed access failure deleted historical recovery facts")
+ }
+ for _, row := range f.m.state.GetSnapshot().PBSBackups {
+ if row.Instance == "pbs-other" && !reflect.DeepEqual(row, other) {
+ t.Error("failure crossed instance scope")
+ }
+ }
+ if len(f.m.state.GetSnapshot().PBSBackups) != 2 {
+ t.Error("other instance inventory disappeared or denied row was resurrected")
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected {
+ t.Errorf("wholly unread collection claimed protection: %+v", p)
+ }
+ }
+ }
+ f.rootStatus.Store(0)
+ f.tenantStatus.Store(0)
+ f.collect(t, true, nil)
+ if len(f.live()) != 2 {
+ t.Fatal("ordinary mixed-failure recovery did not restore both namespaces")
+ }
+ })
+ }
+ }
+}
+
+func TestPBSNamespaceSuccessfulEmptySiblingStaysEmpty(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, "vm")
+ f.collect(t, true, nil)
+ before := f.live()["101"]
+ f.rootEmpty.Store(true)
+ f.tenantStatus.Store(http.StatusServiceUnavailable)
+ for range 2 {
+ // A failed discovery cannot override the root's actual successful empty read.
+ f.collect(t, true, fmt.Errorf("fixture discovery unavailable"))
+ got := f.live()
+ if len(got) != 1 || !reflect.DeepEqual(got["101"], before) {
+ t.Errorf("carry-forward resurrected a confirmed-absent sibling or dropped unread history: %+v", got)
+ }
+ }
+ f.tenantStatus.Store(0)
+ f.tenantEmpty.Store(true)
+ f.collect(t, true, nil)
+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
+ t.Fatal("complete empty recovery retained absent files")
+ }
+}
+
+// The registered Go suite checks each phase. Optional frame export is evidence
+// for the production Coverage/API-hook browser proof, not a separate test page.
+func TestPBSNamespaceContinuityLifecycle(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, "vm")
+ frames := map[string]any{}
+ original := map[string]models.PBSBackup{}
+ for _, phase := range []string{"observed", "partial", "recovered", "denied", "restored", "confirmed-empty"} {
+ switch phase {
+ case "partial":
+ f.tenantStatus.Store(http.StatusServiceUnavailable)
+ f.rootAt.Add(15 * 60)
+ case "recovered", "restored":
+ f.tenantStatus.Store(0)
+ case "denied":
+ f.tenantStatus.Store(http.StatusForbidden)
+ case "confirmed-empty":
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ }
+ f.collect(t, true, nil)
+ rows := f.live()
+ want := 2
+ if phase == "denied" {
+ want = 1
+ }
+ if phase == "confirmed-empty" {
+ want = 0
+ }
+ if len(rows) != want {
+ t.Fatalf("phase %s live rows=%d want=%d", phase, len(rows), want)
+ }
+ if phase == "observed" {
+ original = rows
+ }
+ if phase == "partial" && !reflect.DeepEqual(rows["101"], original["101"]) {
+ t.Fatal("unread row changed")
+ }
+ postures := f.postures(t)
+ for _, p := range postures {
+ healthy := phase == "observed" || phase == "recovered" || phase == "restored"
+ if healthy != (p.State == recovery.ProtectionStateProtected) {
+ t.Fatalf("phase %s posture=%+v", phase, p)
+ }
+ }
+ frames[phase] = map[string]any{"pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": f.points(t, "pbs-main")}
+ }
+ if os.Getenv("PULSE_PBS_NAMESPACE_FRAMES") != "1" {
+ return
+ }
+ source, err := os.ReadFile("monitor_backups.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(source)), "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log("PBS_NAMESPACE_FRAMES=" + string(data))
+}