Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(hub): keep the updater running when the system record lookup fails (#2588)

Mario Mohar596f96442e5b
diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.go
index 7ef9e3223..d03d2bccb 100644
--- a/internal/hub/systems/system.go
+++ b/internal/hub/systems/system.go
@@ -618,8 +618,13 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
// getRecord retrieves the system record from the database.
// If the record is not found, it removes the system from the manager.
+// Other errors, like a query timing out while the hub is stalled, leave the
+// system in place so the updater retries on the next tick (issue #2544).
func (sys *System) getRecord(app core.App) (*core.Record, error) {
record, err := app.FindRecordById("systems", sys.Id)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, err
+ }
if err != nil || record == nil {
_ = sys.manager.RemoveSystem(sys.Id)
if err == nil {
diff --git a/internal/hub/systems/system_get_record_test.go b/internal/hub/systems/system_get_record_test.go
new file mode 100644
index 000000000..24d7c2249
--- /dev/null
+++ b/internal/hub/systems/system_get_record_test.go
@@ -0,0 +1,57 @@
+//go:build testing
+
+package systems
+
+import (
+ "context"
+ "testing"
+
+ "github.com/pocketbase/dbx"
+ "github.com/pocketbase/pocketbase/core"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// failingRecordHub fails record lookups with err, like a query that times out
+// while the hub's VM is frozen.
+type failingRecordHub struct {
+ stubHub
+ err error
+}
+
+func (h failingRecordHub) FindRecordById(collectionModelOrIdentifier any, recordId string, optFilters ...func(q *dbx.SelectQuery) error) (*core.Record, error) {
+ return nil, h.err
+}
+
+func TestGetRecordKeepsSystemOnTransientError(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ sm := NewSystemManager(failingRecordHub{stubHub: stubHub{app}, err: context.DeadlineExceeded})
+ t.Cleanup(sm.cancel)
+ sys.manager = sm
+ sys.ctx, sys.cancel = sys.getContext(sm.ctx)
+ sm.systems.Set(sys.Id, sys)
+
+ record, err := sys.getRecord(sm.hub)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ assert.Nil(t, record)
+ assert.True(t, sm.HasSystem(sys.Id), "system should stay in the manager")
+ assert.NoError(t, sys.ctx.Err(), "updater context should not be cancelled")
+}
+
+func TestGetRecordRemovesDeletedSystem(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ sm := NewSystemManager(stubHub{app})
+ t.Cleanup(sm.cancel)
+ sys.manager = sm
+ sys.ctx, sys.cancel = sys.getContext(sm.ctx)
+ sm.systems.Set(sys.Id, sys)
+
+ record, err := app.FindRecordById("systems", sys.Id)
+ require.NoError(t, err)
+ require.NoError(t, app.Delete(record))
+
+ _, err = sys.getRecord(sm.hub)
+ require.Error(t, err)
+ assert.False(t, sm.HasSystem(sys.Id), "deleted system should be removed")
+ assert.Error(t, sys.ctx.Err(), "updater context should be cancelled")
+}