Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

fix(monitor): the SWPC plasma feed had been serving stale rows for weeks

CarterPerez-dev5af8d1c5abc1
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go
index 4a293f94..fb18aa5d 100644
--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go
+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go
@@ -5,7 +5,6 @@ package swpc
import (
"context"
- "encoding/json"
"fmt"
"strconv"
"time"
@@ -17,8 +16,8 @@ import (
const (
defaultSWPCBaseURL = "https://services.swpc.noaa.gov"
- pathPlasma = "/products/solar-wind/plasma-5-minute.json"
- pathMag = "/products/solar-wind/mag-5-minute.json"
+ pathPlasma = "/json/rtsw/rtsw_wind_1m.json"
+ pathMag = "/json/rtsw/rtsw_mag_1m.json"
pathKp = "/products/noaa-planetary-k-index.json"
pathXray = "/json/goes/primary/xrays-1-day.json"
pathAlerts = "/products/alerts.json"
@@ -26,6 +25,7 @@ const (
defaultSWPCBurst = 5
defaultSWPCBudget = 5
defaultSWPCBreaker = 60 * time.Second
+ rtswWindowRows = 180
)
type ClientConfig struct {
@@ -90,45 +90,126 @@ type AlertItem struct {
Message string
}
+type rtswWindRow struct {
+ TimeTag string `json:"time_tag"`
+ Active bool `json:"active"`
+ ProtonSpeed *float64 `json:"proton_speed"`
+ ProtonDensity *float64 `json:"proton_density"`
+ ProtonTemperature *float64 `json:"proton_temperature"`
+}
+
+type rtswMagRow struct {
+ TimeTag string `json:"time_tag"`
+ Active bool `json:"active"`
+ Bt *float64 `json:"bt"`
+ BxGSM *float64 `json:"bx_gsm"`
+ ByGSM *float64 `json:"by_gsm"`
+ BzGSM *float64 `json:"bz_gsm"`
+ ThetaGSM *float64 `json:"theta_gsm"`
+ PhiGSM *float64 `json:"phi_gsm"`
+}
+
func (c *Client) FetchPlasma(ctx context.Context) ([]PlasmaTick, error) {
- rows, err := c.fetchRowArray(ctx, pathPlasma)
+ var raw []rtswWindRow
+ if err := c.hx.GetJSON(ctx, pathPlasma, nil, &raw); err != nil {
+ return nil, fmt.Errorf("fetch rtsw wind %s: %w", pathPlasma, err)
+ }
+ window, err := activeWindow(
+ raw,
+ pathPlasma,
+ func(r rtswWindRow) bool { return r.Active },
+ )
if err != nil {
return nil, err
}
- out := make([]PlasmaTick, 0, len(rows))
- for _, r := range rows {
- ts, _ := ParseTime(r["time_tag"])
+ out := make([]PlasmaTick, 0, len(window))
+ for _, r := range window {
+ ts, err := ParseTime(r.TimeTag)
+ if err != nil || r.ProtonSpeed == nil {
+ continue
+ }
out = append(out, PlasmaTick{
TimeTag: ts,
- Density: r["density"],
- Speed: r["speed"],
- Temperature: r["temperature"],
+ Density: formatSWPCFloat(r.ProtonDensity),
+ Speed: formatSWPCFloat(r.ProtonSpeed),
+ Temperature: formatSWPCFloat(r.ProtonTemperature),
})
}
+ reverseInPlace(out)
return out, nil
}
func (c *Client) FetchMag(ctx context.Context) ([]MagTick, error) {
- rows, err := c.fetchRowArray(ctx, pathMag)
+ var raw []rtswMagRow
+ if err := c.hx.GetJSON(ctx, pathMag, nil, &raw); err != nil {
+ return nil, fmt.Errorf("fetch rtsw mag %s: %w", pathMag, err)
+ }
+ window, err := activeWindow(
+ raw,
+ pathMag,
+ func(r rtswMagRow) bool { return r.Active },
+ )
if err != nil {
return nil, err
}
- out := make([]MagTick, 0, len(rows))
- for _, r := range rows {
- ts, _ := ParseTime(r["time_tag"])
+ out := make([]MagTick, 0, len(window))
+ for _, r := range window {
+ ts, err := ParseTime(r.TimeTag)
+ if err != nil || r.BzGSM == nil {
+ continue
+ }
out = append(out, MagTick{
TimeTag: ts,
- BxGSM: r["bx_gsm"],
- ByGSM: r["by_gsm"],
- BzGSM: r["bz_gsm"],
- LonGSM: r["lon_gsm"],
- LatGSM: r["lat_gsm"],
- Bt: r["bt"],
+ BxGSM: formatSWPCFloat(r.BxGSM),
+ ByGSM: formatSWPCFloat(r.ByGSM),
+ BzGSM: formatSWPCFloat(r.BzGSM),
+ LonGSM: formatSWPCFloat(r.PhiGSM),
+ LatGSM: formatSWPCFloat(r.ThetaGSM),
+ Bt: formatSWPCFloat(r.Bt),
})
}
+ reverseInPlace(out)
return out, nil
}
+func activeWindow[T any](
+ rows []T,
+ path string,
+ isActive func(T) bool,
+) ([]T, error) {
+ out := make([]T, 0, rtswWindowRows)
+ for _, r := range rows {
+ if !isActive(r) {
+ continue
+ }
+ out = append(out, r)
+ if len(out) == rtswWindowRows {
+ break
+ }
+ }
+ if len(rows) > 0 && len(out) == 0 {
+ return nil, fmt.Errorf(
+ "rtsw %s: %d rows but none from the active spacecraft",
+ path,
+ len(rows),
+ )
+ }
+ return out, nil
+}
+
+func reverseInPlace[T any](rows []T) {
+ for i, j := 0, len(rows)-1; i < j; i, j = i+1, j-1 {
+ rows[i], rows[j] = rows[j], rows[i]
+ }
+}
+
+func formatSWPCFloat(v *float64) string {
+ if v == nil {
+ return ""
+ }
+ return strconv.FormatFloat(*v, 'f', -1, 64)
+}
+
type rawKp struct {
TimeTag string `json:"time_tag"`
Kp float64 `json:"Kp"`
@@ -204,52 +285,6 @@ func (c *Client) FetchAlerts(ctx context.Context) ([]AlertItem, error) {
return out, nil
}
-func (c *Client) fetchRowArray(
- ctx context.Context,
- path string,
-) ([]map[string]string, error) {
- var raw [][]any
- if err := c.hx.GetJSON(ctx, path, nil, &raw); err != nil {
- return nil, fmt.Errorf("fetch row-array %s: %w", path, err)
- }
- if len(raw) < 2 {
- return nil, nil
- }
- headers := make([]string, 0, len(raw[0]))
- for _, h := range raw[0] {
- if s, ok := h.(string); ok {
- headers = append(headers, s)
- }
- }
- out := make([]map[string]string, 0, len(raw)-1)
- for _, row := range raw[1:] {
- m := make(map[string]string, len(headers))
- for i, v := range row {
- if i >= len(headers) {
- break
- }
- m[headers[i]] = anyToString(v)
- }
- out = append(out, m)
- }
- return out, nil
-}
-
-func anyToString(v any) string {
- switch x := v.(type) {
- case string:
- return x
- case float64:
- return strconv.FormatFloat(x, 'f', -1, 64)
- case json.Number:
- return x.String()
- case bool:
- return strconv.FormatBool(x)
- default:
- return ""
- }
-}
-
var swpcTimeFormats = []string{
time.RFC3339Nano,
"2006-01-02T15:04:05Z",
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go
index 3bb76f70..797363b4 100644
--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go
+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go
@@ -35,29 +35,108 @@ func newFakeServer(t *testing.T, route, fixture string) *httptest.Server {
return srv
}
-func TestClient_FetchPlasmaDecodesRowArray(t *testing.T) {
- srv := newFakeServer(t, "plasma-5-minute", "plasma.json")
+func TestClient_FetchPlasmaDecodesObjectArray(t *testing.T) {
+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma.json")
c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
rows, err := c.FetchPlasma(ctx)
require.NoError(t, err)
- require.GreaterOrEqual(t, len(rows), 1)
+ require.Len(t, rows, 2)
require.False(t, rows[0].TimeTag.IsZero())
require.NotEmpty(t, rows[0].Density)
require.NotEmpty(t, rows[0].Speed)
+ require.Equal(t, "391.3", rows[len(rows)-1].Speed)
+ require.Equal(t, "2.09", rows[len(rows)-1].Density)
+ require.NotEqual(
+ t,
+ "391.71",
+ rows[len(rows)-1].Speed,
+ "must not read the inactive ACE spacecraft",
+ )
}
-func TestClient_FetchMagDecodesRowArray(t *testing.T) {
- srv := newFakeServer(t, "mag-5-minute", "mag.json")
+func TestClient_FetchPlasmaIgnoresInactiveSpacecraft(t *testing.T) {
+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma.json")
+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ rows, err := c.FetchPlasma(ctx)
+ require.NoError(t, err)
+ for _, r := range rows {
+ require.NotEqual(t, "391.71", r.Speed)
+ require.NotEqual(t, "396.68", r.Speed)
+ }
+}
+
+func TestClient_FetchPlasmaErrorsWhenNoActiveSpacecraft(t *testing.T) {
+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma_inactive.json")
+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ _, err := c.FetchPlasma(ctx)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "active spacecraft")
+}
+
+func TestClient_FetchPlasmaReturnsAscendingSoLastRowIsNewest(t *testing.T) {
+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma.json")
+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ rows, err := c.FetchPlasma(ctx)
+ require.NoError(t, err)
+ require.Greater(t, len(rows), 1)
+ for i := 1; i < len(rows); i++ {
+ require.True(
+ t,
+ rows[i].TimeTag.After(rows[i-1].TimeTag),
+ "rows must be ascending: index %d is not after %d", i, i-1,
+ )
+ }
+}
+
+func TestClient_FetchMagDecodesObjectArray(t *testing.T) {
+ srv := newFakeServer(t, "rtsw_mag_1m", "mag.json")
c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
rows, err := c.FetchMag(ctx)
require.NoError(t, err)
- require.GreaterOrEqual(t, len(rows), 1)
+ require.Len(t, rows, 2)
+ require.Equal(t, "1.32", rows[len(rows)-1].BzGSM)
+ require.Equal(t, "5.56", rows[len(rows)-1].Bt)
+ require.Equal(t, "316.47", rows[len(rows)-1].LonGSM)
+ require.Equal(t, "13.64", rows[len(rows)-1].LatGSM)
+ require.NotEqual(
+ t,
+ "-1.07",
+ rows[len(rows)-1].BzGSM,
+ "must not read the inactive ACE spacecraft",
+ )
+}
+
+func TestClient_FetchMagReturnsAscendingSoLastRowIsNewest(t *testing.T) {
+ srv := newFakeServer(t, "rtsw_mag_1m", "mag.json")
+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ rows, err := c.FetchMag(ctx)
+ require.NoError(t, err)
+ require.Greater(t, len(rows), 1)
+ for i := 1; i < len(rows); i++ {
+ require.True(
+ t,
+ rows[i].TimeTag.After(rows[i-1].TimeTag),
+ "rows must be ascending: index %d is not after %d", i, i-1,
+ )
+ }
}
func TestClient_FetchKpDecodesObjectArray(t *testing.T) {
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json
index b594b512..468b15ce 100644
--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json
+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json
@@ -1,38 +1,57 @@
[
- [
- "time_tag",
- "bx_gsm",
- "by_gsm",
- "bz_gsm",
- "lon_gsm",
- "lat_gsm",
- "bt"
- ],
- [
- "2026-05-02 08:20:00.000",
- "0.36",
- "0.12",
- "4.41",
- "18.07",
- "85.10",
- "4.43"
- ],
- [
- "2026-05-02 08:21:00.000",
- "-0.32",
- "-0.19",
- "4.50",
- "210.52",
- "85.21",
- "4.52"
- ],
- [
- "2026-05-02 08:22:00.000",
- "-0.63",
- "0.12",
- "4.53",
- "168.89",
- "81.97",
- "4.57"
- ]
+ {
+ "time_tag": "2026-09-02T10:40:00",
+ "active": false,
+ "source": "ACE",
+ "bt": 5.19,
+ "bx_gsm": -3.72,
+ "by_gsm": 3.46,
+ "bz_gsm": -1.07,
+ "theta_gsm": -11.94,
+ "phi_gsm": 137.13
+ },
+ {
+ "time_tag": "2026-09-02T10:40:00",
+ "active": false,
+ "source": "IMAP",
+ "bt": 5.44,
+ "bx_gsm": 3.81,
+ "by_gsm": -3.66,
+ "bz_gsm": 1.36,
+ "theta_gsm": 14.42,
+ "phi_gsm": 316.11
+ },
+ {
+ "time_tag": "2026-09-02T10:40:00",
+ "active": true,
+ "source": "SOLAR1",
+ "bt": 5.56,
+ "bx_gsm": 3.91,
+ "by_gsm": -3.72,
+ "bz_gsm": 1.32,
+ "theta_gsm": 13.64,
+ "phi_gsm": 316.47
+ },
+ {
+ "time_tag": "2026-09-02T10:39:00",
+ "active": true,
+ "source": "SOLAR1",
+ "bt": null,
+ "bx_gsm": null,
+ "by_gsm": null,
+ "bz_gsm": null,
+ "theta_gsm": null,
+ "phi_gsm": null
+ },
+ {
+ "time_tag": "2026-09-02T10:38:00",
+ "active": true,
+ "source": "SOLAR1",
+ "bt": 4.98,
+ "bx_gsm": 3.49,
+ "by_gsm": -3.41,
+ "bz_gsm": 1.53,
+ "theta_gsm": 17.15,
+ "phi_gsm": 315.71
+ }
]
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json
index 4dac31f0..ca72b499 100644
--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json
+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json
@@ -1,26 +1,42 @@
[
- [
- "time_tag",
- "density",
- "speed",
- "temperature"
- ],
- [
- "2026-05-02 08:20:00.000",
- "2.94",
- "450.6",
- "93030"
- ],
- [
- "2026-05-02 08:21:00.000",
- "2.90",
- "448.0",
- "96988"
- ],
- [
- "2026-05-02 08:22:00.000",
- "2.65",
- "452.1",
- "84580"
- ]
+ {
+ "time_tag": "2026-09-02T10:41:00",
+ "active": false,
+ "source": "ACE",
+ "proton_speed": 391.71,
+ "proton_temperature": 33258,
+ "proton_density": 2.18
+ },
+ {
+ "time_tag": "2026-09-02T10:40:06",
+ "active": false,
+ "source": "IMAP",
+ "proton_speed": 396.68,
+ "proton_temperature": 34110,
+ "proton_density": 2.31
+ },
+ {
+ "time_tag": "2026-09-02T10:40:00",
+ "active": true,
+ "source": "SOLAR1",
+ "proton_speed": 391.3,
+ "proton_temperature": 31004,
+ "proton_density": 2.09
+ },
+ {
+ "time_tag": "2026-09-02T10:39:00",
+ "active": true,
+ "source": "SOLAR1",
+ "proton_speed": null,
+ "proton_temperature": null,
+ "proton_density": null
+ },
+ {
+ "time_tag": "2026-09-02T10:38:00",
+ "active": true,
+ "source": "SOLAR1",
+ "proton_speed": 388.4,
+ "proton_temperature": 30115,
+ "proton_density": 2.02
+ }
]
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma_inactive.json b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma_inactive.json
new file mode 100644
index 00000000..dc943738
--- /dev/null
+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma_inactive.json
@@ -0,0 +1,18 @@
+[
+ {
+ "time_tag": "2026-09-02T10:41:00",
+ "active": false,
+ "source": "ACE",
+ "proton_speed": 391.71,
+ "proton_temperature": 33258,
+ "proton_density": 2.18
+ },
+ {
+ "time_tag": "2026-09-02T10:40:06",
+ "active": false,
+ "source": "IMAP",
+ "proton_speed": 396.68,
+ "proton_temperature": 34110,
+ "proton_density": 2.31
+ }
+]