Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
fix(monitor): the SWPC plasma feed had been serving stale rows for weeks
CarterPerez-dev5af8d1c5abc1
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.goindex 4a293f94..fb18aa5d 100644--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client.go@@ -5,7 +5,6 @@ package swpcimport ("context"- "encoding/json""fmt""strconv""time"@@ -17,8 +16,8 @@ import (const (defaultSWPCBaseURL = "https://services.swpc.noaa.gov"- pathPlasma = "/products/solar-wind/plasma-5-minute.json"- pathMag = "/products/solar-wind/mag-5-minute.json"+ pathPlasma = "/json/rtsw/rtsw_wind_1m.json"+ pathMag = "/json/rtsw/rtsw_mag_1m.json"pathKp = "/products/noaa-planetary-k-index.json"pathXray = "/json/goes/primary/xrays-1-day.json"pathAlerts = "/products/alerts.json"@@ -26,6 +25,7 @@ const (defaultSWPCBurst = 5defaultSWPCBudget = 5defaultSWPCBreaker = 60 * time.Second+ rtswWindowRows = 180)type ClientConfig struct {@@ -90,45 +90,126 @@ type AlertItem struct {Message string}+type rtswWindRow struct {+ TimeTag string `json:"time_tag"`+ Active bool `json:"active"`+ ProtonSpeed *float64 `json:"proton_speed"`+ ProtonDensity *float64 `json:"proton_density"`+ ProtonTemperature *float64 `json:"proton_temperature"`+}++type rtswMagRow struct {+ TimeTag string `json:"time_tag"`+ Active bool `json:"active"`+ Bt *float64 `json:"bt"`+ BxGSM *float64 `json:"bx_gsm"`+ ByGSM *float64 `json:"by_gsm"`+ BzGSM *float64 `json:"bz_gsm"`+ ThetaGSM *float64 `json:"theta_gsm"`+ PhiGSM *float64 `json:"phi_gsm"`+}+func (c *Client) FetchPlasma(ctx context.Context) ([]PlasmaTick, error) {- rows, err := c.fetchRowArray(ctx, pathPlasma)+ var raw []rtswWindRow+ if err := c.hx.GetJSON(ctx, pathPlasma, nil, &raw); err != nil {+ return nil, fmt.Errorf("fetch rtsw wind %s: %w", pathPlasma, err)+ }+ window, err := activeWindow(+ raw,+ pathPlasma,+ func(r rtswWindRow) bool { return r.Active },+ )if err != nil {return nil, err}- out := make([]PlasmaTick, 0, len(rows))- for _, r := range rows {- ts, _ := ParseTime(r["time_tag"])+ out := make([]PlasmaTick, 0, len(window))+ for _, r := range window {+ ts, err := ParseTime(r.TimeTag)+ if err != nil || r.ProtonSpeed == nil {+ continue+ }out = append(out, PlasmaTick{TimeTag: ts,- Density: r["density"],- Speed: r["speed"],- Temperature: r["temperature"],+ Density: formatSWPCFloat(r.ProtonDensity),+ Speed: formatSWPCFloat(r.ProtonSpeed),+ Temperature: formatSWPCFloat(r.ProtonTemperature),})}+ reverseInPlace(out)return out, nil}func (c *Client) FetchMag(ctx context.Context) ([]MagTick, error) {- rows, err := c.fetchRowArray(ctx, pathMag)+ var raw []rtswMagRow+ if err := c.hx.GetJSON(ctx, pathMag, nil, &raw); err != nil {+ return nil, fmt.Errorf("fetch rtsw mag %s: %w", pathMag, err)+ }+ window, err := activeWindow(+ raw,+ pathMag,+ func(r rtswMagRow) bool { return r.Active },+ )if err != nil {return nil, err}- out := make([]MagTick, 0, len(rows))- for _, r := range rows {- ts, _ := ParseTime(r["time_tag"])+ out := make([]MagTick, 0, len(window))+ for _, r := range window {+ ts, err := ParseTime(r.TimeTag)+ if err != nil || r.BzGSM == nil {+ continue+ }out = append(out, MagTick{TimeTag: ts,- BxGSM: r["bx_gsm"],- ByGSM: r["by_gsm"],- BzGSM: r["bz_gsm"],- LonGSM: r["lon_gsm"],- LatGSM: r["lat_gsm"],- Bt: r["bt"],+ BxGSM: formatSWPCFloat(r.BxGSM),+ ByGSM: formatSWPCFloat(r.ByGSM),+ BzGSM: formatSWPCFloat(r.BzGSM),+ LonGSM: formatSWPCFloat(r.PhiGSM),+ LatGSM: formatSWPCFloat(r.ThetaGSM),+ Bt: formatSWPCFloat(r.Bt),})}+ reverseInPlace(out)return out, nil}+func activeWindow[T any](+ rows []T,+ path string,+ isActive func(T) bool,+) ([]T, error) {+ out := make([]T, 0, rtswWindowRows)+ for _, r := range rows {+ if !isActive(r) {+ continue+ }+ out = append(out, r)+ if len(out) == rtswWindowRows {+ break+ }+ }+ if len(rows) > 0 && len(out) == 0 {+ return nil, fmt.Errorf(+ "rtsw %s: %d rows but none from the active spacecraft",+ path,+ len(rows),+ )+ }+ return out, nil+}++func reverseInPlace[T any](rows []T) {+ for i, j := 0, len(rows)-1; i < j; i, j = i+1, j-1 {+ rows[i], rows[j] = rows[j], rows[i]+ }+}++func formatSWPCFloat(v *float64) string {+ if v == nil {+ return ""+ }+ return strconv.FormatFloat(*v, 'f', -1, 64)+}+type rawKp struct {TimeTag string `json:"time_tag"`Kp float64 `json:"Kp"`@@ -204,52 +285,6 @@ func (c *Client) FetchAlerts(ctx context.Context) ([]AlertItem, error) {return out, nil}-func (c *Client) fetchRowArray(- ctx context.Context,- path string,-) ([]map[string]string, error) {- var raw [][]any- if err := c.hx.GetJSON(ctx, path, nil, &raw); err != nil {- return nil, fmt.Errorf("fetch row-array %s: %w", path, err)- }- if len(raw) < 2 {- return nil, nil- }- headers := make([]string, 0, len(raw[0]))- for _, h := range raw[0] {- if s, ok := h.(string); ok {- headers = append(headers, s)- }- }- out := make([]map[string]string, 0, len(raw)-1)- for _, row := range raw[1:] {- m := make(map[string]string, len(headers))- for i, v := range row {- if i >= len(headers) {- break- }- m[headers[i]] = anyToString(v)- }- out = append(out, m)- }- return out, nil-}--func anyToString(v any) string {- switch x := v.(type) {- case string:- return x- case float64:- return strconv.FormatFloat(x, 'f', -1, 64)- case json.Number:- return x.String()- case bool:- return strconv.FormatBool(x)- default:- return ""- }-}-var swpcTimeFormats = []string{time.RFC3339Nano,"2006-01-02T15:04:05Z",diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.goindex 3bb76f70..797363b4 100644--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/client_test.go@@ -35,29 +35,108 @@ func newFakeServer(t *testing.T, route, fixture string) *httptest.Server {return srv}-func TestClient_FetchPlasmaDecodesRowArray(t *testing.T) {- srv := newFakeServer(t, "plasma-5-minute", "plasma.json")+func TestClient_FetchPlasmaDecodesObjectArray(t *testing.T) {+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma.json")c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)defer cancel()rows, err := c.FetchPlasma(ctx)require.NoError(t, err)- require.GreaterOrEqual(t, len(rows), 1)+ require.Len(t, rows, 2)require.False(t, rows[0].TimeTag.IsZero())require.NotEmpty(t, rows[0].Density)require.NotEmpty(t, rows[0].Speed)+ require.Equal(t, "391.3", rows[len(rows)-1].Speed)+ require.Equal(t, "2.09", rows[len(rows)-1].Density)+ require.NotEqual(+ t,+ "391.71",+ rows[len(rows)-1].Speed,+ "must not read the inactive ACE spacecraft",+ )}-func TestClient_FetchMagDecodesRowArray(t *testing.T) {- srv := newFakeServer(t, "mag-5-minute", "mag.json")+func TestClient_FetchPlasmaIgnoresInactiveSpacecraft(t *testing.T) {+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma.json")+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()++ rows, err := c.FetchPlasma(ctx)+ require.NoError(t, err)+ for _, r := range rows {+ require.NotEqual(t, "391.71", r.Speed)+ require.NotEqual(t, "396.68", r.Speed)+ }+}++func TestClient_FetchPlasmaErrorsWhenNoActiveSpacecraft(t *testing.T) {+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma_inactive.json")+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()++ _, err := c.FetchPlasma(ctx)+ require.Error(t, err)+ require.Contains(t, err.Error(), "active spacecraft")+}++func TestClient_FetchPlasmaReturnsAscendingSoLastRowIsNewest(t *testing.T) {+ srv := newFakeServer(t, "rtsw_wind_1m", "plasma.json")+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()++ rows, err := c.FetchPlasma(ctx)+ require.NoError(t, err)+ require.Greater(t, len(rows), 1)+ for i := 1; i < len(rows); i++ {+ require.True(+ t,+ rows[i].TimeTag.After(rows[i-1].TimeTag),+ "rows must be ascending: index %d is not after %d", i, i-1,+ )+ }+}++func TestClient_FetchMagDecodesObjectArray(t *testing.T) {+ srv := newFakeServer(t, "rtsw_mag_1m", "mag.json")c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)defer cancel()rows, err := c.FetchMag(ctx)require.NoError(t, err)- require.GreaterOrEqual(t, len(rows), 1)+ require.Len(t, rows, 2)+ require.Equal(t, "1.32", rows[len(rows)-1].BzGSM)+ require.Equal(t, "5.56", rows[len(rows)-1].Bt)+ require.Equal(t, "316.47", rows[len(rows)-1].LonGSM)+ require.Equal(t, "13.64", rows[len(rows)-1].LatGSM)+ require.NotEqual(+ t,+ "-1.07",+ rows[len(rows)-1].BzGSM,+ "must not read the inactive ACE spacecraft",+ )+}++func TestClient_FetchMagReturnsAscendingSoLastRowIsNewest(t *testing.T) {+ srv := newFakeServer(t, "rtsw_mag_1m", "mag.json")+ c := swpc.NewClient(swpc.ClientConfig{BaseURL: srv.URL})+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()++ rows, err := c.FetchMag(ctx)+ require.NoError(t, err)+ require.Greater(t, len(rows), 1)+ for i := 1; i < len(rows); i++ {+ require.True(+ t,+ rows[i].TimeTag.After(rows[i-1].TimeTag),+ "rows must be ascending: index %d is not after %d", i, i-1,+ )+ }}func TestClient_FetchKpDecodesObjectArray(t *testing.T) {diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.jsonindex b594b512..468b15ce 100644--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/mag.json@@ -1,38 +1,57 @@[- [- "time_tag",- "bx_gsm",- "by_gsm",- "bz_gsm",- "lon_gsm",- "lat_gsm",- "bt"- ],- [- "2026-05-02 08:20:00.000",- "0.36",- "0.12",- "4.41",- "18.07",- "85.10",- "4.43"- ],- [- "2026-05-02 08:21:00.000",- "-0.32",- "-0.19",- "4.50",- "210.52",- "85.21",- "4.52"- ],- [- "2026-05-02 08:22:00.000",- "-0.63",- "0.12",- "4.53",- "168.89",- "81.97",- "4.57"- ]+ {+ "time_tag": "2026-09-02T10:40:00",+ "active": false,+ "source": "ACE",+ "bt": 5.19,+ "bx_gsm": -3.72,+ "by_gsm": 3.46,+ "bz_gsm": -1.07,+ "theta_gsm": -11.94,+ "phi_gsm": 137.13+ },+ {+ "time_tag": "2026-09-02T10:40:00",+ "active": false,+ "source": "IMAP",+ "bt": 5.44,+ "bx_gsm": 3.81,+ "by_gsm": -3.66,+ "bz_gsm": 1.36,+ "theta_gsm": 14.42,+ "phi_gsm": 316.11+ },+ {+ "time_tag": "2026-09-02T10:40:00",+ "active": true,+ "source": "SOLAR1",+ "bt": 5.56,+ "bx_gsm": 3.91,+ "by_gsm": -3.72,+ "bz_gsm": 1.32,+ "theta_gsm": 13.64,+ "phi_gsm": 316.47+ },+ {+ "time_tag": "2026-09-02T10:39:00",+ "active": true,+ "source": "SOLAR1",+ "bt": null,+ "bx_gsm": null,+ "by_gsm": null,+ "bz_gsm": null,+ "theta_gsm": null,+ "phi_gsm": null+ },+ {+ "time_tag": "2026-09-02T10:38:00",+ "active": true,+ "source": "SOLAR1",+ "bt": 4.98,+ "bx_gsm": 3.49,+ "by_gsm": -3.41,+ "bz_gsm": 1.53,+ "theta_gsm": 17.15,+ "phi_gsm": 315.71+ }]diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.jsonindex 4dac31f0..ca72b499 100644--- a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma.json@@ -1,26 +1,42 @@[- [- "time_tag",- "density",- "speed",- "temperature"- ],- [- "2026-05-02 08:20:00.000",- "2.94",- "450.6",- "93030"- ],- [- "2026-05-02 08:21:00.000",- "2.90",- "448.0",- "96988"- ],- [- "2026-05-02 08:22:00.000",- "2.65",- "452.1",- "84580"- ]+ {+ "time_tag": "2026-09-02T10:41:00",+ "active": false,+ "source": "ACE",+ "proton_speed": 391.71,+ "proton_temperature": 33258,+ "proton_density": 2.18+ },+ {+ "time_tag": "2026-09-02T10:40:06",+ "active": false,+ "source": "IMAP",+ "proton_speed": 396.68,+ "proton_temperature": 34110,+ "proton_density": 2.31+ },+ {+ "time_tag": "2026-09-02T10:40:00",+ "active": true,+ "source": "SOLAR1",+ "proton_speed": 391.3,+ "proton_temperature": 31004,+ "proton_density": 2.09+ },+ {+ "time_tag": "2026-09-02T10:39:00",+ "active": true,+ "source": "SOLAR1",+ "proton_speed": null,+ "proton_temperature": null,+ "proton_density": null+ },+ {+ "time_tag": "2026-09-02T10:38:00",+ "active": true,+ "source": "SOLAR1",+ "proton_speed": 388.4,+ "proton_temperature": 30115,+ "proton_density": 2.02+ }]diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma_inactive.json b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma_inactive.jsonnew file mode 100644index 00000000..dc943738--- /dev/null+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/swpc/testdata/plasma_inactive.json@@ -0,0 +1,18 @@+[+ {+ "time_tag": "2026-09-02T10:41:00",+ "active": false,+ "source": "ACE",+ "proton_speed": 391.71,+ "proton_temperature": 33258,+ "proton_density": 2.18+ },+ {+ "time_tag": "2026-09-02T10:40:06",+ "active": false,+ "source": "IMAP",+ "proton_speed": 396.68,+ "proton_temperature": 34110,+ "proton_density": 2.31+ }+]