Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(monitor/collectors/usgs): 1m collector with id-diff and earthquake emit
CarterPerez-dev2368ed8f18b3
diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/usgs/collector.go b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/usgs/collector.gonew file mode 100644index 00000000..68cdb5a5--- /dev/null+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/usgs/collector.go@@ -0,0 +1,135 @@+// ©AngelaMos | 2026+// collector.go++package usgs++import (+ "context"+ "encoding/json"+ "log/slog"+ "time"++ "github.com/carterperez-dev/monitor-the-situation/backend/internal/events"+)++const (+ Name = "usgs"+ defaultUSGSInterval = time.Minute+ defaultLogPlaceLimit = 64+)++type Fetcher interface {+ Fetch(ctx context.Context) (Feed, error)+}++type Repository interface {+ Upsert(ctx context.Context, row Row) error+ KnownIDs(ctx context.Context, ids []string) (map[string]bool, error)+}++type Emitter interface {+ Emit(ev events.Event)+}++type StateRecorder interface {+ RecordSuccess(ctx context.Context, name string, eventCount int64) error+ RecordError(ctx context.Context, name, errMsg string) error+}++type CollectorConfig struct {+ Interval time.Duration+ Fetcher Fetcher+ Repo Repository+ Emitter Emitter+ State StateRecorder+ Logger *slog.Logger+}++type Collector struct {+ cfg CollectorConfig+ logger *slog.Logger+}++func NewCollector(cfg CollectorConfig) *Collector {+ if cfg.Interval <= 0 {+ cfg.Interval = defaultUSGSInterval+ }+ if cfg.Logger == nil {+ cfg.Logger = slog.Default()+ }+ return &Collector{cfg: cfg, logger: cfg.Logger}+}++func (c *Collector) Name() string { return Name }++func (c *Collector) Run(ctx context.Context) error {+ ticker := time.NewTicker(c.cfg.Interval)+ defer ticker.Stop()+ c.tick(ctx)+ for {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-ticker.C:+ c.tick(ctx)+ }+ }+}++func (c *Collector) tick(ctx context.Context) {+ feed, err := c.cfg.Fetcher.Fetch(ctx)+ if err != nil {+ c.logger.Warn("usgs fetch", "err", err)+ _ = c.cfg.State.RecordError(ctx, Name, err.Error())+ return+ }++ ids := make([]string, 0, len(feed.Features))+ for _, f := range feed.Features {+ ids = append(ids, f.ID)+ }+ known, err := c.cfg.Repo.KnownIDs(ctx, ids)+ if err != nil {+ c.logger.Warn("usgs known ids", "err", err)+ _ = c.cfg.State.RecordError(ctx, Name, err.Error())+ return+ }++ emitted := int64(0)+ now := time.Now().UTC()+ for _, f := range feed.Features {+ raw, _ := json.Marshal(f)+ row := Row{+ ID: f.ID,+ OccurredAt: f.Properties.OccurredAt(),+ Mag: f.Properties.Mag,+ Place: f.Properties.Place,+ GeomLat: coord(f.Geometry.Coordinates, 1),+ GeomLon: coord(f.Geometry.Coordinates, 0),+ DepthKm: coord(f.Geometry.Coordinates, 2),+ Payload: raw,+ }+ if err := c.cfg.Repo.Upsert(ctx, row); err != nil {+ c.logger.Warn("usgs upsert", "id", row.ID, "err", err)+ continue+ }+ if known[f.ID] {+ continue+ }+ c.cfg.Emitter.Emit(events.Event{+ Topic: events.TopicEarthquake,+ Timestamp: now,+ Source: Name,+ Payload: json.RawMessage(raw),+ })+ emitted+++ }+ _ = c.cfg.State.RecordSuccess(ctx, Name, emitted)+}++func coord(c []float64, i int) float64 {+ if i < len(c) {+ return c[i]+ }+ return 0+}diff --git a/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/usgs/collector_test.go b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/usgs/collector_test.gonew file mode 100644index 00000000..d1711de0--- /dev/null+++ b/PROJECTS/advanced/monitor-the-situation-dashboard/backend/internal/collectors/usgs/collector_test.go@@ -0,0 +1,194 @@+// ©AngelaMos | 2026+// collector_test.go++package usgs_test++import (+ "context"+ "errors"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/carterperez-dev/monitor-the-situation/backend/internal/collectors/usgs"+ "github.com/carterperez-dev/monitor-the-situation/backend/internal/events"+)++type fakeFetcher struct {+ feed usgs.Feed+ err error+ mu sync.Mutex+ n int+}++func (f *fakeFetcher) Fetch(_ context.Context) (usgs.Feed, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.n+++ if f.err != nil {+ return usgs.Feed{}, f.err+ }+ return f.feed, nil+}++type fakeRepo struct {+ mu sync.Mutex+ known map[string]bool+ upserts []usgs.Row+}++func (r *fakeRepo) Upsert(_ context.Context, row usgs.Row) error {+ r.mu.Lock()+ defer r.mu.Unlock()+ r.upserts = append(r.upserts, row)+ if r.known == nil {+ r.known = make(map[string]bool)+ }+ r.known[row.ID] = true+ return nil+}++func (r *fakeRepo) KnownIDs(_ context.Context, ids []string) (map[string]bool, error) {+ r.mu.Lock()+ defer r.mu.Unlock()+ out := make(map[string]bool, len(ids))+ for _, id := range ids {+ if r.known[id] {+ out[id] = true+ }+ }+ return out, nil+}++func (r *fakeRepo) Upserts() int {+ r.mu.Lock()+ defer r.mu.Unlock()+ return len(r.upserts)+}++type fakeEmitter struct {+ mu sync.Mutex+ events []events.Event+}++func (e *fakeEmitter) Emit(ev events.Event) {+ e.mu.Lock()+ defer e.mu.Unlock()+ e.events = append(e.events, ev)+}++func (e *fakeEmitter) Count() int {+ e.mu.Lock()+ defer e.mu.Unlock()+ return len(e.events)+}++type recordingState struct {+ mu sync.Mutex+ successes int+ failures int+ lastErr string+}++func (s *recordingState) RecordSuccess(_ context.Context, _ string, _ int64) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.successes+++ return nil+}++func (s *recordingState) RecordError(_ context.Context, _, msg string) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.failures+++ s.lastErr = msg+ return nil+}++func TestCollector_TickPersistsAndEmitsNewQuakes(t *testing.T) {+ feed := usgs.Feed{+ Type: "FeatureCollection",+ Features: []usgs.Feature{+ {ID: "q1", Properties: usgs.Properties{Mag: 4.5, Place: "test 1", Time: time.Now().UnixMilli()}, Geometry: usgs.Geometry{Coordinates: []float64{-120, 49, 5}}},+ {ID: "q2", Properties: usgs.Properties{Mag: 6.5, Place: "test 2", Time: time.Now().UnixMilli()}, Geometry: usgs.Geometry{Coordinates: []float64{140, -30, 10}}},+ {ID: "q3", Properties: usgs.Properties{Mag: 3.0, Place: "test 3", Time: time.Now().UnixMilli()}, Geometry: usgs.Geometry{Coordinates: []float64{0, 0, 1}}},+ },+ }+ ftch := &fakeFetcher{feed: feed}+ repo := &fakeRepo{}+ emt := &fakeEmitter{}+ st := &recordingState{}++ c := usgs.NewCollector(usgs.CollectorConfig{+ Interval: 20 * time.Millisecond,+ Fetcher: ftch,+ Repo: repo,+ Emitter: emt,+ State: st,+ })++ ctx, cancel := context.WithTimeout(context.Background(), 70*time.Millisecond)+ defer cancel()+ _ = c.Run(ctx)++ require.GreaterOrEqual(t, repo.Upserts(), 3, "should upsert all 3 features at least once")+ require.GreaterOrEqual(t, emt.Count(), 3, "should emit 3 new-event events")+ for _, ev := range emt.events {+ require.Equal(t, events.TopicEarthquake, ev.Topic)+ require.Equal(t, usgs.Name, ev.Source)+ }+ require.Greater(t, st.successes, 0)+ require.Equal(t, 0, st.failures)+}++func TestCollector_FetchErrorRecordsState(t *testing.T) {+ ftch := &fakeFetcher{err: errors.New("upstream 503")}+ repo := &fakeRepo{}+ emt := &fakeEmitter{}+ st := &recordingState{}++ c := usgs.NewCollector(usgs.CollectorConfig{+ Interval: 20 * time.Millisecond,+ Fetcher: ftch,+ Repo: repo,+ Emitter: emt,+ State: st,+ })++ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)+ defer cancel()+ _ = c.Run(ctx)++ require.Equal(t, 0, repo.Upserts())+ require.Equal(t, 0, emt.Count())+ require.Greater(t, st.failures, 0)+ require.Contains(t, st.lastErr, "upstream 503")+}++func TestCollector_KnownQuakesNotReEmitted(t *testing.T) {+ feed := usgs.Feed{+ Features: []usgs.Feature{+ {ID: "qx", Properties: usgs.Properties{Mag: 4.5, Time: time.Now().UnixMilli()}, Geometry: usgs.Geometry{Coordinates: []float64{0, 0, 1}}},+ },+ }+ ftch := &fakeFetcher{feed: feed}+ repo := &fakeRepo{known: map[string]bool{"qx": true}}+ emt := &fakeEmitter{}+ st := &recordingState{}++ c := usgs.NewCollector(usgs.CollectorConfig{+ Interval: 20 * time.Millisecond,+ Fetcher: ftch,+ Repo: repo,+ Emitter: emt,+ State: st,+ })++ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)+ defer cancel()+ _ = c.Run(ctx)++ require.Equal(t, 0, emt.Count(), "known quake should not re-emit")+}