Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(canary): wire event + notify into runtime + create-tier rate limits
CarterPerez-devf9b012699a3b
diff --git a/PROJECTS/beginner/canary-token-generator/backend/cmd/canary/main.go b/PROJECTS/beginner/canary-token-generator/backend/cmd/canary/main.goindex f91a06c2..38592634 100644--- a/PROJECTS/beginner/canary-token-generator/backend/cmd/canary/main.go+++ b/PROJECTS/beginner/canary-token-generator/backend/cmd/canary/main.go@@ -5,10 +5,12 @@ package mainimport ("context"+ "encoding/json""errors""flag""fmt""log/slog"+ "net/http""os""os/signal""sync"@@ -22,6 +24,9 @@ import ("github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/event""github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/health""github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/middleware"+ "github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/notify"+ "github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/notify/telegram"+ "github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/notify/webhook""github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/server""github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/token""github.com/CarterPerez-dev/cybersecurity-projects/canary-token-generator/backend/internal/token/generators/mysql"@@ -86,6 +91,14 @@ func run(configPath string) error {}logger.Info("redis connected")+ notifySvc, eventSvc := buildEventStack(+ cfg,+ logger,+ eventRepo,+ tokenRepo,+ rdb,+ )+genRegistry := registry.Build(registry.Config{BaseURL: cfg.Canary.BaseURL,MySQLPublicHost: cfg.MySQL.PublicHost,@@ -108,19 +121,19 @@ func run(configPath string) error {healthH := health.NewHandler(db, rdb)tokenH := token.NewHandler(tokenSvc,- &directEventRecorder{+ &eventRecorderAdapter{svc: eventSvc},+ &fingerprintRecorderAdapter{repo: eventRepo,- tokens: tokenRepo,- logger: logger,+ window: cfg.Notify.FingerprintWindow,},- nil,logger,)srv := mountRouter(cfg, logger, rdb, healthH, tokenH, verifier)var wg sync.WaitGroup- spawnMySQLListener(ctx, cfg, logger, &wg, tokenSvc, eventRepo, tokenRepo)+ spawnMySQLListener(ctx, cfg, logger, &wg, tokenSvc, eventSvc)+ spawnRetentionLoop(ctx, cfg, &wg, eventSvc)errChan := make(chan error, 1)go func() { errChan <- srv.Start() }()@@ -133,18 +146,60 @@ func run(configPath string) error {}shutdownErr := gracefulShutdown(cfg, logger, srv, telemetry, rdb, db)+ logger.Info("waiting for in-flight notifications")+ notifySvc.Wait()wg.Wait()return shutdownErr}+func buildEventStack(+ cfg *config.Config,+ logger *slog.Logger,+ eventRepo *event.Repository,+ tokenRepo *token.Repository,+ rdb *core.Redis,+) (*notify.Service, *event.Service) {+ tgSender := telegram.NewSender(telegram.Config{+ APIBase: cfg.Notify.TelegramAPIBase,+ ManageURL: cfg.Canary.ManageURL,+ MaxTries: cfg.Notify.MaxTries,+ MaxElapsed: cfg.Notify.MaxElapsed,+ InitialInterval: cfg.Notify.InitialInterval,+ })+ whSender := webhook.NewSender(webhook.Config{+ ManageURL: cfg.Canary.ManageURL,+ HMACSecret: cfg.Notify.WebhookHMACSecret,+ MaxTries: cfg.Notify.MaxTries,+ MaxElapsed: cfg.Notify.MaxElapsed,+ InitialInterval: cfg.Notify.InitialInterval,+ })++ notifySvc := notify.NewService(eventRepo,+ notify.WithLogger(logger),+ notify.WithSendTimeout(cfg.Notify.SendTimeout),+ )+ notifySvc.Register(tgSender, whSender)++ eventSvc := event.NewService(+ eventRepo,+ tokenRepo,+ rdb.Client,+ notifySvc,+ event.ServiceConfig{+ DedupTTL: cfg.Notify.DedupTTL,+ Logger: logger,+ },+ )+ return notifySvc, eventSvc+}+func spawnMySQLListener(ctx context.Context,cfg *config.Config,logger *slog.Logger,wg *sync.WaitGroup,tokenSvc *token.Service,- eventRepo *event.Repository,- tokenRepo *token.Repository,+ eventSvc *event.Service,) {if !cfg.MySQL.Enabled {return@@ -154,11 +209,7 @@ func spawnMySQLListener(defer wg.Done()handler := mysql.NewHandler(&mysqlTokenLookup{svc: tokenSvc},- &mysqlEventRecorder{- repo: eventRepo,- tokens: tokenRepo,- logger: logger,- },+ &eventRecorderAdapter{svc: eventSvc},)if mErr := mysql.Run(ctx, cfg.MySQL.Addr, handler); mErr != nil {logger.Error("mysql server error", "error", mErr)@@ -166,6 +217,26 @@ func spawnMySQLListener(}()}+func spawnRetentionLoop(+ ctx context.Context,+ cfg *config.Config,+ wg *sync.WaitGroup,+ eventSvc *event.Service,+) {+ if cfg.Notify.RetentionInterval <= 0 || cfg.Notify.RetentionLimit <= 0 {+ return+ }+ wg.Add(1)+ go func() {+ defer wg.Done()+ eventSvc.RunRetentionLoop(+ ctx,+ cfg.Notify.RetentionInterval,+ cfg.Notify.RetentionLimit,+ )+ }()+}+func mountRouter(cfg *config.Config,logger *slog.Logger,@@ -189,6 +260,29 @@ func mountRouter(healthH.RegisterRoutes(r)tokenH.RegisterTriggerRoutes(r)+ createMin := middleware.NewRateLimiter(+ rdb.Client,+ middleware.RateLimitConfig{+ Limit: middleware.PerMinute(+ cfg.RateLimit.CreateMinRate,+ cfg.RateLimit.CreateMinBurst,+ ),+ KeyFunc: keyByCreateMin,+ FailOpen: true,+ },+ ).Handler+ createHour := middleware.NewRateLimiter(+ rdb.Client,+ middleware.RateLimitConfig{+ Limit: middleware.PerHour(+ cfg.RateLimit.CreateHourRate,+ cfg.RateLimit.CreateHourBurst,+ ),+ KeyFunc: keyByCreateHour,+ FailOpen: true,+ },+ ).Handler+r.Route("/api", func(api chi.Router) {api.Use(middleware.CORS(cfg.CORS))api.Use(@@ -202,13 +296,24 @@ func mountRouter(}).Handler,)api.Get("/tokens/types", tokenH.GetTypes)- api.With(middleware.TurnstileVerify(verifier)).- Post("/tokens", tokenH.CreateToken)+ api.With(+ createMin,+ createHour,+ middleware.TurnstileVerify(verifier),+ ).Post("/tokens", tokenH.CreateToken)})return srv}+func keyByCreateMin(r *http.Request) string {+ return "ratelimit:create:min:" + middleware.ExtractFingerprint(r)+}++func keyByCreateHour(r *http.Request) string {+ return "ratelimit:create:hour:" + middleware.ExtractFingerprint(r)+}+func gracefulShutdown(cfg *config.Config,logger *slog.Logger,@@ -291,25 +396,35 @@ func (a registryAdapter) Get(t token.Type) (token.Generator, bool) {return g, ok}-type directEventRecorder struct {- repo *event.Repository- tokens *token.Repository- logger *slog.Logger+type eventRecorderAdapter struct {+ svc *event.Service}-func (d *directEventRecorder) Record(+func (a *eventRecorderAdapter) Record(ctx context.Context,t *token.Token,evt *event.Event,) error {- if err := d.repo.Insert(ctx, evt); err != nil {- return fmt.Errorf("insert event: %w", err)- }- if err := d.tokens.IncrementTriggerCount(ctx, t.ID); err != nil {- d.logger.WarnContext(ctx, "increment trigger count",- "error", err, "token_id", t.ID)- }- return nil+ return a.svc.Record(ctx, t.NotifyInfo(), evt)+}++type fingerprintRecorderAdapter struct {+ repo *event.Repository+ window time.Duration+}++func (f *fingerprintRecorderAdapter) AttachFingerprint(+ ctx context.Context,+ tokenID, sourceIP string,+ fingerprint json.RawMessage,+) error {+ return f.repo.AttachFingerprint(+ ctx,+ tokenID,+ sourceIP,+ fingerprint,+ f.window,+ )}type mysqlTokenLookup struct{ svc *token.Service }@@ -320,24 +435,3 @@ func (m *mysqlTokenLookup) GetByID() (*token.Token, error) {return m.svc.GetByID(ctx, id)}--type mysqlEventRecorder struct {- repo *event.Repository- tokens *token.Repository- logger *slog.Logger-}--func (m *mysqlEventRecorder) Record(- ctx context.Context,- t *token.Token,- evt *event.Event,-) error {- if err := m.repo.Insert(ctx, evt); err != nil {- return fmt.Errorf("insert event: %w", err)- }- if err := m.tokens.IncrementTriggerCount(ctx, t.ID); err != nil {- m.logger.WarnContext(ctx, "mysql: increment trigger count",- "error", err, "token_id", t.ID)- }- return nil-}diff --git a/PROJECTS/beginner/canary-token-generator/backend/internal/config/config.go b/PROJECTS/beginner/canary-token-generator/backend/internal/config/config.goindex 1beb55b2..2e561c35 100644--- a/PROJECTS/beginner/canary-token-generator/backend/internal/config/config.go+++ b/PROJECTS/beginner/canary-token-generator/backend/internal/config/config.go@@ -26,6 +26,20 @@ type Config struct {Canary CanaryConfig `koanf:"canary"`Turnstile TurnstileConfig `koanf:"turnstile"`MySQL MySQLConfig `koanf:"mysql"`+ Notify NotifyConfig `koanf:"notify"`+}++type NotifyConfig struct {+ DedupTTL time.Duration `koanf:"dedup_ttl"`+ SendTimeout time.Duration `koanf:"send_timeout"`+ MaxTries uint `koanf:"max_tries"`+ MaxElapsed time.Duration `koanf:"max_elapsed"`+ InitialInterval time.Duration `koanf:"initial_interval"`+ RetentionInterval time.Duration `koanf:"retention_interval"`+ RetentionLimit int `koanf:"retention_limit"`+ WebhookHMACSecret string `koanf:"webhook_hmac_secret"`+ TelegramAPIBase string `koanf:"telegram_api_base"`+ FingerprintWindow time.Duration `koanf:"fingerprint_window"`}type CanaryConfig struct {@@ -75,9 +89,13 @@ type RedisConfig struct {}type RateLimitConfig struct {- Requests int `koanf:"requests"`- Window time.Duration `koanf:"window"`- Burst int `koanf:"burst"`+ Requests int `koanf:"requests"`+ Window time.Duration `koanf:"window"`+ Burst int `koanf:"burst"`+ CreateMinRate int `koanf:"create_min_rate"`+ CreateMinBurst int `koanf:"create_min_burst"`+ CreateHourRate int `koanf:"create_hour_rate"`+ CreateHourBurst int `koanf:"create_hour_burst"`}type CORSConfig struct {@@ -182,9 +200,13 @@ func loadDefaults(k *koanf.Koanf) error {"redis.pool_size": 10,"redis.min_idle_conns": 5,- "rate_limit.requests": 100,- "rate_limit.window": "1m",- "rate_limit.burst": 20,+ "rate_limit.requests": 100,+ "rate_limit.window": "1m",+ "rate_limit.burst": 20,+ "rate_limit.create_min_rate": 5,+ "rate_limit.create_min_burst": 5,+ "rate_limit.create_hour_rate": 20,+ "rate_limit.create_hour_burst": 5,"cors.allowed_origins": []string{"http://localhost:3000"},"cors.allowed_methods": []string{@@ -222,6 +244,17 @@ func loadDefaults(k *koanf.Koanf) error {"mysql.addr": "0.0.0.0:3306","mysql.public_host": "localhost","mysql.public_port": 3306,++ "notify.dedup_ttl": "15m",+ "notify.send_timeout": "30s",+ "notify.max_tries": 3,+ "notify.max_elapsed": "30s",+ "notify.initial_interval": "500ms",+ "notify.retention_interval": "1h",+ "notify.retention_limit": 100,+ "notify.webhook_hmac_secret": "",+ "notify.telegram_api_base": "https://api.telegram.org",+ "notify.fingerprint_window": "5m",}for key, value := range defaults {@@ -234,30 +267,44 @@ func loadDefaults(k *koanf.Koanf) error {}var envKeyMap = map[string]string{- "DATABASE_URL": "database.url",- "REDIS_URL": "redis.url",- "APP_ENVIRONMENT": "app.environment",- "HOST": "server.host",- "PORT": "server.port",- "LOG_LEVEL": "log.level",- "LOG_FORMAT": "log.format",- "RATE_LIMIT_REQUESTS": "rate_limit.requests",- "RATE_LIMIT_WINDOW": "rate_limit.window",- "RATE_LIMIT_BURST": "rate_limit.burst",- "OTEL_ENDPOINT": "otel.endpoint",- "OTEL_EXPORTER_OTLP_ENDPOINT": "otel.endpoint",- "OTEL_SERVICE_NAME": "otel.service_name",- "OTEL_ENABLED": "otel.enabled",- "OTEL_INSECURE": "otel.insecure",- "OTEL_SAMPLE_RATE": "otel.sample_rate",- "CANARY_BASE_URL": "canary.base_url",- "CANARY_MANAGE_URL": "canary.manage_url",- "TURNSTILE_SECRET_KEY": "turnstile.secret_key",- "TURNSTILE_SITE_KEY": "turnstile.site_key",- "MYSQL_ENABLED": "mysql.enabled",- "MYSQL_ADDR": "mysql.addr",- "MYSQL_PUBLIC_HOST": "mysql.public_host",- "MYSQL_PUBLIC_PORT": "mysql.public_port",+ "DATABASE_URL": "database.url",+ "REDIS_URL": "redis.url",+ "APP_ENVIRONMENT": "app.environment",+ "HOST": "server.host",+ "PORT": "server.port",+ "LOG_LEVEL": "log.level",+ "LOG_FORMAT": "log.format",+ "RATE_LIMIT_REQUESTS": "rate_limit.requests",+ "RATE_LIMIT_WINDOW": "rate_limit.window",+ "RATE_LIMIT_BURST": "rate_limit.burst",+ "OTEL_ENDPOINT": "otel.endpoint",+ "OTEL_EXPORTER_OTLP_ENDPOINT": "otel.endpoint",+ "OTEL_SERVICE_NAME": "otel.service_name",+ "OTEL_ENABLED": "otel.enabled",+ "OTEL_INSECURE": "otel.insecure",+ "OTEL_SAMPLE_RATE": "otel.sample_rate",+ "CANARY_BASE_URL": "canary.base_url",+ "CANARY_MANAGE_URL": "canary.manage_url",+ "TURNSTILE_SECRET_KEY": "turnstile.secret_key",+ "TURNSTILE_SITE_KEY": "turnstile.site_key",+ "MYSQL_ENABLED": "mysql.enabled",+ "MYSQL_ADDR": "mysql.addr",+ "MYSQL_PUBLIC_HOST": "mysql.public_host",+ "MYSQL_PUBLIC_PORT": "mysql.public_port",+ "RATE_LIMIT_CREATE_MIN_RATE": "rate_limit.create_min_rate",+ "RATE_LIMIT_CREATE_MIN_BURST": "rate_limit.create_min_burst",+ "RATE_LIMIT_CREATE_HOUR_RATE": "rate_limit.create_hour_rate",+ "RATE_LIMIT_CREATE_HOUR_BURST": "rate_limit.create_hour_burst",+ "NOTIFY_DEDUP_TTL": "notify.dedup_ttl",+ "NOTIFY_SEND_TIMEOUT": "notify.send_timeout",+ "NOTIFY_MAX_TRIES": "notify.max_tries",+ "NOTIFY_MAX_ELAPSED": "notify.max_elapsed",+ "NOTIFY_INITIAL_INTERVAL": "notify.initial_interval",+ "NOTIFY_RETENTION_INTERVAL": "notify.retention_interval",+ "NOTIFY_RETENTION_LIMIT": "notify.retention_limit",+ "WEBHOOK_HMAC_SECRET": "notify.webhook_hmac_secret",+ "NOTIFY_TELEGRAM_API_BASE": "notify.telegram_api_base",+ "NOTIFY_FINGERPRINT_WINDOW": "notify.fingerprint_window",}func envKeyReplacer(s string) string {diff --git a/PROJECTS/beginner/canary-token-generator/backend/internal/token/dto.go b/PROJECTS/beginner/canary-token-generator/backend/internal/token/dto.goindex e53daa03..ed326d85 100644--- a/PROJECTS/beginner/canary-token-generator/backend/internal/token/dto.go+++ b/PROJECTS/beginner/canary-token-generator/backend/internal/token/dto.go@@ -17,7 +17,7 @@ type CreateRequest struct {TelegramChat string `json:"telegram_chat" validate:"required_if=AlertChannel telegram"`WebhookURL string `json:"webhook_url" validate:"required_if=AlertChannel webhook,omitempty,url"`Metadata json.RawMessage `json:"metadata"`- TurnstileResp string `json:"cf_turnstile_response" validate:"required"`+ TurnstileResp string `json:"cf_turnstile_response"`}type Response struct {