Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
// AiSOC Ingest Service
// Handles raw event ingestion, OCSF normalization, ATT&CK mapping, and Kafka publishing
// Part of the AiSOC platform (MIT License)
package main
import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/beenuar/aisoc/services/ingest/internal/config"
configsnap "github.com/beenuar/aisoc/services/ingest/internal/config_snapshot"
"github.com/beenuar/aisoc/services/ingest/internal/envmode"
"github.com/beenuar/aisoc/services/ingest/internal/graph"
"github.com/beenuar/aisoc/services/ingest/internal/graph_ws"
"github.com/beenuar/aisoc/services/ingest/internal/handler"
"github.com/beenuar/aisoc/services/ingest/internal/inbox"
"github.com/beenuar/aisoc/services/ingest/internal/ingestauth"
"github.com/beenuar/aisoc/services/ingest/internal/kafkainput"
"github.com/beenuar/aisoc/services/ingest/internal/normalizer"
"github.com/beenuar/aisoc/services/ingest/internal/otlp"
"github.com/beenuar/aisoc/services/ingest/internal/publisher"
"github.com/beenuar/aisoc/services/ingest/internal/server"
"github.com/beenuar/aisoc/services/ingest/internal/syslog"
"github.com/beenuar/aisoc/services/ingest/internal/telemetry"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
func main() {
// Configure structured logging. Use the human-friendly console writer
// in any dev-class environment (development, dev, local, demo, test) —
// previously this exact-matched ENV == "development" only, so
// ENVIRONMENT=development (the alias the Python API treats as
// equivalent) silently flipped this service to JSON logs and made
// local debugging confusing. envmode.IsDevRuntime keeps both layers
// in lock-step.
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
if envmode.IsDevRuntime() {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
}
log.Info().Str("service", "ingest").Msg("Starting AiSOC Ingest Service")
// Tracing. `ingest` and `realtime` were the two uninstrumented ends of
// the Kafka spine, so a trace started at the API stopped at the
// pipeline boundary — exactly where the interesting latency is. No-ops
// unless OTEL_EXPORTER_OTLP_ENDPOINT is set, because emitting spans
// into a connection error is worse than emitting none.
tracingCtx := context.Background()
shutdownTracing, err := telemetry.Setup(tracingCtx)
if err != nil {
// Degraded, not fatal: ingest is the pipeline's front door and
// must not refuse traffic because a collector is unreachable.
log.Warn().Err(err).Msg("tracing disabled — exporter could not start")
}
defer func() {
if err := shutdownTracing(tracingCtx); err != nil {
log.Warn().Err(err).Msg("tracing shutdown incomplete; recent spans may be lost")
}
}()
cfg, err := config.Load()
if err != nil {
log.Fatal().Err(err).Msg("Failed to load configuration")
}
// Initialize components
norm, err := normalizer.New(cfg)
if err != nil {
log.Fatal().Err(err).Msg("Failed to initialize normalizer")
}
pub, err := publisher.New(cfg)
if err != nil {
log.Fatal().Err(err).Msg("Failed to initialize Kafka publisher")
}
defer pub.Close()
h := handler.New(norm, pub, cfg)
// T1.1 (v8.0) — ingest-side graph writer. Runs in fan-out: failures in
// the graph writer NEVER block fusion ingest; the writer's queue is
// bounded and drops on full + emits a metric.
if cfg.GraphEnabled {
// Derived from the package's own retry window rather than written
// here, plus headroom for the final attempt. An independent
// number is how the retry came to be capped at a sixth of it.
gctx, gcancel := context.WithTimeout(context.Background(), graph.ConnectRetryWindow()+15*time.Second)
gw, err := graph.New(gctx, graph.Config{
URI: cfg.Neo4jURI,
Username: cfg.Neo4jUser,
Password: cfg.Neo4jPassword,
Database: cfg.Neo4jDatabase,
BatchSize: cfg.GraphBatchSize,
FlushInterval: time.Duration(cfg.GraphFlushIntervalMs) * time.Millisecond,
QueueSize: cfg.GraphQueueSize,
Publisher: pub,
})
gcancel()
if err != nil {
// Soft fail: graph writer is opt-in. Ingest still runs without it.
log.Warn().Err(err).Msg("graph: writer disabled (Neo4j unreachable)")
} else {
defer func() { _ = gw.Close() }()
h.SetGraphWriter(gw)
log.Info().
Str("uri", cfg.Neo4jURI).
Str("schema_version", graph.SchemaVersion).
Int("batch_size", cfg.GraphBatchSize).
Int("flush_ms", cfg.GraphFlushIntervalMs).
Str("updates_topic", cfg.GraphUpdatesTopic).
Msg("graph: ingest-side writer enabled")
// T1.2 (v8.0) — config snapshots. Wired only when both the
// graph writer is up *and* the operator opts in. Falls back
// to in-memory cache when Redis is unhealthy; falls back to
// HTTPProvider against the connectors service when configured,
// otherwise NoopProvider (every snapshot returns
// ErrNotImplemented and we log skips). Failures NEVER block
// fusion ingest — same contract as T1.1.
if cfg.SnapshotEnabled {
ttl := time.Duration(cfg.SnapshotCacheTTLSecs) * time.Second
cacheCtx, cacheCancel := context.WithTimeout(context.Background(), 2*time.Second)
cache := configsnap.NewRedisCache(cacheCtx, configsnap.RedisConfig{
Addr: cfg.RedisAddr,
TTL: ttl,
})
cacheCancel()
var provider configsnap.Provider
if cfg.SnapshotProviderURL != "" {
provider = configsnap.NewHTTPProvider(
cfg.SnapshotProviderURL,
time.Duration(cfg.SnapshotProviderTimeoutMs)*time.Millisecond,
)
} else {
provider = configsnap.NoopProvider{}
}
snapper, err := configsnap.New(configsnap.Config{
Provider: provider,
Cache: cache,
TTL: ttl,
})
if err != nil {
log.Warn().Err(err).Msg("snapshot: disabled (constructor failed)")
} else {
defer func() { _ = snapper.Close() }()
h.SetSnapshotApplier(snapper)
log.Info().
Str("provider_url", cfg.SnapshotProviderURL).
Dur("cache_ttl", ttl).
Msg("snapshot: T1.2 config snapshots enabled")
}
} else {
log.Info().Msg("snapshot: disabled (AISOC_SNAPSHOT_ENABLED!=true)")
}
}
} else {
log.Info().Msg("graph: writer disabled (AISOC_GRAPH_ENABLED!=true)")
}
// Workstream 6 — universal capture push paths.
//
// We need a Postgres pool to resolve inbox tokens and a YAML registry
// to find the matching template. Both are optional in dev (no
// DATABASE_DSN means /v1/inbox/* is disabled but /v1/ingest still
// works, so the connector path keeps running).
// One Postgres pool serves two consumers: the inbox token resolver and
// the /v1/ingest credential check. It is opened whenever DATABASE_DSN
// is set rather than only when the inbox is enabled, because turning
// the inbox off must not also remove the ability to authenticate a
// connector push.
var dbPool *pgxpool.Pool
if cfg.DatabaseDSN != "" {
poolCtx, poolCancel := context.WithTimeout(context.Background(), 10*time.Second)
pool, err := pgxpool.New(poolCtx, cfg.DatabaseDSN)
poolCancel()
if err != nil {
log.Error().Err(err).
Msg("Failed to connect to Postgres; minted ingest tokens cannot be resolved and /v1/inbox/* is disabled")
} else {
defer pool.Close()
dbPool = pool
}
}
// /v1/ingest credential check. Before this the endpoint read a tenant
// out of a header and authenticated nothing, so anyone who could reach
// the port could write alerts into any tenant.
var (
tokenStore *inbox.Store
tokenResolver ingestauth.TokenResolver
tenantDirectory ingestauth.TenantDirectory
)
if dbPool != nil {
tokenStore = inbox.NewStore(dbPool)
tokenResolver = tokenStore
tenantDirectory = ingestauth.NewTenantDirectory(dbPool)
}
auth := ingestauth.New(tokenResolver, tenantDirectory, cfg.ServiceToken, cfg.TenantHeaderKey)
h.SetAuthenticator(auth)
if auth.Configured() {
log.Info().
Bool("minted_tokens", tokenResolver != nil).
Bool("service_token", cfg.ServiceToken != "").
Msg("ingest: /v1/ingest requires an authenticated, tenant-scoped credential")
} else {
// Loud, and at error: this deployment will refuse every push. A
// warning here would be indistinguishable from the ordinary
// startup chatter an operator scrolls past.
log.Error().
Msg("ingest: no credential source configured — /v1/ingest will refuse every request. " +
"Set DATABASE_DSN so minted push tokens resolve, or AISOC_SERVICE_TOKEN for service-to-service pushes")
}
var inboxHandler *inbox.Handler
if cfg.InboxEnabled && tokenStore != nil {
registry := inbox.NewRegistry()
// Embedded first — these ship in the binary and are the set the
// service is tested against. The on-disk directory is an operator
// override layered on top, and is allowed to be absent.
if err := registry.LoadEmbedded(); err != nil {
log.Error().Err(err).
Msg("Failed to load embedded inbox templates; /v1/inbox/* will return 503 for every template")
}
if err := registry.Load(cfg.InboxTemplatesDir); err != nil {
log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).
Msg("Failed to load inbox template overrides from disk")
}
log.Info().
Strs("templates", registry.IDs()).
Int64("max_body_bytes", cfg.InboxMaxBodyBytes).
Float64("rate_requests_per_second", cfg.InboxRateRequestsPerSecond).
Float64("rate_events_per_second", cfg.InboxRateEventsPerSecond).
Msg("inbox: universal-capture push paths enabled")
inboxHandler = inbox.NewHandler(tokenStore, registry, pub, cfg.InboxMaxBodyBytes).
WithLimiter(inbox.NewLimiter(
cfg.InboxRateRequestsPerSecond,
cfg.InboxRateRequestBurst,
cfg.InboxRateEventsPerSecond,
cfg.InboxRateEventBurst,
))
} else if !cfg.InboxEnabled {
log.Info().Msg("inbox: disabled via INBOX_ENABLED=false")
} else {
log.Info().Msg("inbox: skipped (no DATABASE_DSN)")
}
// T1.4 (v8.0) — graph-update WebSocket fan-out. Opt-in via
// AISOC_GRAPH_WS_ENABLED=true. The broadcaster owns a single
// Kafka consumer against GraphUpdatesTopic and fans envelopes
// out to subscribed WebSocket clients with per-tenant filtering.
// Failures NEVER block the ingest path — the Kafka publish side
// (T1.1) is independent and the broadcaster is consumer-only.
var graphWSServer *graph_ws.Server
var graphWSBroker *graph_ws.Broadcaster
if cfg.GraphWSEnabled {
// One state object shared by the source and the broadcaster: the
// errors kafka-go reports to its own logger and the ones the consume
// loop sees are the same subscription failing, and splitting them
// across two records is how half of them stay invisible.
graphWSHealth := graph_ws.NewSourceState(cfg.GraphUpdatesTopic, 0)
src, err := graph_ws.NewKafkaSource(graph_ws.KafkaSourceConfig{
Brokers: cfg.KafkaBrokers,
Topic: cfg.GraphUpdatesTopic,
GroupID: cfg.GraphWSGroupID,
Health: graphWSHealth,
})
if err != nil {
log.Warn().Err(err).Msg("graph_ws: disabled (Kafka source init failed)")
} else {
graphWSBroker = graph_ws.New(src, graph_ws.Options{
BufferSize: cfg.GraphWSSubscriberBuffer,
Health: graphWSHealth,
})
graphWSServer = graph_ws.NewServer(graphWSBroker)
// /readyz names this subscription and its state. A consumer
// detached from its topic behind a 200 that never mentions it is
// indistinguishable from an idle one.
h.RegisterSubscription("graph_ws", func() handler.SubscriptionStatus {
health := graphWSBroker.Health()
return handler.SubscriptionStatus{
Attached: health.Attached,
NotResolving: health.NotResolving,
Detail: health.Reason(),
}
})
log.Info().
Str("topic", cfg.GraphUpdatesTopic).
Int("buffer", cfg.GraphWSSubscriberBuffer).
Msg("graph_ws: T1.4 WebSocket broadcaster enabled")
}
} else {
log.Info().Msg("graph_ws: disabled (AISOC_GRAPH_WS_ENABLED!=true)")
}
// Depth plan 4.2 — the syslog listener. Off unless the operator asks
// for it, because binding 514 needs a decision they have to make.
//
// Needs the same two things the inbox needs (a token store to resolve
// the tenant, a template registry to normalise) plus a publisher, and
// refuses to start rather than listening without them: a bound socket
// that drops everything is indistinguishable from a working one.
var syslogListener *syslog.Listener
if cfg.SyslogEnabled {
switch {
case tokenStore == nil:
log.Error().Msg("syslog: not started — DATABASE_DSN is unset, so the configured ingest token cannot be resolved to a tenant")
case cfg.SyslogToken == "":
log.Error().Msg("syslog: not started — AISOC_SYSLOG_TOKEN is unset, so there is no tenant to attribute messages to. " +
"Mint one with POST /api/v1/inbox/tokens using the 'syslog' template")
default:
registry := inbox.NewRegistry()
if err := registry.LoadEmbedded(); err != nil {
log.Error().Err(err).Msg("syslog: not started — embedded templates failed to load")
break
}
if err := registry.Load(cfg.InboxTemplatesDir); err != nil {
log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("syslog: template overrides on disk could not be read")
}
sink, err := syslog.NewPublishingSink(tokenStore, registry, pub, cfg.SyslogToken)
if err != nil {
log.Error().Err(err).Msg("syslog: not started")
break
}
listener, err := syslog.New(syslog.Config{
UDPAddr: cfg.SyslogUDPAddr,
TCPAddr: cfg.SyslogTCPAddr,
BatchSize: cfg.SyslogBatchSize,
FlushInterval: time.Duration(cfg.SyslogFlushMs) * time.Millisecond,
MaxConnections: cfg.SyslogMaxConnections,
}, sink)
if err != nil {
// Not fatal: the HTTP ingest path is this service's
// primary job and must not refuse traffic because 514 was
// taken. Loud, though — the operator asked for a listener
// and does not have one.
log.Error().Err(err).Msg("syslog: listener disabled")
break
}
syslogListener = listener
udpAddr, tcpAddr := listener.Addrs()
// /readyz names this listener and its state. A receiver nobody
// is sending to and a receiver that failed to bind look the
// same from outside unless readiness says which.
h.RegisterSubscription("syslog", func() handler.SubscriptionStatus {
state := listener.Health()
bound := state.UDPBound || state.TCPBound
return handler.SubscriptionStatus{
Attached: bound,
NotResolving: !bound,
Detail: syslogDetail(state),
}
})
log.Info().
Str("udp", udpAddr).
Str("tcp", tcpAddr).
Int("batch_size", cfg.SyslogBatchSize).
Msg("syslog: listener accepting RFC 5424, RFC 3164, CEF and LEEF")
}
} else {
log.Info().Msg("syslog: disabled (AISOC_SYSLOG_ENABLED!=true)")
}
// Depth plan 4.2 — the OTLP logs receiver. Off unless the operator asks
// for it. It shares the HTTP port rather than binding one of its own,
// because an OpenTelemetry exporter is configured with a single
// endpoint URL and appends /v1/logs itself.
var otlpReceiver *otlp.Receiver
if cfg.OTLPEnabled {
if tokenStore == nil {
log.Error().Msg("otlp: not started — DATABASE_DSN is unset, so an exporter's ingest token cannot be resolved to a tenant")
} else {
registry := inbox.NewRegistry()
if err := registry.LoadEmbedded(); err != nil {
log.Error().Err(err).Msg("otlp: not started — embedded templates failed to load")
} else {
if err := registry.Load(cfg.InboxTemplatesDir); err != nil {
log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("otlp: template overrides on disk could not be read")
}
receiver, err := otlp.New(tokenStore, registry, pub)
if err != nil {
log.Error().Err(err).Msg("otlp: receiver disabled")
} else {
otlpReceiver = receiver
// /readyz names the receiver. An endpoint nobody is
// exporting to and one refusing every request look the
// same from outside unless readiness says which.
h.RegisterSubscription("otlp_logs", func() handler.SubscriptionStatus {
return handler.SubscriptionStatus{
Attached: true,
NotResolving: false,
Detail: receiver.Detail(),
}
})
log.Info().Str("path", "/v1/logs").Str("template", otlp.TemplateID).
Msg("otlp: logs receiver accepting protobuf and JSON")
}
}
}
} else {
log.Info().Msg("otlp: disabled (AISOC_OTLP_LOGS_ENABLED!=true)")
}
// Depth plan 4.2 — customer Kafka topics. One source per tenant; the
// tenant comes from each source's minted token and from nothing in the
// record.
var kafkaSources []*kafkainput.Source
if sources, err := kafkainput.ParseSources(cfg.KafkaInputSources); err != nil {
// Loud and specific. A malformed source list that started the
// service with the input silently off is the exact shape of defect
// this repository keeps finding.
log.Error().Err(err).Msg("kafka_input: no sources started — AISOC_KAFKA_INPUT_SOURCES could not be read")
} else if len(sources) > 0 {
if tokenStore == nil {
log.Error().Int("sources", len(sources)).
Msg("kafka_input: no sources started — DATABASE_DSN is unset, so their ingest tokens cannot be resolved to tenants")
} else {
registry := inbox.NewRegistry()
if err := registry.LoadEmbedded(); err != nil {
log.Error().Err(err).Msg("kafka_input: no sources started — embedded templates failed to load")
} else {
if err := registry.Load(cfg.InboxTemplatesDir); err != nil {
log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("kafka_input: template overrides on disk could not be read")
}
for _, sc := range sources {
src, err := kafkainput.NewSource(sc, tokenStore, registry, pub)
if err != nil {
log.Error().Err(err).Str("source", sc.Name).Msg("kafka_input: source disabled")
continue
}
kafkaSources = append(kafkaSources, src)
name := sc.Name
h.RegisterSubscription("kafka_input:"+name, func() handler.SubscriptionStatus {
health := src.Health()
return handler.SubscriptionStatus{
Attached: health.Attached,
NotResolving: health.NotResolving || health.PermanentFailure,
Detail: health.Detail(),
}
})
log.Info().Str("source", sc.Name).Str("topic", sc.Topic).Str("group", sc.GroupID).
Msg("kafka_input: consuming a customer topic")
}
}
}
} else {
log.Info().Msg("kafka_input: no sources configured (AISOC_KAFKA_INPUT_SOURCES is empty)")
}
srv := server.New(cfg, h, inboxHandler, graphWSServer, otlpReceiver)
// Graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if graphWSBroker != nil {
graphWSBroker.Start(ctx)
defer graphWSBroker.Stop()
}
for _, src := range kafkaSources {
src.Start(ctx)
defer src.Stop()
}
if syslogListener != nil {
syslogListener.Start(ctx)
defer func() {
// Given its own short deadline rather than ctx: ctx is already
// cancelled by the time deferred shutdown runs, and a drain on
// a cancelled context publishes nothing.
drainCtx, drainCancel := context.WithTimeout(context.Background(), 5*time.Second)
syslogListener.Stop(drainCtx)
drainCancel()
}()
}
// Drain VulnMatches → Kafka in a background goroutine
if norm.VulnMatches != nil {
go func() {
for match := range norm.VulnMatches {
pubCtx, pubCancel := context.WithTimeout(ctx, 5*time.Second)
if err := pub.PublishVulnMatch(pubCtx, match); err != nil {
log.Warn().Err(err).Str("cve", match.CVE).Msg("Failed to publish VulnMatch")
}
pubCancel()
}
}()
}
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigCh
log.Info().Str("signal", sig.String()).Msg("Received shutdown signal")
cancel()
}()
log.Info().Int("port", cfg.HTTPPort).Msg("HTTP server listening")
if err := srv.Start(ctx); err != nil && err != http.ErrServerClosed {
log.Fatal().Err(err).Msg("Server error")
}
log.Info().Msg("Ingest service shut down gracefully")
}
// healthResponse is returned by the /health endpoint
type healthResponse struct {
Status string `json:"status"`
Version string `json:"version"`
Timestamp time.Time `json:"timestamp"`
}
// Version is set at build time via ldflags
var Version = "dev"
func init() {
// Set version in health check
_ = fmt.Sprintf("aisoc-ingest/%s", Version)
}
// syslogDetail turns listener state into the one sentence /readyz shows.
//
// "attached" alone would be a lie of omission for a receiver: a syslog
// listener is legitimately silent for hours, so the detail carries what it
// has bound, what it has received and what it last failed on — which is
// what separates "nothing is being sent" from "everything is being dropped".
func syslogDetail(state syslog.State) string {
if !state.UDPBound && !state.TCPBound {
return "no transport bound"
}
bound := make([]string, 0, 2)
if state.UDPBound {
bound = append(bound, "udp "+state.UDPAddr)
}
if state.TCPBound {
bound = append(bound, "tcp "+state.TCPAddr)
}
detail := fmt.Sprintf("listening on %s; received %d, rejected %d", strings.Join(bound, ", "), state.Received, state.Rejected)
if state.LastError != "" {
detail += "; last error: " + state.LastError
}
return detail
}