Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
// Package handler implements HTTP handlers for the ingest service
package handler
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/beenuar/aisoc/services/ingest/internal/config"
"github.com/beenuar/aisoc/services/ingest/internal/graph"
"github.com/beenuar/aisoc/services/ingest/internal/ingestauth"
"github.com/beenuar/aisoc/services/ingest/internal/normalizer"
"github.com/rs/zerolog/log"
)
// GraphWriter is the subset of *graph.Writer the handler depends on.
// Defined as an interface so the HTTP path can be unit-tested without
// constructing a real Neo4j-backed writer (and so a nil writer is allowed
// when graph projection is disabled).
type GraphWriter interface {
WriteEvent(ctx context.Context, ev *graph.Event) error
}
type graphReadiness interface {
Ready(context.Context) error
}
// EventPublisher is the subset of *publisher.Publisher the HTTP path
// depends on. Declared as an interface for the same reason GraphWriter is:
// so a request can be driven end to end in a unit test — including which
// tenant its events were published for — without a Kafka broker.
type EventPublisher interface {
PublishBatch(ctx context.Context, events []*normalizer.NormalizedEvent) error
Ready(ctx context.Context) error
}
// SnapshotApplier mirrors *config_snapshot.Snapshotter.Apply. Defined as
// an interface so the handler can be unit-tested without standing up a
// real provider/cache pair, and so a nil applier is a clean "T1.2
// disabled" signal.
type SnapshotApplier interface {
Apply(ctx context.Context, ev *graph.Event)
}
// SubscriptionStatus is what a background subscription reports about itself
// to /readyz. Declared here, with no dependency on the package that owns the
// subscription, so wiring one up cannot create an import cycle.
type SubscriptionStatus struct {
// Attached: the loop is iterating its source right now.
Attached bool `json:"attached"`
// NotResolving: it has been failing for longer than a transient fault
// explains, or the failure is one no retry can clear. This is the field
// that separates "quiet" from "broken", which is the distinction a
// probe reporting only on the process cannot make.
NotResolving bool `json:"not_resolving"`
// Detail is the operator-facing reason, empty when healthy.
Detail string `json:"detail,omitempty"`
}
// Handler holds handler dependencies
type Handler struct {
norm *normalizer.Normalizer
pub EventPublisher
graph GraphWriter
snapshot SnapshotApplier
cfg *config.Config
auth *ingestauth.Authenticator
subscriptions map[string]func() SubscriptionStatus
}
// New creates a new Handler
func New(norm *normalizer.Normalizer, pub EventPublisher, cfg *config.Config) *Handler {
return &Handler{norm: norm, pub: pub, cfg: cfg, subscriptions: map[string]func() SubscriptionStatus{}}
}
// RegisterSubscription makes /readyz report on a background consumer.
//
// Call once per subscription the service owns, before serving. probe must be
// cheap and non-blocking — it runs on every readiness request — so it should
// read state the consumer already maintains rather than round-trip to a
// broker.
func (h *Handler) RegisterSubscription(name string, probe func() SubscriptionStatus) {
if h.subscriptions == nil {
h.subscriptions = map[string]func() SubscriptionStatus{}
}
h.subscriptions[name] = probe
}
// SetGraphWriter wires in the graph projection writer. nil disables graph
// fan-out — the rest of the pipeline behaves identically.
//
// The fan-out is intentional: the graph writer runs concurrently with the
// fusion publish, and a failure / queue-full in the graph writer must NOT
// block the fusion path (T1.1 acceptance criterion).
func (h *Handler) SetGraphWriter(g GraphWriter) {
h.graph = g
}
// SetSnapshotApplier wires in the T1.2 config-snapshot applier. nil leaves
// graph projections without :Configuration nodes — the writer still upserts
// every other node and edge, so disabling snapshots NEVER stalls ingest.
func (h *Handler) SetSnapshotApplier(s SnapshotApplier) {
h.snapshot = s
}
// SetAuthenticator wires in the /v1/ingest credential check.
//
// Unlike the other setters on this type, nil is not a "feature disabled"
// signal: a nil *ingestauth.Authenticator refuses every request. Ingest is
// a write path into tenant-owned data, so a wiring regression has to fail
// closed. There is no flag that turns this off — AISOC_DEV_MODE does not
// reach it, and services/api/tests/test_security_defaults.py exists to keep
// it that way.
func (h *Handler) SetAuthenticator(a *ingestauth.Authenticator) {
h.auth = a
}
// IngestRequest is the API payload for submitting events
type IngestRequest struct {
ConnectorID string `json:"connector_id"`
ConnectorType string `json:"connector_type"`
SourceFormat string `json:"source_format"`
Events []map[string]interface{} `json:"events"`
}
// IngestResponse reports processing results
type IngestResponse struct {
Accepted int `json:"accepted"`
Rejected int `json:"rejected"`
RequestID string `json:"request_id"`
Errors []string `json:"errors,omitempty"`
}
// IngestEvents handles POST /v1/ingest
func (h *Handler) IngestEvents(w http.ResponseWriter, r *http.Request) {
// Authenticate before touching the body. An unauthenticated caller
// should not get to spend this process's memory budget, and the
// credential is what decides the tenant, so nothing below can run
// without it.
identity, authErr := h.auth.Authenticate(r.Context(), r)
if authErr != nil {
writeAuthError(w, r, authErr)
return
}
body, ok := h.readIngestBody(w, r)
if !ok {
return
}
if err := ingestauth.VerifySignature(r, identity, body); err != nil {
writeAuthError(w, r, err)
return
}
// The tenant header is honoured only where it agrees with what the
// credential authorises. Naming another tenant narrows the scope to
// nothing and is refused here, rather than reaching into that tenant
// — which is precisely what this endpoint used to do for anyone who
// could reach the port.
tenantUUID, scopeErr := ingestauth.Resolve(identity.Principal, h.auth.DeclaredTenant(r))
if scopeErr != nil {
log.Warn().
Str("subject", sanitizeForLog(identity.Principal.Subject)).
Str("requested_tenant", sanitizeForLog(h.auth.DeclaredTenant(r))).
Bool("delegated", identity.Principal.Delegated).
Msg("ingest: refused a write outside the credential's tenant scope")
writeError(w, http.StatusForbidden, scopeErr.Error())
return
}
tenantID := tenantUUID.String()
var req IngestRequest
if err := json.Unmarshal(body, &req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
return
}
if req.ConnectorID == "" || req.ConnectorType == "" {
writeError(w, http.StatusBadRequest, "connector_id and connector_type are required")
return
}
if len(req.Events) == 0 {
writeJSON(w, http.StatusOK, IngestResponse{RequestID: newRequestID()})
return
}
if len(req.Events) > h.cfg.MaxBatchSize {
// strconv, not string(rune(…)): the conversion produces the *code point*
// with that value, so a limit of 1000 rendered as "Ϩ" and the caller was
// told their batch exceeded a maximum of one Greek letter.
writeError(w, http.StatusRequestEntityTooLarge,
"batch size exceeds maximum of "+strconv.Itoa(h.cfg.MaxBatchSize))
return
}
normalized := make([]*normalizer.NormalizedEvent, 0, len(req.Events))
errs := []string{}
rejected := 0
for i, payload := range req.Events {
raw := &normalizer.RawEvent{
ConnectorID: req.ConnectorID,
ConnectorType: req.ConnectorType,
TenantID: tenantID,
ReceivedAt: time.Now().UTC().Format(time.RFC3339Nano),
Payload: payload,
SourceFormat: req.SourceFormat,
}
event, err := h.norm.Normalize(raw)
if err != nil {
log.Warn().Err(err).Int("event_index", i).Msg("Normalization failed")
errs = append(errs, err.Error())
rejected++
continue
}
normalized = append(normalized, event)
}
if len(normalized) > 0 {
// Fan-out graph writer (T1.1, v8.0). Runs concurrently with fusion
// publish. A failure in WriteEvent never propagates here — the
// writer drops the event onto an internal queue with backpressure
// handled by drop-and-metric, so this loop is non-blocking.
if h.graph != nil {
for _, ev := range normalized {
gev := graph.ExtractFromOCSF(ev.ID, ev.TenantID, req.ConnectorType, ev.OcsfEvent)
if gev == nil {
continue
}
// T1.2 — attach :Configuration nodes + :CONFIGURED_AS
// edges. Apply is best-effort: any provider failure is
// logged + skipped; the rest of the projection still
// flushes through WriteEvent below.
if h.snapshot != nil {
h.snapshot.Apply(r.Context(), gev)
}
// WriteEvent is non-blocking; ctx is only used to honor
// shutdown. We deliberately don't wait on it.
_ = h.graph.WriteEvent(r.Context(), gev)
}
}
if err := h.pub.PublishBatch(r.Context(), normalized); err != nil {
log.Error().Err(err).Str("tenant_id", tenantID).Msg("Failed to publish batch")
writeError(w, http.StatusInternalServerError, "failed to publish events")
return
}
}
writeJSON(w, http.StatusOK, IngestResponse{
Accepted: len(normalized),
Rejected: rejected,
RequestID: newRequestID(),
Errors: errs,
})
}
// Health handles GET /health
// Health is liveness: is this process running at all? It deliberately checks
// nothing else, so a restart loop is distinguishable from a dependency
// outage.
func (h *Handler) Health(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"service": "ingest",
"timestamp": time.Now().UTC().Format(time.RFC3339),
})
}
// Livez is the Kubernetes spelling of Health. Same answer, conventional name.
func (h *Handler) Livez(w http.ResponseWriter, r *http.Request) {
h.Health(w, r)
}
// Readyz is readiness: can this service do its job right now?
//
// For ingest that means one thing — is Kafka reachable. Without it every
// accepted event is dropped, and the service used to report a cheerful 200
// throughout, so "ingest is healthy" and "nothing is reaching the pipeline"
// were simultaneously true with no way to tell from the outside.
//
// 503 with the reason, rather than 200 with a status field, so a load
// balancer and a human both get a usable answer.
//
// Subscriptions are reported, not voted on
// ----------------------------------------
// Every registered background subscription is listed in the body whether or
// not anything is wrong, so a 200 says which ones were checked rather than
// only that nothing failed — a detached consumer behind a 200 with no
// mention of it is precisely what made the UEBA defect invisible.
//
// What a detached subscription does NOT do is flip the status code. Unlike
// the Python services, where consuming a topic IS the service's job, ingest
// serves two different kinds of traffic on one port: the publish path
// (/v1/ingest, /v1/inbox) and the opt-in graph-update WebSocket. Failing
// readiness for the second would pull the pipeline's front door out of the
// load balancer over an auxiliary feature, dropping events to fix a
// broadcast. The surface that depends on the subscription reports on it
// directly instead: /v1/graph_ws/stream answers 503 with the reason, and
// aisoc_graph_ws_source_not_resolving is the alertable signal.
func (h *Handler) Readyz(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
subs := h.subscriptionReport()
body := map[string]interface{}{
"service": "ingest",
// Present on every response, including the failing ones. A field
// that only renders on the happy path is absent at exactly the
// moment somebody is reading the output to find out what is wrong.
"subscriptions": subs,
"timestamp": time.Now().UTC().Format(time.RFC3339),
"graph": false,
}
// Named at the top level as well, so the degraded case is legible
// without reading the per-subscription map.
if degraded := degradedSubscriptions(subs); len(degraded) > 0 {
body["degraded_subscriptions"] = degraded
}
if graphProbe, ok := h.graph.(graphReadiness); ok {
graphCtx, graphCancel := context.WithTimeout(ctx, 1500*time.Millisecond)
if err := graphProbe.Ready(graphCtx); err == nil {
body["graph"] = true
} else {
body["graph_error"] = err.Error()
}
graphCancel()
}
if h.pub == nil {
body["kafka"] = false
body["status"] = "not_ready"
body["reason"] = "no kafka publisher configured"
writeJSON(w, http.StatusServiceUnavailable, body)
return
}
kafkaCtx, kafkaCancel := context.WithTimeout(ctx, 1500*time.Millisecond)
defer kafkaCancel()
if err := h.pub.Ready(kafkaCtx); err != nil {
body["kafka"] = false
body["status"] = "not_ready"
body["reason"] = err.Error()
body["hint"] = "docker compose logs kafka | tail -40"
writeJSON(w, http.StatusServiceUnavailable, body)
return
}
body["status"] = "ready"
body["kafka"] = true
body["checks"] = map[string]string{"kafka": "reachable"}
if _, degraded := body["degraded_subscriptions"]; degraded || body["graph"] != true {
body["status"] = "ready_degraded"
}
writeJSON(w, http.StatusOK, body)
}
// subscriptionReport evaluates every registered probe. A probe that panics
// is reported as detached rather than taking the readiness endpoint down
// with it: asking a live object whether it is working must not be able to
// make the answer unavailable.
func (h *Handler) subscriptionReport() map[string]SubscriptionStatus {
out := make(map[string]SubscriptionStatus, len(h.subscriptions))
for name, probe := range h.subscriptions {
out[name] = safeProbe(name, probe)
}
return out
}
func safeProbe(name string, probe func() SubscriptionStatus) (status SubscriptionStatus) {
defer func() {
if rec := recover(); rec != nil {
log.Error().Str("subscription", name).Interface("panic", rec).
Msg("readiness probe panicked; reporting the subscription as detached")
status = SubscriptionStatus{Attached: false, NotResolving: true, Detail: "probe panicked"}
}
}()
return probe()
}
func degradedSubscriptions(subs map[string]SubscriptionStatus) []string {
var names []string
for name, status := range subs {
if !status.Attached || status.NotResolving {
names = append(names, name)
}
}
sort.Strings(names)
return names
}
// readIngestBody slurps the body under a size cap.
//
// The cap is new alongside authentication and belongs with it: MaxBatchSize
// is only checked after the payload has been decoded, so before this the
// only thing standing between the endpoint and a multi-gigabyte body was
// that nobody had sent one.
func (h *Handler) readIngestBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
limit := h.cfg.IngestMaxBodyBytes
if limit <= 0 {
limit = 10 * 1024 * 1024
}
r.Body = http.MaxBytesReader(w, r.Body, limit)
body, err := io.ReadAll(r.Body)
if err != nil {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
writeError(w, http.StatusRequestEntityTooLarge,
fmt.Sprintf("body exceeds %d bytes", limit))
return nil, false
}
writeError(w, http.StatusBadRequest, "failed to read request body: "+err.Error())
return nil, false
}
return body, true
}
// writeAuthError renders an *ingestauth.Error with its own status, and logs
// the refusal. Anything that is not one of those is reported as 401 rather
// than leaking an internal message to an unauthenticated caller.
func writeAuthError(w http.ResponseWriter, r *http.Request, err error) {
var authErr *ingestauth.Error
status, msg := http.StatusUnauthorized, "unauthorized"
if errors.As(err, &authErr) {
status, msg = authErr.Status, authErr.Msg
}
log.Warn().
Int("status", status).
Str("remote_addr", sanitizeForLog(r.RemoteAddr)).
Str("path", sanitizeForLog(r.URL.Path)).
Msg("ingest: refused an unauthenticated or unauthorized push")
writeError(w, status, msg)
}
// sanitizeForLog strips CR/LF and bounds length inline at the call site.
// Done inline rather than behind a helper call for the values that matter:
// the taint tracker does not follow a helper across the boundary, and these
// values are caller-controlled.
func sanitizeForLog(v string) string {
s := strings.ReplaceAll(strings.ReplaceAll(v, "\r", ""), "\n", " ")
if len(s) > 128 {
return s[:128]
}
return s
}
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(v); err != nil {
log.Error().Err(err).Msg("Failed to write JSON response")
}
}
func writeError(w http.ResponseWriter, status int, msg string) {
writeJSON(w, status, map[string]string{"error": msg})
}
func newRequestID() string {
return fmt.Sprintf("req-%d", time.Now().UnixNano())
}