Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(ingest): OTLP logs receiver on POST /v1/logs (depth 4.2)

Beenu Arorae8ea7a97d34e
diff --git a/services/ingest/go.mod b/services/ingest/go.mod
index ca047d6f2..1190dea72 100644
--- a/services/ingest/go.mod
+++ b/services/ingest/go.mod
@@ -17,6 +17,8 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
+ go.opentelemetry.io/proto/otlp v1.11.0
+ google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1
)
@@ -48,7 +50,6 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
- go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/sync v0.23.0 // indirect
@@ -57,5 +58,4 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/grpc v1.83.2 // indirect
- google.golang.org/protobuf v1.36.12 // indirect
)
diff --git a/services/ingest/internal/config/config.go b/services/ingest/internal/config/config.go
index 6143bfad0..95d13506f 100644
--- a/services/ingest/internal/config/config.go
+++ b/services/ingest/internal/config/config.go
@@ -88,6 +88,24 @@ type Config struct {
SyslogFlushMs int
SyslogMaxConnections int
+ // Depth plan 4.2 — the OTLP logs receiver and the customer Kafka input.
+ //
+ // Both off by default, and both take the tenant from a minted ingest
+ // token for the same reason syslog does: neither OTLP nor a Kafka
+ // record carries an authenticated principal, so a tenant read out of
+ // a resource attribute or a record header would be a cross-tenant
+ // write any instrumented process could perform.
+ //
+ // OTLPEnabled adds POST /v1/logs. There is no address of its own: the
+ // receiver shares the service's HTTP port, which is what an
+ // OpenTelemetry exporter expects to find behind one endpoint URL.
+ OTLPEnabled bool
+ // KafkaInputSources is a JSON array of source objects. An array rather
+ // than numbered variables because a deployment reading three tenants'
+ // topics needs three of everything, and numbered variables are how a
+ // fourth source silently gets the third one's token.
+ KafkaInputSources string
+
// Workstream 6 — universal capture push paths.
// InboxEnabled toggles the /v1/inbox/* routes. Off by default in
// development if no DATABASE_DSN is set, since the inbox store needs
@@ -246,9 +264,12 @@ func Load() (*Config, error) {
SyslogBatchSize: mustGetEnvInt("AISOC_SYSLOG_BATCH_SIZE", 500),
SyslogFlushMs: mustGetEnvInt("AISOC_SYSLOG_FLUSH_MS", 2000),
SyslogMaxConnections: mustGetEnvInt("AISOC_SYSLOG_MAX_CONNECTIONS", 256),
- InboxEnabled: getEnv("INBOX_ENABLED", "true") == "true",
- InboxTemplatesDir: getEnv("INBOX_TEMPLATES_DIR", "/app/templates"),
- InboxMaxBodyBytes: int64(mustGetEnvInt("INBOX_MAX_BODY_BYTES", 10*1024*1024)),
+
+ OTLPEnabled: getEnv("AISOC_OTLP_LOGS_ENABLED", "false") == "true",
+ KafkaInputSources: getEnv("AISOC_KAFKA_INPUT_SOURCES", ""),
+ InboxEnabled: getEnv("INBOX_ENABLED", "true") == "true",
+ InboxTemplatesDir: getEnv("INBOX_TEMPLATES_DIR", "/app/templates"),
+ InboxMaxBodyBytes: int64(mustGetEnvInt("INBOX_MAX_BODY_BYTES", 10*1024*1024)),
// Defaults sized to be invisible to a well-behaved vendor and
// firmly bounding to a runaway one: 50 req/s sustained with a
// 200 burst, and 5k events/s with a 20k burst (one large batch
diff --git a/services/ingest/internal/normalizer/templates/otlp-logs.yaml b/services/ingest/internal/normalizer/templates/otlp-logs.yaml
new file mode 100644
index 000000000..7e1cef7ce
--- /dev/null
+++ b/services/ingest/internal/normalizer/templates/otlp-logs.yaml
@@ -0,0 +1,72 @@
+# OpenTelemetry Protocol (OTLP) logs.
+#
+# Used by the OTLP logs receiver in services/ingest, which accepts
+# ExportLogsServiceRequest over HTTP in both the protobuf and the JSON
+# encodings the specification defines.
+#
+# Class 1007 Process Activity, in category 1 (System Activity), for the
+# same reason syslog.yaml is: an OTLP log record is an application saying
+# something about itself. The promoter promotes category 2 unconditionally,
+# so filing routine application logs as findings would turn every line into
+# an alert, which is precisely the noise this platform exists to remove. A
+# detection rule is what turns one of these into an alert.
+#
+# The receiver flattens a record's resource attributes, scope and log
+# attributes into a single namespace before this template is applied, so a
+# rule can clause on `service.name` or any attribute the emitter set.
+#
+# References:
+# https://opentelemetry.io/docs/specs/otlp/
+# https://opentelemetry.io/docs/specs/otel/logs/data-model/
+id: otlp-logs
+vendor_name: OpenTelemetry
+product_name: OTLP
+class_uid: 1007
+class_name: Process Activity
+activity_id: 1
+field_map:
+ body: message
+ severity_text: metadata.labels.severity_text
+ severity_number: metadata.labels.severity_number
+ trace_id: metadata.correlation_uid
+ span_id: metadata.labels.span_id
+ scope_name: metadata.logged_by
+ host.name: device.hostname
+ service.name: actor.process.name
+ process.pid: actor.process.pid
+ event.name: metadata.event_code
+ raw: raw_data
+# OTLP severity text is free-form but the specification names these short
+# names, and emitters overwhelmingly use them. The number is the
+# authoritative field; the receiver derives this text from it when the
+# emitter left it unset, so the ladder below always has something to read.
+severity_field: severity_text
+severity_map:
+ trace: 1
+ trace2: 1
+ trace3: 1
+ trace4: 1
+ debug: 1
+ debug2: 1
+ debug3: 1
+ debug4: 1
+ info: 1
+ info2: 1
+ info3: 1
+ info4: 1
+ warn: 3
+ warn2: 3
+ warn3: 3
+ warn4: 3
+ error: 4
+ error2: 4
+ error3: 4
+ error4: 4
+ fatal: 6
+ fatal2: 6
+ fatal3: 6
+ fatal4: 6
+time_field: timestamp
+message_field: body
+constants:
+ source_type: otlp
diff --git a/services/ingest/internal/otlp/receiver.go b/services/ingest/internal/otlp/receiver.go
new file mode 100644
index 000000000..d55135675
--- /dev/null
+++ b/services/ingest/internal/otlp/receiver.go
@@ -0,0 +1,586 @@
+// Package otlp implements an OpenTelemetry Protocol logs receiver.
+//
+// Depth item 4.2 / parity 6.10. An estate that already runs an OpenTelemetry
+// Collector has a logs pipeline and an exporter; what it does not have is
+// somewhere to point the exporter. This is that endpoint:
+//
+// POST /v1/logs Content-Type: application/x-protobuf
+// POST /v1/logs Content-Type: application/json
+//
+// Both encodings are in the specification and real collectors emit both, so
+// a receiver that reads one of them is a receiver half the estates cannot
+// use. gzip request bodies are accepted because the OTLP/HTTP specification
+// says a client may send them and the collector's default exporter does.
+//
+// Where the tenant comes from
+// ---------------------------
+// From a minted ingest token presented as `Authorization: Bearer <token>`,
+// and from nowhere else. OTLP has no notion of a tenant, and every
+// attribute on a record is set by the emitter, so a tenant read out of
+// `resource.attributes` would be a cross-tenant write any instrumented
+// process could perform. An OTLP exporter sets request headers through
+// OTEL_EXPORTER_OTLP_HEADERS, which is how the token gets here.
+//
+// What a client is told
+// ---------------------
+// The specification's partial-success contract, which is the part receivers
+// usually skip. A request whose records are partly unreadable returns 200
+// with `partial_success.rejected_log_records` set and a human message —
+// *not* a blanket 200, which would tell a collector everything landed, and
+// not a 400, which would make it retry records that did land. A full
+// failure returns 503 with Retry-After so the collector's queue retries
+// rather than dropping.
+package otlp
+
+import (
+ "compress/gzip"
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/beenuar/aisoc/services/ingest/internal/inbox"
+ "github.com/beenuar/aisoc/services/ingest/internal/normalizer"
+ "github.com/google/uuid"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+ "github.com/rs/zerolog/log"
+ collogs "go.opentelemetry.io/proto/otlp/collector/logs/v1"
+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"
+ logspb "go.opentelemetry.io/proto/otlp/logs/v1"
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+)
+
+// TemplateID is the OCSF template an OTLP token must be minted with.
+const TemplateID = "otlp-logs"
+
+// MaxBodyBytes caps one request. An uncapped receiver is a way to make the
+// ingest service run out of memory from the outside.
+const MaxBodyBytes int64 = 16 << 20
+
+var (
+ otlpRecords = promauto.NewCounterVec(prometheus.CounterOpts{
+ Name: "aisoc_ingest_otlp_log_records_total",
+ Help: "OTLP log records accepted.",
+ }, []string{"encoding"})
+ otlpRejected = promauto.NewCounterVec(prometheus.CounterOpts{
+ Name: "aisoc_ingest_otlp_rejected_total",
+ Help: "OTLP log records or requests refused, by reason.",
+ }, []string{"reason"})
+)
+
+// TokenResolver is the half of inbox.Store this package needs.
+type TokenResolver interface {
+ Resolve(ctx context.Context, token string) (*inbox.Token, error)
+}
+
+// TemplateSource resolves a template id to a template.
+type TemplateSource interface {
+ Get(id string) (*inbox.Template, error)
+}
+
+// Publisher is the half of publisher.Publisher this package needs.
+type Publisher interface {
+ PublishBatch(ctx context.Context, events []*normalizer.NormalizedEvent) error
+}
+
+// State is what readiness reports about the receiver. A receiver nobody is
+// sending to and a receiver refusing everything look identical from outside
+// unless the state is named, and silence is the normal condition for a logs
+// endpoint.
+type State struct {
+ Enabled bool
+ Received uint64
+ Rejected uint64
+ LastError string
+ LastAt time.Time
+}
+
+// Receiver serves the OTLP logs endpoint.
+type Receiver struct {
+ tokens TokenResolver
+ templates TemplateSource
+ pub Publisher
+
+ mu sync.Mutex
+ state State
+}
+
+// New wires the receiver.
+func New(tokens TokenResolver, templates TemplateSource, pub Publisher) (*Receiver, error) {
+ if tokens == nil || templates == nil || pub == nil {
+ return nil, errors.New("otlp: receiver needs a token resolver, a template source and a publisher")
+ }
+ return &Receiver{tokens: tokens, templates: templates, pub: pub, state: State{Enabled: true}}, nil
+}
+
+// Health is a snapshot for the readiness endpoint.
+func (r *Receiver) Health() State {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.state
+}
+
+// Detail is the sentence /readyz shows, healthy or not.
+func (r *Receiver) Detail() string {
+ s := r.Health()
+ detail := fmt.Sprintf("accepting OTLP logs on POST /v1/logs; records %d, rejected %d", s.Received, s.Rejected)
+ if s.Received == 0 && s.Rejected == 0 {
+ detail += " (nothing has arrived yet)"
+ }
+ if s.LastError != "" {
+ detail += "; last error: " + s.LastError
+ }
+ return detail
+}
+
+// encoding is how one request was framed, which also decides how the
+// response must be framed: the specification requires the response to use
+// the same encoding as the request.
+type encoding string
+
+const (
+ encodingProtobuf encoding = "protobuf"
+ encodingJSON encoding = "json"
+)
+
+// ServeLogs handles POST /v1/logs.
+func (r *Receiver) ServeLogs(w http.ResponseWriter, req *http.Request) {
+ enc, err := encodingOf(req.Header.Get("Content-Type"))
+ if err != nil {
+ r.refuse(w, encodingProtobuf, http.StatusUnsupportedMediaType, "unsupported_media_type", err.Error())
+ return
+ }
+
+ token := bearerToken(req)
+ if token == "" {
+ // Named rather than a bare 401: an OTLP exporter is configured by
+ // someone who has never seen this product's token concept, and
+ // "unauthorized" sends them to look at their collector.
+ r.refuse(w, enc, http.StatusUnauthorized, "no_token",
+ "no ingest token; set OTEL_EXPORTER_OTLP_HEADERS=\"Authorization=Bearer <token>\" with a token minted for the "+TemplateID+" template")
+ return
+ }
+
+ body, err := readBody(req)
+ if err != nil {
+ r.refuse(w, enc, http.StatusBadRequest, "unreadable_body", err.Error())
+ return
+ }
+
+ export := &collogs.ExportLogsServiceRequest{}
+ switch enc {
+ case encodingJSON:
+ // DiscardUnknown: the OTLP schema gains fields, and a collector a
+ // version ahead of this build must not have its whole batch
+ // refused over one field this build has never heard of.
+ if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(body, export); err != nil {
+ r.refuse(w, enc, http.StatusBadRequest, "undecodable", "the body is not an OTLP ExportLogsServiceRequest in JSON: "+err.Error())
+ return
+ }
+ default:
+ if err := proto.Unmarshal(body, export); err != nil {
+ r.refuse(w, enc, http.StatusBadRequest, "undecodable", "the body is not an OTLP ExportLogsServiceRequest in protobuf: "+err.Error())
+ return
+ }
+ }
+
+ tok, err := r.tokens.Resolve(req.Context(), token)
+ if err != nil {
+ r.refuse(w, enc, http.StatusUnauthorized, "bad_token", "the presented ingest token is not valid")
+ return
+ }
+ if tok.TemplateID != TemplateID {
+ r.refuse(w, enc, http.StatusForbidden, "wrong_template",
+ fmt.Sprintf("this token was minted with template %q; the OTLP receiver needs one minted with %q", tok.TemplateID, TemplateID))
+ return
+ }
+ tmpl, err := r.templates.Get(TemplateID)
+ if err != nil {
+ r.refuse(w, enc, http.StatusServiceUnavailable, "no_template", "this build has no "+TemplateID+" template, so OTLP records cannot be normalised")
+ return
+ }
+
+ events, rejected, reason := r.project(export, tok, tmpl)
+
+ if len(events) == 0 {
+ if rejected == 0 {
+ // An empty but well-formed batch. The collector did nothing
+ // wrong and must be told so, or it retries forever.
+ r.writeSuccess(w, enc, 0, "")
+ return
+ }
+ r.note(0, rejected, reason)
+ otlpRejected.WithLabelValues("unprojectable").Add(float64(rejected))
+ r.writeSuccess(w, enc, rejected, reason)
+ return
+ }
+
+ if err := r.pub.PublishBatch(req.Context(), events); err != nil {
+ // 503 with Retry-After, not 200 and not 400. The collector's queue
+ // is the thing that will hold these records until the broker is
+ // back, and both of the other answers throw them away: 200 says
+ // they landed, 400 says they never will.
+ r.note(0, uint64(len(events)), err.Error())
+ otlpRejected.WithLabelValues("publish_error").Add(float64(len(events)))
+ log.Error().Err(err).Int("records", len(events)).Msg("otlp: publish failed; answering 503 so the collector retries")
+ w.Header().Set("Retry-After", "5")
+ r.refuse(w, enc, http.StatusServiceUnavailable, "publish_error", "the event bus is unreachable; retry")
+ return
+ }
+
+ r.note(uint64(len(events)), rejected, reason)
+ otlpRecords.WithLabelValues(string(enc)).Add(float64(len(events)))
+ if rejected > 0 {
+ otlpRejected.WithLabelValues("unprojectable").Add(float64(rejected))
+ }
+ r.writeSuccess(w, enc, rejected, reason)
+}
+
+// project turns an export request into normalised events.
+//
+// A record that cannot be projected is counted and skipped rather than
+// failing the batch: one emitter sending something odd must not cost every
+// other emitter in the same collector their records.
+func (r *Receiver) project(export *collogs.ExportLogsServiceRequest, tok *inbox.Token, tmpl *inbox.Template) ([]*normalizer.NormalizedEvent, uint64, string) {
+ receivedAt := time.Now().UTC().Format(time.RFC3339Nano)
+ tenantID := tok.TenantID.String()
+ var (
+ events []*normalizer.NormalizedEvent
+ rejected uint64
+ reason string
+ )
+
+ for _, rl := range export.GetResourceLogs() {
+ resourceAttrs := attributesToMap(rl.GetResource().GetAttributes())
+ for _, sl := range rl.GetScopeLogs() {
+ scope := sl.GetScope()
+ scopeAttrs := attributesToMap(scope.GetAttributes())
+ for _, record := range sl.GetLogRecords() {
+ payload, err := recordPayload(record, resourceAttrs, scopeAttrs, scope.GetName(), scope.GetVersion())
+ if err != nil {
+ rejected++
+ if reason == "" {
+ reason = err.Error()
+ }
+ continue
+ }
+ ocsf := tmpl.Apply(payload, tenantID, "otlp:logs", receivedAt)
+ events = append(events, &normalizer.NormalizedEvent{
+ ID: uuid.NewString(),
+ ConnectorID: "otlp:logs",
+ TenantID: tenantID,
+ OcsfEvent: ocsf,
+ NormalizationVersion: "otlp-logs/1.0",
+ })
+ }
+ }
+ }
+ return events, rejected, reason
+}
+
+// recordPayload flattens one log record into the namespace the template
+// reads.
+//
+// Attribute keys in OpenTelemetry are dotted by convention (`service.name`,
+// `host.name`), and the template's path lookup splits on dots, so a dotted
+// key is written as nested maps. A key that would have to overwrite a
+// scalar already at that path is kept under its literal name instead —
+// silently dropping it would lose an attribute, and clobbering would lose
+// the other one.
+func recordPayload(record *logspb.LogRecord, resourceAttrs, scopeAttrs map[string]any, scopeName, scopeVersion string) (map[string]any, error) {
+ if record == nil {
+ return nil, errors.New("a nil log record")
+ }
+ payload := map[string]any{}
+
+ // Resource first, then scope, then the record's own: the narrower
+ // scope wins, which is the precedence the data model describes.
+ for _, attrs := range []map[string]any{resourceAttrs, scopeAttrs, attributesToMap(record.GetAttributes())} {
+ for k, v := range attrs {
+ assign(payload, k, v)
+ }
+ }
+
+ payload["body"] = anyValueToString(record.GetBody())
+ payload["severity_number"] = int(record.GetSeverityNumber())
+ severityText := strings.TrimSpace(record.GetSeverityText())
+ if severityText == "" {
+ // The number is the authoritative field; an emitter may set only
+ // it. Deriving the text here is what keeps the template's ladder
+ // from reading an empty string and filing everything as Unknown.
+ severityText = severityTextFor(record.GetSeverityNumber())
+ }
+ payload["severity_text"] = severityText
+
+ if ts := record.GetTimeUnixNano(); ts > 0 {
+ payload["timestamp"] = time.Unix(0, int64(ts)).UTC().Format(time.RFC3339Nano)
+ } else if ts := record.GetObservedTimeUnixNano(); ts > 0 {
+ payload["timestamp"] = time.Unix(0, int64(ts)).UTC().Format(time.RFC3339Nano)
+ }
+ if id := record.GetTraceId(); len(id) > 0 {
+ payload["trace_id"] = hex.EncodeToString(id)
+ }
+ if id := record.GetSpanId(); len(id) > 0 {
+ payload["span_id"] = hex.EncodeToString(id)
+ }
+ if scopeName != "" {
+ payload["scope_name"] = scopeName
+ }
+ if scopeVersion != "" {
+ payload["scope_version"] = scopeVersion
+ }
+ if n := record.GetDroppedAttributesCount(); n > 0 {
+ // Carried rather than ignored: a record that arrived with
+ // attributes missing is a different thing from one that never had
+ // them, and only the emitter knows which.
+ payload["dropped_attributes_count"] = int(n)
+ }
+
+ // raw_data is written as a JSON string, which is what ingest does
+ // everywhere else in this service; a reader expecting a map here is
+ // the defect this convention exists to avoid.
+ if raw, err := json.Marshal(payload); err == nil {
+ payload["raw"] = string(raw)
+ }
+ return payload, nil
+}
+
+// assign writes a possibly-dotted attribute key into the payload as nested
+// maps, refusing to overwrite a scalar that is already in the way.
+func assign(payload map[string]any, key string, value any) {
+ if key == "" {
+ return
+ }
+ parts := strings.Split(key, ".")
+ cur := payload
+ for i, p := range parts {
+ if i == len(parts)-1 {
+ cur[p] = value
+ return
+ }
+ next, ok := cur[p].(map[string]any)
+ if !ok {
+ if _, occupied := cur[p]; occupied {
+ // A scalar is in the way: `service` is already a string
+ // and `service.name` wants it to be a map. Keep both by
+ // writing the literal key at the top level.
+ payload[key] = value
+ return
+ }
+ next = map[string]any{}
+ cur[p] = next
+ }
+ cur = next
+ }
+}
+
+func attributesToMap(attrs []*commonpb.KeyValue) map[string]any {
+ if len(attrs) == 0 {
+ return nil
+ }
+ out := make(map[string]any, len(attrs))
+ for _, kv := range attrs {
+ if kv == nil || kv.GetKey() == "" {
+ continue
+ }
+ out[kv.GetKey()] = anyValueToGo(kv.GetValue())
+ }
+ return out
+}
+
+// anyValueToGo unwraps an OTLP AnyValue. Arrays and maps are rendered as
+// their JSON, because the OCSF event is a flat-ish document and a nested
+// protobuf oneof is not something a detection rule can clause on.
+func anyValueToGo(v *commonpb.AnyValue) any {
+ if v == nil {
+ return nil
+ }
+ switch value := v.GetValue().(type) {
+ case *commonpb.AnyValue_StringValue:
+ return value.StringValue
+ case *commonpb.AnyValue_BoolValue:
+ return value.BoolValue
+ case *commonpb.AnyValue_IntValue:
+ return value.IntValue
+ case *commonpb.AnyValue_DoubleValue:
+ return value.DoubleValue
+ case *commonpb.AnyValue_BytesValue:
+ return hex.EncodeToString(value.BytesValue)
+ case *commonpb.AnyValue_ArrayValue:
+ items := make([]any, 0, len(value.ArrayValue.GetValues()))
+ for _, item := range value.ArrayValue.GetValues() {
+ items = append(items, anyValueToGo(item))
+ }
+ return items
+ case *commonpb.AnyValue_KvlistValue:
+ return attributesToMap(value.KvlistValue.GetValues())
+ default:
+ return nil
+ }
+}
+
+// anyValueToString renders a record body as the one line a log is.
+func anyValueToString(v *commonpb.AnyValue) string {
+ switch got := anyValueToGo(v).(type) {
+ case nil:
+ return ""
+ case string:
+ return got
+ case bool:
+ return strconv.FormatBool(got)
+ case int64:
+ return strconv.FormatInt(got, 10)
+ case float64:
+ return strconv.FormatFloat(got, 'f', -1, 64)
+ default:
+ if encoded, err := json.Marshal(got); err == nil {
+ return string(encoded)
+ }
+ return fmt.Sprintf("%v", got)
+ }
+}
+
+// severityTextFor maps the OTLP severity number onto the short name the
+// specification defines for it. The ladder in otlp-logs.yaml reads the
+// text, so a record carrying only a number still gets a severity.
+func severityTextFor(n logspb.SeverityNumber) string {
+ switch {
+ case n >= 21:
+ return "fatal"
+ case n >= 17:
+ return "error"
+ case n >= 13:
+ return "warn"
+ case n >= 9:
+ return "info"
+ case n >= 5:
+ return "debug"
+ case n >= 1:
+ return "trace"
+ default:
+ return ""
+ }
+}
+
+func encodingOf(contentType string) (encoding, error) {
+ media := strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])
+ switch strings.ToLower(media) {
+ case "application/x-protobuf", "application/protobuf":
+ return encodingProtobuf, nil
+ case "application/json":
+ return encodingJSON, nil
+ case "":
+ // The specification requires the header. Defaulting silently to
+ // protobuf would turn a JSON body into "undecodable", which sends
+ // the operator to debug their payload instead of their header.
+ return encodingProtobuf, errors.New("no Content-Type; OTLP/HTTP requires application/x-protobuf or application/json")
+ default:
+ return encodingProtobuf, fmt.Errorf("content type %q is not an OTLP encoding; use application/x-protobuf or application/json", media)
+ }
+}
+
+func bearerToken(req *http.Request) string {
+ if h := req.Header.Get("Authorization"); h != "" {
+ if after, found := strings.CutPrefix(h, "Bearer "); found {
+ return strings.TrimSpace(after)
+ }
+ if after, found := strings.CutPrefix(h, "bearer "); found {
+ return strings.TrimSpace(after)
+ }
+ }
+ // A collector that cannot be made to send Authorization can send the
+ // same header every other push path on this service accepts.
+ return strings.TrimSpace(req.Header.Get("X-Inbox-Token"))
+}
+
+func readBody(req *http.Request) ([]byte, error) {
+ var reader io.Reader = http.MaxBytesReader(nil, req.Body, MaxBodyBytes)
+ if strings.EqualFold(req.Header.Get("Content-Encoding"), "gzip") {
+ gz, err := gzip.NewReader(reader)
+ if err != nil {
+ return nil, fmt.Errorf("Content-Encoding is gzip but the body is not: %w", err)
+ }
+ defer func() { _ = gz.Close() }()
+ reader = io.LimitReader(gz, MaxBodyBytes)
+ }
+ body, err := io.ReadAll(reader)
+ if err != nil {
+ return nil, fmt.Errorf("reading the request body: %w", err)
+ }
+ if len(body) == 0 {
+ return nil, errors.New("empty body")
+ }
+ return body, nil
+}
+
+func (r *Receiver) note(received, rejected uint64, lastError string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.state.Received += received
+ r.state.Rejected += rejected
+ if lastError != "" {
+ r.state.LastError = lastError
+ }
+ r.state.LastAt = time.Now().UTC()
+}
+
+// writeSuccess answers 200 with the specification's partial-success body.
+func (r *Receiver) writeSuccess(w http.ResponseWriter, enc encoding, rejected uint64, message string) {
+ resp := &collogs.ExportLogsServiceResponse{}
+ if rejected > 0 {
+ if message == "" {
+ message = "some records could not be projected"
+ }
+ resp.PartialSuccess = &collogs.ExportLogsPartialSuccess{
+ RejectedLogRecords: int64(rejected),
+ ErrorMessage: message,
+ }
+ }
+ r.writeProto(w, enc, http.StatusOK, resp)
+}
+
+// refuse answers a status with a body in the request's own encoding. The
+// message is the operator's next move, not a restatement of the status.
+func (r *Receiver) refuse(w http.ResponseWriter, enc encoding, status int, reason, message string) {
+ otlpRejected.WithLabelValues(reason).Inc()
+ r.note(0, 0, message)
+ // The OTLP specification says a non-200 carries a google.rpc.Status.
+ // Returning the partial-success shape with the message is what every
+ // collector logs, and it keeps one body type on this endpoint.
+ resp := &collogs.ExportLogsServiceResponse{
+ PartialSuccess: &collogs.ExportLogsPartialSuccess{ErrorMessage: message},
+ }
+ r.writeProto(w, enc, status, resp)
+}
+
+func (r *Receiver) writeProto(w http.ResponseWriter, enc encoding, status int, resp *collogs.ExportLogsServiceResponse) {
+ var (
+ body []byte
+ err error
+ )
+ if enc == encodingJSON {
+ w.Header().Set("Content-Type", "application/json")
+ body, err = protojson.Marshal(resp)
+ } else {
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ body, err = proto.Marshal(resp)
+ }
+ if err != nil {
+ log.Error().Err(err).Msg("otlp: encoding the response failed")
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ w.WriteHeader(status)
+ if _, err := w.Write(body); err != nil {
+ log.Debug().Err(err).Msg("otlp: the client went away before the response was written")
+ }
+}
diff --git a/services/ingest/internal/server/server.go b/services/ingest/internal/server/server.go
index d3d9ba8af..68446a8bc 100644
--- a/services/ingest/internal/server/server.go
+++ b/services/ingest/internal/server/server.go
@@ -13,6 +13,7 @@ import (
"github.com/beenuar/aisoc/services/ingest/internal/graph_ws"
"github.com/beenuar/aisoc/services/ingest/internal/handler"
"github.com/beenuar/aisoc/services/ingest/internal/inbox"
+ "github.com/beenuar/aisoc/services/ingest/internal/otlp"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/cors"
@@ -67,7 +68,10 @@ type Server struct {
// update fan-out). Nil disables the /v1/graph_ws/stream route, which
// is the expected default until an operator opts in via
// AISOC_GRAPH_WS_ENABLED=true.
-func New(cfg *config.Config, h *handler.Handler, inboxHandler *inbox.Handler, graphWSServer *graph_ws.Server) *Server {
+// otlpReceiver is the optional depth-4.2 OTLP logs endpoint. Nil leaves
+// POST /v1/logs unmounted, which is the default until an operator sets
+// AISOC_OTLP_LOGS_ENABLED=true.
+func New(cfg *config.Config, h *handler.Handler, inboxHandler *inbox.Handler, graphWSServer *graph_ws.Server, otlpReceiver *otlp.Receiver) *Server {
r := chi.NewRouter()
// Middleware
@@ -155,6 +159,14 @@ func New(cfg *config.Config, h *handler.Handler, inboxHandler *inbox.Handler, gr
if graphWSServer != nil {
r.Handle("/graph_ws/stream", graphWSServer.Handler())
}
+
+ // Depth plan 4.2 — OTLP logs. The path is fixed by the OTLP/HTTP
+ // specification: an exporter configured with an endpoint appends
+ // /v1/logs itself, so this cannot be mounted anywhere else and
+ // still be reachable by a stock collector.
+ if otlpReceiver != nil {
+ r.Post("/logs", otlpReceiver.ServeLogs)
+ }
})
return &Server{
diff --git a/services/ingest/main.go b/services/ingest/main.go
index f456c7f6a..944471c81 100644
--- a/services/ingest/main.go
+++ b/services/ingest/main.go
@@ -21,7 +21,9 @@ import (
"github.com/beenuar/aisoc/services/ingest/internal/handler"
"github.com/beenuar/aisoc/services/ingest/internal/inbox"
"github.com/beenuar/aisoc/services/ingest/internal/ingestauth"
+ "github.com/beenuar/aisoc/services/ingest/internal/kafkainput"
"github.com/beenuar/aisoc/services/ingest/internal/normalizer"
+ "github.com/beenuar/aisoc/services/ingest/internal/otlp"
"github.com/beenuar/aisoc/services/ingest/internal/publisher"
"github.com/beenuar/aisoc/services/ingest/internal/server"
"github.com/beenuar/aisoc/services/ingest/internal/syslog"
@@ -366,7 +368,93 @@ func main() {
log.Info().Msg("syslog: disabled (AISOC_SYSLOG_ENABLED!=true)")
}
- srv := server.New(cfg, h, inboxHandler, graphWSServer)
+ // Depth plan 4.2 — the OTLP logs receiver. Off unless the operator asks
+ // for it. It shares the HTTP port rather than binding one of its own,
+ // because an OpenTelemetry exporter is configured with a single
+ // endpoint URL and appends /v1/logs itself.
+ var otlpReceiver *otlp.Receiver
+ if cfg.OTLPEnabled {
+ if tokenStore == nil {
+ log.Error().Msg("otlp: not started — DATABASE_DSN is unset, so an exporter's ingest token cannot be resolved to a tenant")
+ } else {
+ registry := inbox.NewRegistry()
+ if err := registry.LoadEmbedded(); err != nil {
+ log.Error().Err(err).Msg("otlp: not started — embedded templates failed to load")
+ } else {
+ if err := registry.Load(cfg.InboxTemplatesDir); err != nil {
+ log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("otlp: template overrides on disk could not be read")
+ }
+ receiver, err := otlp.New(tokenStore, registry, pub)
+ if err != nil {
+ log.Error().Err(err).Msg("otlp: receiver disabled")
+ } else {
+ otlpReceiver = receiver
+ // /readyz names the receiver. An endpoint nobody is
+ // exporting to and one refusing every request look the
+ // same from outside unless readiness says which.
+ h.RegisterSubscription("otlp_logs", func() handler.SubscriptionStatus {
+ return handler.SubscriptionStatus{
+ Attached: true,
+ NotResolving: false,
+ Detail: receiver.Detail(),
+ }
+ })
+ log.Info().Str("path", "/v1/logs").Str("template", otlp.TemplateID).
+ Msg("otlp: logs receiver accepting protobuf and JSON")
+ }
+ }
+ }
+ } else {
+ log.Info().Msg("otlp: disabled (AISOC_OTLP_LOGS_ENABLED!=true)")
+ }
+
+ // Depth plan 4.2 — customer Kafka topics. One source per tenant; the
+ // tenant comes from each source's minted token and from nothing in the
+ // record.
+ var kafkaSources []*kafkainput.Source
+ if sources, err := kafkainput.ParseSources(cfg.KafkaInputSources); err != nil {
+ // Loud and specific. A malformed source list that started the
+ // service with the input silently off is the exact shape of defect
+ // this repository keeps finding.
+ log.Error().Err(err).Msg("kafka_input: no sources started — AISOC_KAFKA_INPUT_SOURCES could not be read")
+ } else if len(sources) > 0 {
+ if tokenStore == nil {
+ log.Error().Int("sources", len(sources)).
+ Msg("kafka_input: no sources started — DATABASE_DSN is unset, so their ingest tokens cannot be resolved to tenants")
+ } else {
+ registry := inbox.NewRegistry()
+ if err := registry.LoadEmbedded(); err != nil {
+ log.Error().Err(err).Msg("kafka_input: no sources started — embedded templates failed to load")
+ } else {
+ if err := registry.Load(cfg.InboxTemplatesDir); err != nil {
+ log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("kafka_input: template overrides on disk could not be read")
+ }
+ for _, sc := range sources {
+ src, err := kafkainput.NewSource(sc, tokenStore, registry, pub)
+ if err != nil {
+ log.Error().Err(err).Str("source", sc.Name).Msg("kafka_input: source disabled")
+ continue
+ }
+ kafkaSources = append(kafkaSources, src)
+ name := sc.Name
+ h.RegisterSubscription("kafka_input:"+name, func() handler.SubscriptionStatus {
+ health := src.Health()
+ return handler.SubscriptionStatus{
+ Attached: health.Attached,
+ NotResolving: health.NotResolving || health.PermanentFailure,
+ Detail: health.Detail(),
+ }
+ })
+ log.Info().Str("source", sc.Name).Str("topic", sc.Topic).Str("group", sc.GroupID).
+ Msg("kafka_input: consuming a customer topic")
+ }
+ }
+ }
+ } else {
+ log.Info().Msg("kafka_input: no sources configured (AISOC_KAFKA_INPUT_SOURCES is empty)")
+ }
+
+ srv := server.New(cfg, h, inboxHandler, graphWSServer, otlpReceiver)
// Graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
@@ -377,6 +465,11 @@ func main() {
defer graphWSBroker.Stop()
}
+ for _, src := range kafkaSources {
+ src.Start(ctx)
+ defer src.Stop()
+ }
+
if syslogListener != nil {
syslogListener.Start(ctx)
defer func() {