Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(ingest): OTLP logs receiver on POST /v1/logs (depth 4.2)
Beenu Arorae8ea7a97d34e
diff --git a/services/ingest/go.mod b/services/ingest/go.modindex ca047d6f2..1190dea72 100644--- a/services/ingest/go.mod+++ b/services/ingest/go.mod@@ -17,6 +17,8 @@ require (go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0go.opentelemetry.io/otel/sdk v1.47.0go.opentelemetry.io/otel/trace v1.47.0+ go.opentelemetry.io/proto/otlp v1.11.0+ google.golang.org/protobuf v1.36.12gopkg.in/yaml.v3 v3.0.1)@@ -48,7 +50,6 @@ require (go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirectgo.opentelemetry.io/otel/log v1.47.0 // indirectgo.opentelemetry.io/otel/metric v1.47.0 // indirect- go.opentelemetry.io/proto/otlp v1.11.0 // indirectgo.uber.org/atomic v1.11.0 // indirectgolang.org/x/net v0.60.0 // indirectgolang.org/x/sync v0.23.0 // indirect@@ -57,5 +58,4 @@ require (google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirectgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirectgoogle.golang.org/grpc v1.83.2 // indirect- google.golang.org/protobuf v1.36.12 // indirect)diff --git a/services/ingest/internal/config/config.go b/services/ingest/internal/config/config.goindex 6143bfad0..95d13506f 100644--- a/services/ingest/internal/config/config.go+++ b/services/ingest/internal/config/config.go@@ -88,6 +88,24 @@ type Config struct {SyslogFlushMs intSyslogMaxConnections int+ // Depth plan 4.2 — the OTLP logs receiver and the customer Kafka input.+ //+ // Both off by default, and both take the tenant from a minted ingest+ // token for the same reason syslog does: neither OTLP nor a Kafka+ // record carries an authenticated principal, so a tenant read out of+ // a resource attribute or a record header would be a cross-tenant+ // write any instrumented process could perform.+ //+ // OTLPEnabled adds POST /v1/logs. There is no address of its own: the+ // receiver shares the service's HTTP port, which is what an+ // OpenTelemetry exporter expects to find behind one endpoint URL.+ OTLPEnabled bool+ // KafkaInputSources is a JSON array of source objects. An array rather+ // than numbered variables because a deployment reading three tenants'+ // topics needs three of everything, and numbered variables are how a+ // fourth source silently gets the third one's token.+ KafkaInputSources string+// Workstream 6 — universal capture push paths.// InboxEnabled toggles the /v1/inbox/* routes. Off by default in// development if no DATABASE_DSN is set, since the inbox store needs@@ -246,9 +264,12 @@ func Load() (*Config, error) {SyslogBatchSize: mustGetEnvInt("AISOC_SYSLOG_BATCH_SIZE", 500),SyslogFlushMs: mustGetEnvInt("AISOC_SYSLOG_FLUSH_MS", 2000),SyslogMaxConnections: mustGetEnvInt("AISOC_SYSLOG_MAX_CONNECTIONS", 256),- InboxEnabled: getEnv("INBOX_ENABLED", "true") == "true",- InboxTemplatesDir: getEnv("INBOX_TEMPLATES_DIR", "/app/templates"),- InboxMaxBodyBytes: int64(mustGetEnvInt("INBOX_MAX_BODY_BYTES", 10*1024*1024)),++ OTLPEnabled: getEnv("AISOC_OTLP_LOGS_ENABLED", "false") == "true",+ KafkaInputSources: getEnv("AISOC_KAFKA_INPUT_SOURCES", ""),+ InboxEnabled: getEnv("INBOX_ENABLED", "true") == "true",+ InboxTemplatesDir: getEnv("INBOX_TEMPLATES_DIR", "/app/templates"),+ InboxMaxBodyBytes: int64(mustGetEnvInt("INBOX_MAX_BODY_BYTES", 10*1024*1024)),// Defaults sized to be invisible to a well-behaved vendor and// firmly bounding to a runaway one: 50 req/s sustained with a// 200 burst, and 5k events/s with a 20k burst (one large batchdiff --git a/services/ingest/internal/normalizer/templates/otlp-logs.yaml b/services/ingest/internal/normalizer/templates/otlp-logs.yamlnew file mode 100644index 000000000..7e1cef7ce--- /dev/null+++ b/services/ingest/internal/normalizer/templates/otlp-logs.yaml@@ -0,0 +1,72 @@+# OpenTelemetry Protocol (OTLP) logs.+#+# Used by the OTLP logs receiver in services/ingest, which accepts+# ExportLogsServiceRequest over HTTP in both the protobuf and the JSON+# encodings the specification defines.+#+# Class 1007 Process Activity, in category 1 (System Activity), for the+# same reason syslog.yaml is: an OTLP log record is an application saying+# something about itself. The promoter promotes category 2 unconditionally,+# so filing routine application logs as findings would turn every line into+# an alert, which is precisely the noise this platform exists to remove. A+# detection rule is what turns one of these into an alert.+#+# The receiver flattens a record's resource attributes, scope and log+# attributes into a single namespace before this template is applied, so a+# rule can clause on `service.name` or any attribute the emitter set.+#+# References:+# https://opentelemetry.io/docs/specs/otlp/+# https://opentelemetry.io/docs/specs/otel/logs/data-model/+id: otlp-logs+vendor_name: OpenTelemetry+product_name: OTLP+class_uid: 1007+class_name: Process Activity+activity_id: 1+field_map:+ body: message+ severity_text: metadata.labels.severity_text+ severity_number: metadata.labels.severity_number+ trace_id: metadata.correlation_uid+ span_id: metadata.labels.span_id+ scope_name: metadata.logged_by+ host.name: device.hostname+ service.name: actor.process.name+ process.pid: actor.process.pid+ event.name: metadata.event_code+ raw: raw_data+# OTLP severity text is free-form but the specification names these short+# names, and emitters overwhelmingly use them. The number is the+# authoritative field; the receiver derives this text from it when the+# emitter left it unset, so the ladder below always has something to read.+severity_field: severity_text+severity_map:+ trace: 1+ trace2: 1+ trace3: 1+ trace4: 1+ debug: 1+ debug2: 1+ debug3: 1+ debug4: 1+ info: 1+ info2: 1+ info3: 1+ info4: 1+ warn: 3+ warn2: 3+ warn3: 3+ warn4: 3+ error: 4+ error2: 4+ error3: 4+ error4: 4+ fatal: 6+ fatal2: 6+ fatal3: 6+ fatal4: 6+time_field: timestamp+message_field: body+constants:+ source_type: otlpdiff --git a/services/ingest/internal/otlp/receiver.go b/services/ingest/internal/otlp/receiver.gonew file mode 100644index 000000000..d55135675--- /dev/null+++ b/services/ingest/internal/otlp/receiver.go@@ -0,0 +1,586 @@+// Package otlp implements an OpenTelemetry Protocol logs receiver.+//+// Depth item 4.2 / parity 6.10. An estate that already runs an OpenTelemetry+// Collector has a logs pipeline and an exporter; what it does not have is+// somewhere to point the exporter. This is that endpoint:+//+// POST /v1/logs Content-Type: application/x-protobuf+// POST /v1/logs Content-Type: application/json+//+// Both encodings are in the specification and real collectors emit both, so+// a receiver that reads one of them is a receiver half the estates cannot+// use. gzip request bodies are accepted because the OTLP/HTTP specification+// says a client may send them and the collector's default exporter does.+//+// Where the tenant comes from+// ---------------------------+// From a minted ingest token presented as `Authorization: Bearer <token>`,+// and from nowhere else. OTLP has no notion of a tenant, and every+// attribute on a record is set by the emitter, so a tenant read out of+// `resource.attributes` would be a cross-tenant write any instrumented+// process could perform. An OTLP exporter sets request headers through+// OTEL_EXPORTER_OTLP_HEADERS, which is how the token gets here.+//+// What a client is told+// ---------------------+// The specification's partial-success contract, which is the part receivers+// usually skip. A request whose records are partly unreadable returns 200+// with `partial_success.rejected_log_records` set and a human message —+// *not* a blanket 200, which would tell a collector everything landed, and+// not a 400, which would make it retry records that did land. A full+// failure returns 503 with Retry-After so the collector's queue retries+// rather than dropping.+package otlp++import (+ "compress/gzip"+ "context"+ "encoding/hex"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "strconv"+ "strings"+ "sync"+ "time"++ "github.com/beenuar/aisoc/services/ingest/internal/inbox"+ "github.com/beenuar/aisoc/services/ingest/internal/normalizer"+ "github.com/google/uuid"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+ "github.com/rs/zerolog/log"+ collogs "go.opentelemetry.io/proto/otlp/collector/logs/v1"+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ logspb "go.opentelemetry.io/proto/otlp/logs/v1"+ "google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"+)++// TemplateID is the OCSF template an OTLP token must be minted with.+const TemplateID = "otlp-logs"++// MaxBodyBytes caps one request. An uncapped receiver is a way to make the+// ingest service run out of memory from the outside.+const MaxBodyBytes int64 = 16 << 20++var (+ otlpRecords = promauto.NewCounterVec(prometheus.CounterOpts{+ Name: "aisoc_ingest_otlp_log_records_total",+ Help: "OTLP log records accepted.",+ }, []string{"encoding"})+ otlpRejected = promauto.NewCounterVec(prometheus.CounterOpts{+ Name: "aisoc_ingest_otlp_rejected_total",+ Help: "OTLP log records or requests refused, by reason.",+ }, []string{"reason"})+)++// TokenResolver is the half of inbox.Store this package needs.+type TokenResolver interface {+ Resolve(ctx context.Context, token string) (*inbox.Token, error)+}++// TemplateSource resolves a template id to a template.+type TemplateSource interface {+ Get(id string) (*inbox.Template, error)+}++// Publisher is the half of publisher.Publisher this package needs.+type Publisher interface {+ PublishBatch(ctx context.Context, events []*normalizer.NormalizedEvent) error+}++// State is what readiness reports about the receiver. A receiver nobody is+// sending to and a receiver refusing everything look identical from outside+// unless the state is named, and silence is the normal condition for a logs+// endpoint.+type State struct {+ Enabled bool+ Received uint64+ Rejected uint64+ LastError string+ LastAt time.Time+}++// Receiver serves the OTLP logs endpoint.+type Receiver struct {+ tokens TokenResolver+ templates TemplateSource+ pub Publisher++ mu sync.Mutex+ state State+}++// New wires the receiver.+func New(tokens TokenResolver, templates TemplateSource, pub Publisher) (*Receiver, error) {+ if tokens == nil || templates == nil || pub == nil {+ return nil, errors.New("otlp: receiver needs a token resolver, a template source and a publisher")+ }+ return &Receiver{tokens: tokens, templates: templates, pub: pub, state: State{Enabled: true}}, nil+}++// Health is a snapshot for the readiness endpoint.+func (r *Receiver) Health() State {+ r.mu.Lock()+ defer r.mu.Unlock()+ return r.state+}++// Detail is the sentence /readyz shows, healthy or not.+func (r *Receiver) Detail() string {+ s := r.Health()+ detail := fmt.Sprintf("accepting OTLP logs on POST /v1/logs; records %d, rejected %d", s.Received, s.Rejected)+ if s.Received == 0 && s.Rejected == 0 {+ detail += " (nothing has arrived yet)"+ }+ if s.LastError != "" {+ detail += "; last error: " + s.LastError+ }+ return detail+}++// encoding is how one request was framed, which also decides how the+// response must be framed: the specification requires the response to use+// the same encoding as the request.+type encoding string++const (+ encodingProtobuf encoding = "protobuf"+ encodingJSON encoding = "json"+)++// ServeLogs handles POST /v1/logs.+func (r *Receiver) ServeLogs(w http.ResponseWriter, req *http.Request) {+ enc, err := encodingOf(req.Header.Get("Content-Type"))+ if err != nil {+ r.refuse(w, encodingProtobuf, http.StatusUnsupportedMediaType, "unsupported_media_type", err.Error())+ return+ }++ token := bearerToken(req)+ if token == "" {+ // Named rather than a bare 401: an OTLP exporter is configured by+ // someone who has never seen this product's token concept, and+ // "unauthorized" sends them to look at their collector.+ r.refuse(w, enc, http.StatusUnauthorized, "no_token",+ "no ingest token; set OTEL_EXPORTER_OTLP_HEADERS=\"Authorization=Bearer <token>\" with a token minted for the "+TemplateID+" template")+ return+ }++ body, err := readBody(req)+ if err != nil {+ r.refuse(w, enc, http.StatusBadRequest, "unreadable_body", err.Error())+ return+ }++ export := &collogs.ExportLogsServiceRequest{}+ switch enc {+ case encodingJSON:+ // DiscardUnknown: the OTLP schema gains fields, and a collector a+ // version ahead of this build must not have its whole batch+ // refused over one field this build has never heard of.+ if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(body, export); err != nil {+ r.refuse(w, enc, http.StatusBadRequest, "undecodable", "the body is not an OTLP ExportLogsServiceRequest in JSON: "+err.Error())+ return+ }+ default:+ if err := proto.Unmarshal(body, export); err != nil {+ r.refuse(w, enc, http.StatusBadRequest, "undecodable", "the body is not an OTLP ExportLogsServiceRequest in protobuf: "+err.Error())+ return+ }+ }++ tok, err := r.tokens.Resolve(req.Context(), token)+ if err != nil {+ r.refuse(w, enc, http.StatusUnauthorized, "bad_token", "the presented ingest token is not valid")+ return+ }+ if tok.TemplateID != TemplateID {+ r.refuse(w, enc, http.StatusForbidden, "wrong_template",+ fmt.Sprintf("this token was minted with template %q; the OTLP receiver needs one minted with %q", tok.TemplateID, TemplateID))+ return+ }+ tmpl, err := r.templates.Get(TemplateID)+ if err != nil {+ r.refuse(w, enc, http.StatusServiceUnavailable, "no_template", "this build has no "+TemplateID+" template, so OTLP records cannot be normalised")+ return+ }++ events, rejected, reason := r.project(export, tok, tmpl)++ if len(events) == 0 {+ if rejected == 0 {+ // An empty but well-formed batch. The collector did nothing+ // wrong and must be told so, or it retries forever.+ r.writeSuccess(w, enc, 0, "")+ return+ }+ r.note(0, rejected, reason)+ otlpRejected.WithLabelValues("unprojectable").Add(float64(rejected))+ r.writeSuccess(w, enc, rejected, reason)+ return+ }++ if err := r.pub.PublishBatch(req.Context(), events); err != nil {+ // 503 with Retry-After, not 200 and not 400. The collector's queue+ // is the thing that will hold these records until the broker is+ // back, and both of the other answers throw them away: 200 says+ // they landed, 400 says they never will.+ r.note(0, uint64(len(events)), err.Error())+ otlpRejected.WithLabelValues("publish_error").Add(float64(len(events)))+ log.Error().Err(err).Int("records", len(events)).Msg("otlp: publish failed; answering 503 so the collector retries")+ w.Header().Set("Retry-After", "5")+ r.refuse(w, enc, http.StatusServiceUnavailable, "publish_error", "the event bus is unreachable; retry")+ return+ }++ r.note(uint64(len(events)), rejected, reason)+ otlpRecords.WithLabelValues(string(enc)).Add(float64(len(events)))+ if rejected > 0 {+ otlpRejected.WithLabelValues("unprojectable").Add(float64(rejected))+ }+ r.writeSuccess(w, enc, rejected, reason)+}++// project turns an export request into normalised events.+//+// A record that cannot be projected is counted and skipped rather than+// failing the batch: one emitter sending something odd must not cost every+// other emitter in the same collector their records.+func (r *Receiver) project(export *collogs.ExportLogsServiceRequest, tok *inbox.Token, tmpl *inbox.Template) ([]*normalizer.NormalizedEvent, uint64, string) {+ receivedAt := time.Now().UTC().Format(time.RFC3339Nano)+ tenantID := tok.TenantID.String()+ var (+ events []*normalizer.NormalizedEvent+ rejected uint64+ reason string+ )++ for _, rl := range export.GetResourceLogs() {+ resourceAttrs := attributesToMap(rl.GetResource().GetAttributes())+ for _, sl := range rl.GetScopeLogs() {+ scope := sl.GetScope()+ scopeAttrs := attributesToMap(scope.GetAttributes())+ for _, record := range sl.GetLogRecords() {+ payload, err := recordPayload(record, resourceAttrs, scopeAttrs, scope.GetName(), scope.GetVersion())+ if err != nil {+ rejected+++ if reason == "" {+ reason = err.Error()+ }+ continue+ }+ ocsf := tmpl.Apply(payload, tenantID, "otlp:logs", receivedAt)+ events = append(events, &normalizer.NormalizedEvent{+ ID: uuid.NewString(),+ ConnectorID: "otlp:logs",+ TenantID: tenantID,+ OcsfEvent: ocsf,+ NormalizationVersion: "otlp-logs/1.0",+ })+ }+ }+ }+ return events, rejected, reason+}++// recordPayload flattens one log record into the namespace the template+// reads.+//+// Attribute keys in OpenTelemetry are dotted by convention (`service.name`,+// `host.name`), and the template's path lookup splits on dots, so a dotted+// key is written as nested maps. A key that would have to overwrite a+// scalar already at that path is kept under its literal name instead —+// silently dropping it would lose an attribute, and clobbering would lose+// the other one.+func recordPayload(record *logspb.LogRecord, resourceAttrs, scopeAttrs map[string]any, scopeName, scopeVersion string) (map[string]any, error) {+ if record == nil {+ return nil, errors.New("a nil log record")+ }+ payload := map[string]any{}++ // Resource first, then scope, then the record's own: the narrower+ // scope wins, which is the precedence the data model describes.+ for _, attrs := range []map[string]any{resourceAttrs, scopeAttrs, attributesToMap(record.GetAttributes())} {+ for k, v := range attrs {+ assign(payload, k, v)+ }+ }++ payload["body"] = anyValueToString(record.GetBody())+ payload["severity_number"] = int(record.GetSeverityNumber())+ severityText := strings.TrimSpace(record.GetSeverityText())+ if severityText == "" {+ // The number is the authoritative field; an emitter may set only+ // it. Deriving the text here is what keeps the template's ladder+ // from reading an empty string and filing everything as Unknown.+ severityText = severityTextFor(record.GetSeverityNumber())+ }+ payload["severity_text"] = severityText++ if ts := record.GetTimeUnixNano(); ts > 0 {+ payload["timestamp"] = time.Unix(0, int64(ts)).UTC().Format(time.RFC3339Nano)+ } else if ts := record.GetObservedTimeUnixNano(); ts > 0 {+ payload["timestamp"] = time.Unix(0, int64(ts)).UTC().Format(time.RFC3339Nano)+ }+ if id := record.GetTraceId(); len(id) > 0 {+ payload["trace_id"] = hex.EncodeToString(id)+ }+ if id := record.GetSpanId(); len(id) > 0 {+ payload["span_id"] = hex.EncodeToString(id)+ }+ if scopeName != "" {+ payload["scope_name"] = scopeName+ }+ if scopeVersion != "" {+ payload["scope_version"] = scopeVersion+ }+ if n := record.GetDroppedAttributesCount(); n > 0 {+ // Carried rather than ignored: a record that arrived with+ // attributes missing is a different thing from one that never had+ // them, and only the emitter knows which.+ payload["dropped_attributes_count"] = int(n)+ }++ // raw_data is written as a JSON string, which is what ingest does+ // everywhere else in this service; a reader expecting a map here is+ // the defect this convention exists to avoid.+ if raw, err := json.Marshal(payload); err == nil {+ payload["raw"] = string(raw)+ }+ return payload, nil+}++// assign writes a possibly-dotted attribute key into the payload as nested+// maps, refusing to overwrite a scalar that is already in the way.+func assign(payload map[string]any, key string, value any) {+ if key == "" {+ return+ }+ parts := strings.Split(key, ".")+ cur := payload+ for i, p := range parts {+ if i == len(parts)-1 {+ cur[p] = value+ return+ }+ next, ok := cur[p].(map[string]any)+ if !ok {+ if _, occupied := cur[p]; occupied {+ // A scalar is in the way: `service` is already a string+ // and `service.name` wants it to be a map. Keep both by+ // writing the literal key at the top level.+ payload[key] = value+ return+ }+ next = map[string]any{}+ cur[p] = next+ }+ cur = next+ }+}++func attributesToMap(attrs []*commonpb.KeyValue) map[string]any {+ if len(attrs) == 0 {+ return nil+ }+ out := make(map[string]any, len(attrs))+ for _, kv := range attrs {+ if kv == nil || kv.GetKey() == "" {+ continue+ }+ out[kv.GetKey()] = anyValueToGo(kv.GetValue())+ }+ return out+}++// anyValueToGo unwraps an OTLP AnyValue. Arrays and maps are rendered as+// their JSON, because the OCSF event is a flat-ish document and a nested+// protobuf oneof is not something a detection rule can clause on.+func anyValueToGo(v *commonpb.AnyValue) any {+ if v == nil {+ return nil+ }+ switch value := v.GetValue().(type) {+ case *commonpb.AnyValue_StringValue:+ return value.StringValue+ case *commonpb.AnyValue_BoolValue:+ return value.BoolValue+ case *commonpb.AnyValue_IntValue:+ return value.IntValue+ case *commonpb.AnyValue_DoubleValue:+ return value.DoubleValue+ case *commonpb.AnyValue_BytesValue:+ return hex.EncodeToString(value.BytesValue)+ case *commonpb.AnyValue_ArrayValue:+ items := make([]any, 0, len(value.ArrayValue.GetValues()))+ for _, item := range value.ArrayValue.GetValues() {+ items = append(items, anyValueToGo(item))+ }+ return items+ case *commonpb.AnyValue_KvlistValue:+ return attributesToMap(value.KvlistValue.GetValues())+ default:+ return nil+ }+}++// anyValueToString renders a record body as the one line a log is.+func anyValueToString(v *commonpb.AnyValue) string {+ switch got := anyValueToGo(v).(type) {+ case nil:+ return ""+ case string:+ return got+ case bool:+ return strconv.FormatBool(got)+ case int64:+ return strconv.FormatInt(got, 10)+ case float64:+ return strconv.FormatFloat(got, 'f', -1, 64)+ default:+ if encoded, err := json.Marshal(got); err == nil {+ return string(encoded)+ }+ return fmt.Sprintf("%v", got)+ }+}++// severityTextFor maps the OTLP severity number onto the short name the+// specification defines for it. The ladder in otlp-logs.yaml reads the+// text, so a record carrying only a number still gets a severity.+func severityTextFor(n logspb.SeverityNumber) string {+ switch {+ case n >= 21:+ return "fatal"+ case n >= 17:+ return "error"+ case n >= 13:+ return "warn"+ case n >= 9:+ return "info"+ case n >= 5:+ return "debug"+ case n >= 1:+ return "trace"+ default:+ return ""+ }+}++func encodingOf(contentType string) (encoding, error) {+ media := strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])+ switch strings.ToLower(media) {+ case "application/x-protobuf", "application/protobuf":+ return encodingProtobuf, nil+ case "application/json":+ return encodingJSON, nil+ case "":+ // The specification requires the header. Defaulting silently to+ // protobuf would turn a JSON body into "undecodable", which sends+ // the operator to debug their payload instead of their header.+ return encodingProtobuf, errors.New("no Content-Type; OTLP/HTTP requires application/x-protobuf or application/json")+ default:+ return encodingProtobuf, fmt.Errorf("content type %q is not an OTLP encoding; use application/x-protobuf or application/json", media)+ }+}++func bearerToken(req *http.Request) string {+ if h := req.Header.Get("Authorization"); h != "" {+ if after, found := strings.CutPrefix(h, "Bearer "); found {+ return strings.TrimSpace(after)+ }+ if after, found := strings.CutPrefix(h, "bearer "); found {+ return strings.TrimSpace(after)+ }+ }+ // A collector that cannot be made to send Authorization can send the+ // same header every other push path on this service accepts.+ return strings.TrimSpace(req.Header.Get("X-Inbox-Token"))+}++func readBody(req *http.Request) ([]byte, error) {+ var reader io.Reader = http.MaxBytesReader(nil, req.Body, MaxBodyBytes)+ if strings.EqualFold(req.Header.Get("Content-Encoding"), "gzip") {+ gz, err := gzip.NewReader(reader)+ if err != nil {+ return nil, fmt.Errorf("Content-Encoding is gzip but the body is not: %w", err)+ }+ defer func() { _ = gz.Close() }()+ reader = io.LimitReader(gz, MaxBodyBytes)+ }+ body, err := io.ReadAll(reader)+ if err != nil {+ return nil, fmt.Errorf("reading the request body: %w", err)+ }+ if len(body) == 0 {+ return nil, errors.New("empty body")+ }+ return body, nil+}++func (r *Receiver) note(received, rejected uint64, lastError string) {+ r.mu.Lock()+ defer r.mu.Unlock()+ r.state.Received += received+ r.state.Rejected += rejected+ if lastError != "" {+ r.state.LastError = lastError+ }+ r.state.LastAt = time.Now().UTC()+}++// writeSuccess answers 200 with the specification's partial-success body.+func (r *Receiver) writeSuccess(w http.ResponseWriter, enc encoding, rejected uint64, message string) {+ resp := &collogs.ExportLogsServiceResponse{}+ if rejected > 0 {+ if message == "" {+ message = "some records could not be projected"+ }+ resp.PartialSuccess = &collogs.ExportLogsPartialSuccess{+ RejectedLogRecords: int64(rejected),+ ErrorMessage: message,+ }+ }+ r.writeProto(w, enc, http.StatusOK, resp)+}++// refuse answers a status with a body in the request's own encoding. The+// message is the operator's next move, not a restatement of the status.+func (r *Receiver) refuse(w http.ResponseWriter, enc encoding, status int, reason, message string) {+ otlpRejected.WithLabelValues(reason).Inc()+ r.note(0, 0, message)+ // The OTLP specification says a non-200 carries a google.rpc.Status.+ // Returning the partial-success shape with the message is what every+ // collector logs, and it keeps one body type on this endpoint.+ resp := &collogs.ExportLogsServiceResponse{+ PartialSuccess: &collogs.ExportLogsPartialSuccess{ErrorMessage: message},+ }+ r.writeProto(w, enc, status, resp)+}++func (r *Receiver) writeProto(w http.ResponseWriter, enc encoding, status int, resp *collogs.ExportLogsServiceResponse) {+ var (+ body []byte+ err error+ )+ if enc == encodingJSON {+ w.Header().Set("Content-Type", "application/json")+ body, err = protojson.Marshal(resp)+ } else {+ w.Header().Set("Content-Type", "application/x-protobuf")+ body, err = proto.Marshal(resp)+ }+ if err != nil {+ log.Error().Err(err).Msg("otlp: encoding the response failed")+ w.WriteHeader(http.StatusInternalServerError)+ return+ }+ w.WriteHeader(status)+ if _, err := w.Write(body); err != nil {+ log.Debug().Err(err).Msg("otlp: the client went away before the response was written")+ }+}diff --git a/services/ingest/internal/server/server.go b/services/ingest/internal/server/server.goindex d3d9ba8af..68446a8bc 100644--- a/services/ingest/internal/server/server.go+++ b/services/ingest/internal/server/server.go@@ -13,6 +13,7 @@ import ("github.com/beenuar/aisoc/services/ingest/internal/graph_ws""github.com/beenuar/aisoc/services/ingest/internal/handler""github.com/beenuar/aisoc/services/ingest/internal/inbox"+ "github.com/beenuar/aisoc/services/ingest/internal/otlp""github.com/go-chi/chi/v5""github.com/go-chi/chi/v5/middleware""github.com/go-chi/cors"@@ -67,7 +68,10 @@ type Server struct {// update fan-out). Nil disables the /v1/graph_ws/stream route, which// is the expected default until an operator opts in via// AISOC_GRAPH_WS_ENABLED=true.-func New(cfg *config.Config, h *handler.Handler, inboxHandler *inbox.Handler, graphWSServer *graph_ws.Server) *Server {+// otlpReceiver is the optional depth-4.2 OTLP logs endpoint. Nil leaves+// POST /v1/logs unmounted, which is the default until an operator sets+// AISOC_OTLP_LOGS_ENABLED=true.+func New(cfg *config.Config, h *handler.Handler, inboxHandler *inbox.Handler, graphWSServer *graph_ws.Server, otlpReceiver *otlp.Receiver) *Server {r := chi.NewRouter()// Middleware@@ -155,6 +159,14 @@ func New(cfg *config.Config, h *handler.Handler, inboxHandler *inbox.Handler, grif graphWSServer != nil {r.Handle("/graph_ws/stream", graphWSServer.Handler())}++ // Depth plan 4.2 — OTLP logs. The path is fixed by the OTLP/HTTP+ // specification: an exporter configured with an endpoint appends+ // /v1/logs itself, so this cannot be mounted anywhere else and+ // still be reachable by a stock collector.+ if otlpReceiver != nil {+ r.Post("/logs", otlpReceiver.ServeLogs)+ }})return &Server{diff --git a/services/ingest/main.go b/services/ingest/main.goindex f456c7f6a..944471c81 100644--- a/services/ingest/main.go+++ b/services/ingest/main.go@@ -21,7 +21,9 @@ import ("github.com/beenuar/aisoc/services/ingest/internal/handler""github.com/beenuar/aisoc/services/ingest/internal/inbox""github.com/beenuar/aisoc/services/ingest/internal/ingestauth"+ "github.com/beenuar/aisoc/services/ingest/internal/kafkainput""github.com/beenuar/aisoc/services/ingest/internal/normalizer"+ "github.com/beenuar/aisoc/services/ingest/internal/otlp""github.com/beenuar/aisoc/services/ingest/internal/publisher""github.com/beenuar/aisoc/services/ingest/internal/server""github.com/beenuar/aisoc/services/ingest/internal/syslog"@@ -366,7 +368,93 @@ func main() {log.Info().Msg("syslog: disabled (AISOC_SYSLOG_ENABLED!=true)")}- srv := server.New(cfg, h, inboxHandler, graphWSServer)+ // Depth plan 4.2 — the OTLP logs receiver. Off unless the operator asks+ // for it. It shares the HTTP port rather than binding one of its own,+ // because an OpenTelemetry exporter is configured with a single+ // endpoint URL and appends /v1/logs itself.+ var otlpReceiver *otlp.Receiver+ if cfg.OTLPEnabled {+ if tokenStore == nil {+ log.Error().Msg("otlp: not started — DATABASE_DSN is unset, so an exporter's ingest token cannot be resolved to a tenant")+ } else {+ registry := inbox.NewRegistry()+ if err := registry.LoadEmbedded(); err != nil {+ log.Error().Err(err).Msg("otlp: not started — embedded templates failed to load")+ } else {+ if err := registry.Load(cfg.InboxTemplatesDir); err != nil {+ log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("otlp: template overrides on disk could not be read")+ }+ receiver, err := otlp.New(tokenStore, registry, pub)+ if err != nil {+ log.Error().Err(err).Msg("otlp: receiver disabled")+ } else {+ otlpReceiver = receiver+ // /readyz names the receiver. An endpoint nobody is+ // exporting to and one refusing every request look the+ // same from outside unless readiness says which.+ h.RegisterSubscription("otlp_logs", func() handler.SubscriptionStatus {+ return handler.SubscriptionStatus{+ Attached: true,+ NotResolving: false,+ Detail: receiver.Detail(),+ }+ })+ log.Info().Str("path", "/v1/logs").Str("template", otlp.TemplateID).+ Msg("otlp: logs receiver accepting protobuf and JSON")+ }+ }+ }+ } else {+ log.Info().Msg("otlp: disabled (AISOC_OTLP_LOGS_ENABLED!=true)")+ }++ // Depth plan 4.2 — customer Kafka topics. One source per tenant; the+ // tenant comes from each source's minted token and from nothing in the+ // record.+ var kafkaSources []*kafkainput.Source+ if sources, err := kafkainput.ParseSources(cfg.KafkaInputSources); err != nil {+ // Loud and specific. A malformed source list that started the+ // service with the input silently off is the exact shape of defect+ // this repository keeps finding.+ log.Error().Err(err).Msg("kafka_input: no sources started — AISOC_KAFKA_INPUT_SOURCES could not be read")+ } else if len(sources) > 0 {+ if tokenStore == nil {+ log.Error().Int("sources", len(sources)).+ Msg("kafka_input: no sources started — DATABASE_DSN is unset, so their ingest tokens cannot be resolved to tenants")+ } else {+ registry := inbox.NewRegistry()+ if err := registry.LoadEmbedded(); err != nil {+ log.Error().Err(err).Msg("kafka_input: no sources started — embedded templates failed to load")+ } else {+ if err := registry.Load(cfg.InboxTemplatesDir); err != nil {+ log.Warn().Err(err).Str("dir", cfg.InboxTemplatesDir).Msg("kafka_input: template overrides on disk could not be read")+ }+ for _, sc := range sources {+ src, err := kafkainput.NewSource(sc, tokenStore, registry, pub)+ if err != nil {+ log.Error().Err(err).Str("source", sc.Name).Msg("kafka_input: source disabled")+ continue+ }+ kafkaSources = append(kafkaSources, src)+ name := sc.Name+ h.RegisterSubscription("kafka_input:"+name, func() handler.SubscriptionStatus {+ health := src.Health()+ return handler.SubscriptionStatus{+ Attached: health.Attached,+ NotResolving: health.NotResolving || health.PermanentFailure,+ Detail: health.Detail(),+ }+ })+ log.Info().Str("source", sc.Name).Str("topic", sc.Topic).Str("group", sc.GroupID).+ Msg("kafka_input: consuming a customer topic")+ }+ }+ }+ } else {+ log.Info().Msg("kafka_input: no sources configured (AISOC_KAFKA_INPUT_SOURCES is empty)")+ }++ srv := server.New(cfg, h, inboxHandler, graphWSServer, otlpReceiver)// Graceful shutdownctx, cancel := context.WithCancel(context.Background())@@ -377,6 +465,11 @@ func main() {defer graphWSBroker.Stop()}+ for _, src := range kafkaSources {+ src.Start(ctx)+ defer src.Stop()+ }+if syslogListener != nil {syslogListener.Start(ctx)defer func() {