Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(sentinel): mark the Logs Ingestion DCR Direct; cap Vector batches at 512 KB

shukieshah8c5e779bfb40
diff --git a/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json b/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json
index ee8b87cb..b6e45857 100644
--- a/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json
+++ b/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json
@@ -29,6 +29,7 @@
"apiVersion": "2023-03-11",
"name": "[parameters('dataCollectionRuleName')]",
"location": "[parameters('location')]",
+ "kind": "Direct",
"properties": {
"dataCollectionEndpointId": "[parameters('dataCollectionEndpointResourceId')]",
"streamDeclarations": {
diff --git a/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl
index fa7a461b..970602fc 100644
--- a/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl
+++ b/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl
@@ -50,7 +50,10 @@ azure_tenant_id = "${AZURE_TENANT_ID}"
azure_client_id = "${AZURE_CLIENT_ID}"
azure_client_secret = "${AZURE_CLIENT_SECRET}"
+# The Logs Ingestion API rejects a request over 1 MB with 413, and the sink's own default is 10 MB.
+# 512 KB leaves room for the JSON escaping of RawData; one Beacon event is at most 64 KB.
[sinks.beacon_sentinel.batch]
+max_bytes = 524288
max_events = 1000
timeout_secs = 30
diff --git a/cli/beacon/internal/endpoint/sentinel/sentinel_test.go b/cli/beacon/internal/endpoint/sentinel/sentinel_test.go
index 569b6dbe..cc9faaea 100644
--- a/cli/beacon/internal/endpoint/sentinel/sentinel_test.go
+++ b/cli/beacon/internal/endpoint/sentinel/sentinel_test.go
@@ -5,6 +5,8 @@ import (
"encoding/json"
"os"
"path/filepath"
+ "regexp"
+ "strconv"
"strings"
"testing"
)
@@ -107,6 +109,7 @@ func TestRenderedDCRTemplateContainsTransformFromKQL(t *testing.T) {
// dcrRule is the part of a DCR ARM template the two ingestion paths have to agree on.
type dcrRule struct {
Resources []struct {
+ Kind string `json:"kind"`
Properties struct {
StreamDeclarations map[string]struct {
Columns []struct {
@@ -160,6 +163,11 @@ func TestLogsIngestionPathMatchesTheDCRAndTheAgentPath(t *testing.T) {
if got := ingest.DataFlows[0].Streams; len(got) != 1 || got[0] != stream {
t.Errorf("the Logs Ingestion data flow reads %v, want %s", got, stream)
}
+ // Microsoft's Logs Ingestion templates mark the DCR Direct. Without it Azure treats the rule as
+ // an agent DCR, not one the Logs Ingestion API posts to.
+ if kind := parseDCR(t, "pack/dcr-logs-ingestion-template.json").Resources[0].Kind; kind != "Direct" {
+ t.Errorf("the Logs Ingestion DCR has kind %q, want Direct", kind)
+ }
if ingest.DataFlows[0].OutputStream != agent.DataFlows[0].OutputStream {
t.Errorf("the two paths write to different tables: %q and %q",
ingest.DataFlows[0].OutputStream, agent.DataFlows[0].OutputStream)
@@ -258,3 +266,16 @@ func TestKQLAssetsMentionSentinelTableAndValidation(t *testing.T) {
t.Fatal("queries.kql should include the Sentinel validation phrase")
}
}
+
+// The Logs Ingestion API rejects requests over 1 MB, and the azure_logs_ingestion sink batches up to
+// 10 MB by default, so a busy flush would lose a whole batch to 413s.
+func TestVectorBatchesFitTheLogsIngestionLimit(t *testing.T) {
+ vector := mustRead("pack/vector.toml.tmpl")
+ match := regexp.MustCompile(`(?m)^max_bytes = (\d+)$`).FindStringSubmatch(vector)
+ if match == nil {
+ t.Fatal("vector.toml does not cap batch.max_bytes, so the sink batches up to 10 MB")
+ }
+ if n, _ := strconv.Atoi(match[1]); n <= 0 || n > 1_000_000 {
+ t.Fatalf("batch.max_bytes = %s, want at most 1 MB", match[1])
+ }
+}