The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(sentinel): mark the Logs Ingestion DCR Direct; cap Vector batches at 512 KB
shukieshah8c5e779bfb40
diff --git a/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json b/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.jsonindex ee8b87cb..b6e45857 100644--- a/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json+++ b/cli/beacon/internal/endpoint/sentinel/pack/dcr-logs-ingestion-template.json@@ -29,6 +29,7 @@"apiVersion": "2023-03-11","name": "[parameters('dataCollectionRuleName')]","location": "[parameters('location')]",+ "kind": "Direct","properties": {"dataCollectionEndpointId": "[parameters('dataCollectionEndpointResourceId')]","streamDeclarations": {diff --git a/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmplindex fa7a461b..970602fc 100644--- a/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl+++ b/cli/beacon/internal/endpoint/sentinel/pack/vector.toml.tmpl@@ -50,7 +50,10 @@ azure_tenant_id = "${AZURE_TENANT_ID}"azure_client_id = "${AZURE_CLIENT_ID}"azure_client_secret = "${AZURE_CLIENT_SECRET}"+# The Logs Ingestion API rejects a request over 1 MB with 413, and the sink's own default is 10 MB.+# 512 KB leaves room for the JSON escaping of RawData; one Beacon event is at most 64 KB.[sinks.beacon_sentinel.batch]+max_bytes = 524288max_events = 1000timeout_secs = 30diff --git a/cli/beacon/internal/endpoint/sentinel/sentinel_test.go b/cli/beacon/internal/endpoint/sentinel/sentinel_test.goindex 569b6dbe..cc9faaea 100644--- a/cli/beacon/internal/endpoint/sentinel/sentinel_test.go+++ b/cli/beacon/internal/endpoint/sentinel/sentinel_test.go@@ -5,6 +5,8 @@ import ("encoding/json""os""path/filepath"+ "regexp"+ "strconv""strings""testing")@@ -107,6 +109,7 @@ func TestRenderedDCRTemplateContainsTransformFromKQL(t *testing.T) {// dcrRule is the part of a DCR ARM template the two ingestion paths have to agree on.type dcrRule struct {Resources []struct {+ Kind string `json:"kind"`Properties struct {StreamDeclarations map[string]struct {Columns []struct {@@ -160,6 +163,11 @@ func TestLogsIngestionPathMatchesTheDCRAndTheAgentPath(t *testing.T) {if got := ingest.DataFlows[0].Streams; len(got) != 1 || got[0] != stream {t.Errorf("the Logs Ingestion data flow reads %v, want %s", got, stream)}+ // Microsoft's Logs Ingestion templates mark the DCR Direct. Without it Azure treats the rule as+ // an agent DCR, not one the Logs Ingestion API posts to.+ if kind := parseDCR(t, "pack/dcr-logs-ingestion-template.json").Resources[0].Kind; kind != "Direct" {+ t.Errorf("the Logs Ingestion DCR has kind %q, want Direct", kind)+ }if ingest.DataFlows[0].OutputStream != agent.DataFlows[0].OutputStream {t.Errorf("the two paths write to different tables: %q and %q",ingest.DataFlows[0].OutputStream, agent.DataFlows[0].OutputStream)@@ -258,3 +266,16 @@ func TestKQLAssetsMentionSentinelTableAndValidation(t *testing.T) {t.Fatal("queries.kql should include the Sentinel validation phrase")}}++// The Logs Ingestion API rejects requests over 1 MB, and the azure_logs_ingestion sink batches up to+// 10 MB by default, so a busy flush would lose a whole batch to 413s.+func TestVectorBatchesFitTheLogsIngestionLimit(t *testing.T) {+ vector := mustRead("pack/vector.toml.tmpl")+ match := regexp.MustCompile(`(?m)^max_bytes = (\d+)$`).FindStringSubmatch(vector)+ if match == nil {+ t.Fatal("vector.toml does not cap batch.max_bytes, so the sink batches up to 10 MB")+ }+ if n, _ := strconv.Atoi(match[1]); n <= 0 || n > 1_000_000 {+ t.Fatalf("batch.max_bytes = %s, want at most 1 MB", match[1])+ }+}