Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

8.2 KB
Raw Blame
package cmd

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"unicode/utf8"

	"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"

	"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules"
)

const scanTestRule = `
id: scan-test-secret-read
version: 1
title: Secret file read
severity: high
status: stable
posture: detect
match: 'e.event.action == "file.read" && e.file.path.matches("\\.env")'
emit:
  reason: Secret file was read
tests:
  - name: pos
    verdict: match
    events:
      - event: { action: file.read }
        file: { path: ".env" }
  - name: neg
    verdict: no_match
    events:
      - event: { action: file.read }
        file: { path: "README.md" }
`

// scanFixture writes a temp rules dir + a temp runtime log, and points scanOpts at them.
func scanFixture(t *testing.T, logLines []string) {
	t.Helper()
	rulesDir := t.TempDir()
	if err := os.WriteFile(filepath.Join(rulesDir, "secret.rule.yaml"), []byte(scanTestRule), 0o644); err != nil {
		t.Fatal(err)
	}
	logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
	if err := os.WriteFile(logPath, []byte(strings.Join(logLines, "\n")+"\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	// reset and set scan options for this test
	scanOpts.userMode = true
	scanOpts.systemMode = false
	scanOpts.rulesDir = rulesDir
	scanOpts.logPath = logPath
	scanOpts.jsonOutput = false
	scanOpts.minSeverity = ""
	scanOpts.session = ""
	scanOpts.failOn = ""
}

const evSecretRead = `{"timestamp":"2026-06-13T10:00:00Z","event":{"action":"file.read"},"file":{"path":".env"},"session":{"id":"s1"}}`

func TestScanJSON(t *testing.T) {
	scanFixture(t, []string{evSecretRead})
	scanOpts.jsonOutput = true
	cmd, buf := newCmd()
	if err := runScan(cmd, nil); err != nil {
		t.Fatalf("scan: %v", err)
	}
	var findings []threatrules.Finding
	if err := json.Unmarshal(buf.Bytes(), &findings); err != nil {
		t.Fatalf("scan --json did not emit valid JSON: %v\n%s", err, buf.String())
	}
	if len(findings) != 1 || findings[0].RuleID != "scan-test-secret-read" {
		t.Fatalf("unexpected json findings: %+v", findings)
	}
	if len(findings[0].Events) != 1 || findings[0].Events[0].File == nil {
		t.Fatalf("expected evidence event with file info: %+v", findings[0])
	}
}

func TestScanSessionFilterMatchesDashboardQuery(t *testing.T) {
	secretReadMixedCaseSession := `{"timestamp":"2026-06-13T10:00:00Z","event":{"action":"file.read"},"file":{"path":".env"},"session":{"id":"Session-ABC-123"}}`
	secretReadOtherSession := `{"timestamp":"2026-06-13T10:00:01Z","event":{"action":"file.read"},"file":{"path":".env"},"session":{"id":"session-other"}}`
	scanFixture(t, []string{secretReadMixedCaseSession, secretReadOtherSession})
	scanOpts.session = "abc"
	cmd, buf := newCmd()
	if err := runScan(cmd, nil); err != nil {
		t.Fatalf("scan: %v", err)
	}
	out := buf.String()
	if !strings.Contains(out, "1 finding(s) across 1 events") {
		t.Fatalf("expected partial case-insensitive session filter to scan exactly one event, got:\n%s", out)
	}
	if !strings.Contains(out, "session=Session-ABC-123") {
		t.Fatalf("expected finding from the matching session, got:\n%s", out)
	}
	if strings.Contains(out, "session=session-other") {
		t.Fatalf("did not expect finding from the non-matching session, got:\n%s", out)
	}
}

func TestScanFailOn(t *testing.T) {
	scanFixture(t, []string{evSecretRead})
	scanOpts.failOn = "high"
	cmd, _ := newCmd()
	if err := runScan(cmd, nil); err == nil {
		t.Fatalf("expected non-zero exit (error) with --fail-on high and a high finding")
	}

	// Below threshold: critical is higher than the high finding -> no failure.
	scanFixture(t, []string{evSecretRead})
	scanOpts.failOn = "critical"
	cmd, _ = newCmd()
	if err := runScan(cmd, nil); err != nil {
		t.Fatalf("did not expect failure when no finding reaches --fail-on critical: %v", err)
	}
}

// TestScanFailOnIgnoresMinSeverityFilter guards the CI gate: --min-severity is a
// display-only filter, so a detection at or above --fail-on must still fail the
// scan even when --min-severity hides it from the printed output.
func TestScanFailOnIgnoresMinSeverityFilter(t *testing.T) {
	scanFixture(t, []string{evSecretRead})
	scanOpts.minSeverity = "critical" // high finding filtered out of output
	scanOpts.failOn = "high"          // ...but still meets the fail gate
	cmd, buf := newCmd()
	err := runScan(cmd, nil)
	if err == nil {
		t.Fatalf("expected non-zero exit: a high finding meets --fail-on high even when --min-severity critical hides it; output:\n%s", buf.String())
	}
	if !strings.Contains(err.Error(), "1 finding") {
		t.Fatalf("expected fail gate to count the filtered finding, got: %v", err)
	}
	if !strings.Contains(buf.String(), "No findings") {
		t.Fatalf("expected the high finding hidden from output by --min-severity critical, got:\n%s", buf.String())
	}
}

func TestScanSkipsNewerSpecRuleWithWarning(t *testing.T) {
	scanFixture(t, []string{evSecretRead})
	future := strings.Replace(strings.Replace(scanTestRule, "id: scan-test-secret-read\n", "id: scan-future-rule\nspec: threat-rules/v2\n", 1),
		"emit:", "future_field: true\nemit:", 1)
	if err := os.WriteFile(filepath.Join(scanOpts.rulesDir, "future.rule.yaml"), []byte(future), 0o644); err != nil {
		t.Fatal(err)
	}
	scanOpts.jsonOutput = true
	stdout, stderr := &strings.Builder{}, &strings.Builder{}
	cmd, _ := newCmd()
	cmd.SetOut(stdout)
	cmd.SetErr(stderr)
	if err := runScan(cmd, nil); err != nil {
		t.Fatalf("scan: %v", err)
	}
	var findings []threatrules.Finding
	if err := json.Unmarshal([]byte(stdout.String()), &findings); err != nil {
		t.Fatalf("stdout must stay valid JSON: %v\n%s", err, stdout.String())
	}
	if len(findings) != 1 || findings[0].RuleID != "scan-test-secret-read" {
		t.Fatalf("the current rule should still run: %+v", findings)
	}
	want := `rule "scan-future-rule" requires threat-rules/v2, but this Beacon supports up to ` + threatrules.SupportedSpec + `; upgrade Beacon to load it`
	if !strings.Contains(stderr.String(), "warning: skipped ") || !strings.Contains(stderr.String(), want) {
		t.Fatalf("want the skip warning on stderr, got %q", stderr.String())
	}
}

func TestScanAllRulesSkippedNamesThem(t *testing.T) {
	scanFixture(t, []string{evSecretRead})
	// Replace the fixture's only rule with one that needs a newer spec.
	future := strings.Replace(scanTestRule, "id: scan-test-secret-read\n", "id: scan-future-rule\nspec: threat-rules/v1.3\n", 1)
	if err := os.WriteFile(filepath.Join(scanOpts.rulesDir, "secret.rule.yaml"), []byte(future), 0o644); err != nil {
		t.Fatal(err)
	}
	stdout, stderr := &strings.Builder{}, &strings.Builder{}
	cmd, _ := newCmd()
	cmd.SetOut(stdout)
	cmd.SetErr(stderr)
	err := runScan(cmd, nil)
	if err == nil {
		t.Fatal("want an error when every rule is skipped")
	}
	msg := err.Error()
	if strings.Contains(msg, "store is empty") {
		t.Fatalf("error must not claim the store is empty: %s", msg)
	}
	for _, want := range []string{`rule "scan-future-rule" requires threat-rules/v1.3`, "secret.rule.yaml", "supports up to " + threatrules.SupportedSpec, "upgrade Beacon"} {
		if !strings.Contains(msg, want) {
			t.Errorf("error %q missing %q", msg, want)
		}
	}
	if stderr.Len() != 0 {
		t.Errorf("the error names the skipped rules; no separate warning expected, got %q", stderr.String())
	}
}

func TestScanEmptyRulesDirKeepsMessage(t *testing.T) {
	scanFixture(t, []string{evSecretRead})
	if err := os.Remove(filepath.Join(scanOpts.rulesDir, "secret.rule.yaml")); err != nil {
		t.Fatal(err)
	}
	cmd, _ := newCmd()
	err := runScan(cmd, nil)
	if err == nil || err.Error() != "no rules to run (store is empty and baseline missing)" {
		t.Fatalf("want the unchanged empty-store message, got %v", err)
	}
}

// A finding's prompt or command is cut to fit one line. The cut must land between characters:
// a prompt in Japanese or with accents is mostly multibyte, and a byte cut prints a broken rune.
func TestSummarizeEventTruncatesWholeCharacters(t *testing.T) {
	prompt := strings.Repeat("é", 150)
	got := summarizeEvent(asymptoteobserve.Event{
		Event:  asymptoteobserve.EventInfo{Action: "prompt.submitted"},
		Prompt: &asymptoteobserve.PromptInfo{Text: prompt},
	})
	if !utf8.ValidString(got) {
		t.Fatalf("summary is not valid UTF-8: %q", got)
	}
	want := "prompt.submitted: " + strings.Repeat("é", 99) + "…"
	if got != want {
		t.Fatalf("summary = %q, want %q", got, want)
	}
}