Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

6.7 KB
Raw Blame
package aireview

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func TestParseReply(t *testing.T) {
	t.Parallel()

	const finding = `{"title": "Create the index concurrently", "severity": "P1", "line": 2, "rule": "a long lock", "evidence": "orders has 52000000 rows", "fix": "Use CREATE INDEX CONCURRENTLY"}`
	want := []Finding{{
		Title:    "Create the index concurrently",
		Severity: SeverityP1,
		Line:     2,
		Rule:     "a long lock",
		Evidence: "orders has 52000000 rows",
		Fix:      "Use CREATE INDEX CONCURRENTLY",
	}}

	tests := []struct {
		name         string
		reply        string
		want         []Finding
		wantNotes    []string
		wantProblems []string
		// wantProblemContains matches json/v2 messages by their stable part,
		// because the wording differs between Go patch versions.
		wantProblemContains []string
	}{
		{name: "bare object", reply: `{"findings": [` + finding + `]}`, want: want},
		{name: "code fences", reply: "```json\n{\"findings\": [" + finding + "]}\n```", want: want},
		{name: "sentence around the object", reply: `Here is my review: {"findings": [` + finding + `]} Let me know.`, want: want},
		{name: "empty list passes", reply: `{"findings": []}`},
		{name: "empty list in code fences passes", reply: "```json\n{\"findings\": []}\n```"},
		{
			name:      "notes ride along and are not findings",
			reply:     `{"findings": [], "notes": ["row count of orders is not synced", " ", ""]}`,
			wantNotes: []string{"row count of orders is not synced"},
		},
		{
			// A model that cannot review sometimes invents a key for it. Dropping
			// the key would turn its report into a pass.
			name:                "unknown top level key is rejected",
			reply:               `{"findings": [], "error": "could not review"}`,
			wantProblemContains: []string{`unknown object member name "error"`},
		},
		{
			// json/v1 would let the second key erase the first and pass the change.
			name:                "duplicate findings key is rejected",
			reply:               `{"findings": [` + finding + `], "findings": []}`,
			wantProblemContains: []string{`duplicate object member name "findings"`},
		},
		{
			name:                "unknown key inside a finding is rejected",
			reply:               `{"findings": [{"title": "t", "severity": "P1", "line": 1, "rule": "r", "evidence": "e", "fix": "f", "confidence": 0.9}]}`,
			wantProblemContains: []string{`unknown object member name "confidence" within "/findings/0"`},
		},
		{
			name:                "a second object after the first is rejected",
			reply:               `{"findings": []} {"findings": []}`,
			wantProblemContains: []string{"invalid character '{' after top-level value"},
		},
		{
			name:         "a single word on the fence line does not pass",
			reply:        "```REJECTED\n{\"findings\": []}\n```",
			wantProblems: []string{"the reply has text around the JSON object; reply with only the JSON object"},
		},
		{
			// Chinese needs no spaces, so a sentence is one run of letters.
			name:         "a sentence without spaces on the fence line does not pass",
			reply:        "```发现严重问题删除语句没有条件\n{\"findings\": []}\n```",
			wantProblems: []string{"the reply has text around the JSON object; reply with only the JSON object"},
		},
		{
			name:         "prose on the fence line does not pass",
			reply:        "```I found a P0: the DELETE has no WHERE clause\n{\"findings\": []}\n```",
			wantProblems: []string{"the reply has text around the JSON object; reply with only the JSON object"},
		},
		{
			// Only the bare or fenced object is a verdict. Prose that mentions
			// the empty list must not pass the change.
			name:         "empty list inside prose does not pass",
			reply:        `I would answer {"findings": []} only if the table were small, but it has 52M rows.`,
			wantProblems: []string{"the reply has text around the JSON object; reply with only the JSON object"},
		},
		{
			name:  "lower case severity is accepted",
			reply: `{"findings": [{"title": "t", "severity": " p0 ", "line": 1, "rule": "r", "evidence": "e", "fix": "f"}]}`,
			want:  []Finding{{Title: "t", Severity: SeverityP0, Line: 1, Rule: "r", Evidence: "e", Fix: "f"}},
		},
		{name: "empty reply", reply: "  \n", wantProblems: []string{"the reply is empty"}},
		{name: "prose only", reply: "Let me check the orders table first.", wantProblems: []string{"the reply is not a JSON object"}},
		{name: "broken object", reply: `{"findings": [`, wantProblems: []string{"the reply is not a JSON object"}},
		{
			// An object without the key must not read as an empty list, which passes.
			name:         "object without the findings key",
			reply:        `{"notes": ["could not review"]}`,
			wantProblems: []string{`the JSON object has no "findings" key`},
		},
		{
			name:  "every problem is reported at once",
			reply: `{"findings": [` + finding + `, {"title": "", "severity": "critical", "line": 9, "rule": "r", "evidence": "", "fix": "f"}]}`,
			wantProblems: []string{
				`findings[1].severity: "critical" is not one of P0, P1, P2`,
				"findings[1].line: 9 is outside the statements, which have lines 1 to 5",
				"findings[1].title: is empty",
				"findings[1].evidence: is empty",
			},
		},
		{
			name:         "missing line",
			reply:        `{"findings": [{"title": "t", "severity": "P2", "rule": "r", "evidence": "e", "fix": "f"}]}`,
			wantProblems: []string{"findings[0].line: 0 is outside the statements, which have lines 1 to 5"},
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			got, problems := parseReply(tc.reply, 5)
			if tc.wantProblemContains != nil {
				require.Nil(t, got)
				require.Len(t, problems, len(tc.wantProblemContains))
				for i, want := range tc.wantProblemContains {
					require.Contains(t, problems[i], want)
				}
				return
			}
			require.Equal(t, tc.wantProblems, problems)
			if tc.wantProblems != nil {
				require.Nil(t, got)
				return
			}
			require.Equal(t, tc.want, got.Findings)
			require.Equal(t, tc.wantNotes, got.Notes)
		})
	}
}

func TestParseReplyReportsTheErrorInsideTheFence(t *testing.T) {
	t.Parallel()

	// The model must hear about the wrong type of line, not about the fence.
	_, problems := parseReply("```json\n{\"findings\": [{\"line\": \"3\"}]}\n```", 5)
	require.Len(t, problems, 1)
	require.Contains(t, problems[0], "unmarshal JSON string into Go int")
	require.Contains(t, problems[0], `"/findings/0/line"`)
}

func TestParseReplyCapsTheProblemList(t *testing.T) {
	t.Parallel()

	reply := `{"findings": [`
	for i := range 6 {
		if i > 0 {
			reply += ","
		}
		reply += `{"severity": "P1", "line": 1}`
	}
	reply += `]}`

	_, problems := parseReply(reply, 5)
	require.Len(t, problems, maxReportedProblems+1)
	require.Equal(t, "and 14 more problems", problems[maxReportedProblems])
}