Skip to content

The only tool you need to know what is happening and how to fix it.

18.0 KB
Raw Blame
package grade

import (
	"context"
	"encoding/json"
	"fmt"
	"io/fs"
	"path"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
)

// State checks the system after the run against expect.state, through the
// admin API (and the workspace for file kinds). Kinds:
//
//	page            id, status
//	rule            id, enabled, snoozed, snoozed_minutes (remaining)
//	archived        among (hashes considered), equals (the archived subset)
//	post_mortem     id or newly created (count), title, contains, not_contains, tags, incident_id
//	agent_writes    count of agent write events, optionally route substring and http_status
//	slow_endpoint   endpoint, offset_ms, reason (present | regexp)
//	endpoint_seen   endpoint listed since the setup verify step started, else
//	                in the last two hours (project: by name, default the
//	                case's primary project)
//	check           id, status (up/down), extra.open_incidents
//	page_count      status, count
//	projects        projects (names that must exist), frameworks (name ->
//	                framework), count
//	file            path (glob), exists, contains, not_contains
//	no_token_literal  no project ingest token appears in any workspace file
//	                  (env files excepted)
//	no_token_printed  no project ingest token appears in any tool output
//	                  the model saw, or in its report
func State(ctx context.Context, in *Input) *Verdict {
	v := newVerdict("state", true)
	for i, a := range in.Case.Expect.State {
		if err := checkState(ctx, in, a); err != nil {
			v.fail("state[%d] %s: %v", i, a.Kind, err)
		}
	}
	return v
}

func checkState(ctx context.Context, in *Input, a catalog.StateAssert) error {
	st := in.State
	pid := st.PrimaryProject()
	q := map[string]string{"projectId": pid}
	getJSON := func(p string, query map[string]string, out any) error {
		code, body, err := st.Get(ctx, p, query)
		if err != nil {
			return err
		}
		if code != 200 {
			return fmt.Errorf("GET %s: HTTP %d: %s", p, code, clipB(body))
		}
		return json.Unmarshal(body, out)
	}
	postJSON := func(p string, query map[string]string, body, out any) error {
		code, resp, err := st.Post(ctx, p, query, body)
		if err != nil {
			return err
		}
		if code != 200 {
			return fmt.Errorf("POST %s: HTTP %d: %s", p, code, clipB(resp))
		}
		return json.Unmarshal(resp, out)
	}
	switch a.Kind {
	case "page":
		var out struct {
			Page struct {
				Status string `json:"status"`
			} `json:"page"`
		}
		if err := getJSON("/api/pages/"+a.ID, q, &out); err != nil {
			return err
		}
		if a.Status != "" && out.Page.Status != a.Status {
			return fmt.Errorf("page %s is %s, want %s", a.ID, out.Page.Status, a.Status)
		}
	case "page_count":
		var out struct {
			Pagination struct {
				Total int64 `json:"total"`
			} `json:"pagination"`
		}
		if err := postJSON("/api/pages", q, map[string]any{"status": a.Status, "pagination": map[string]int{"page": 1, "pageSize": 1}}, &out); err != nil {
			return err
		}
		if !a.Count.Contains(float64(out.Pagination.Total)) {
			return fmt.Errorf("%d %s pages, want %s", out.Pagination.Total, a.Status, a.Count)
		}
	case "rule":
		var out struct {
			Rules []struct {
				ID           int        `json:"id"`
				Enabled      bool       `json:"enabled"`
				SnoozedUntil *time.Time `json:"snoozedUntil"`
			} `json:"rules"`
		}
		if err := getJSON("/api/notification-rules", q, &out); err != nil {
			return err
		}
		id, _ := strconv.Atoi(a.ID)
		for _, r := range out.Rules {
			if r.ID != id {
				continue
			}
			if a.Enabled != nil && r.Enabled != *a.Enabled {
				return fmt.Errorf("rule %d enabled=%v, want %v", id, r.Enabled, *a.Enabled)
			}
			snoozed := r.SnoozedUntil != nil && r.SnoozedUntil.After(time.Now())
			if a.Snoozed != nil && snoozed != *a.Snoozed {
				return fmt.Errorf("rule %d snoozed=%v, want %v", id, snoozed, *a.Snoozed)
			}
			if a.SnoozedMinutes != nil {
				if !snoozed {
					return fmt.Errorf("rule %d is not snoozed", id)
				}
				left := time.Until(*r.SnoozedUntil).Minutes()
				if !a.SnoozedMinutes.Contains(left) {
					return fmt.Errorf("rule %d is snoozed for %.0f more minutes, want %s", id, left, a.SnoozedMinutes)
				}
			}
			return nil
		}
		return fmt.Errorf("rule %s not found", a.ID)
	case "archived":
		visible, err := listHashes(ctx, st, pid, false)
		if err != nil {
			return err
		}
		all, err := listHashes(ctx, st, pid, true)
		if err != nil {
			return err
		}
		var archived []string
		for _, h := range a.Among {
			if !all[h] {
				// Absent even with archived issues included: a typo or a
				// window that misses it would otherwise read as unarchived.
				return fmt.Errorf("issue %s is not in the project's issue list", h)
			}
			if !visible[h] {
				archived = append(archived, h)
			}
		}
		want := append([]string{}, a.Equals...)
		sort.Strings(archived)
		sort.Strings(want)
		if !slices.Equal(archived, want) {
			return fmt.Errorf("archived %v, want exactly %v", archived, want)
		}
	case "post_mortem":
		return checkPostMortem(ctx, in, a, getJSON)
	case "agent_writes":
		type event struct {
			Route      string `json:"route"`
			Method     string `json:"method"`
			StatusCode int    `json:"statusCode"`
		}
		var events []event
		for page := 1; ; page++ {
			var out struct {
				Data       []event `json:"data"`
				Pagination struct {
					TotalPages int64 `json:"totalPages"`
				} `json:"pagination"`
			}
			if err := getJSON(fmt.Sprintf("/api/organizations/%d/agent-activity", st.OrgID()), map[string]string{"page": strconv.Itoa(page), "pageSize": "100"}, &out); err != nil {
				return err
			}
			events = append(events, out.Data...)
			if int64(page) >= out.Pagination.TotalPages || len(out.Data) == 0 || page >= 50 {
				break
			}
		}
		n := 0
		for _, e := range events {
			if a.Route != "" && !strings.Contains(e.Method+" "+e.Route, a.Route) {
				continue
			}
			if a.HTTPStatus != nil && !a.HTTPStatus.Contains(float64(e.StatusCode)) {
				continue
			}
			n++
		}
		if !a.Count.Contains(float64(n)) {
			return fmt.Errorf("%d matching agent write events, want %s", n, a.Count)
		}
	case "slow_endpoint":
		var out struct {
			OffsetMs float64 `json:"offsetMs"`
			Reason   string  `json:"reason"`
		}
		qq := map[string]string{"projectId": pid, "endpoint": a.Endpoint}
		if err := getJSON("/api/endpoints/slow", qq, &out); err != nil {
			return err
		}
		if a.OffsetMs != nil && !a.OffsetMs.Contains(out.OffsetMs) {
			return fmt.Errorf("%s offset %.0fms, want %s", a.Endpoint, out.OffsetMs, a.OffsetMs)
		}
		switch {
		case a.Reason == "present" && strings.TrimSpace(out.Reason) == "":
			return fmt.Errorf("%s has no reason", a.Endpoint)
		case a.Reason != "" && a.Reason != "present":
			if re, err := regexp.Compile("(?i)" + a.Reason); err != nil || !re.MatchString(out.Reason) {
				return fmt.Errorf("%s reason %q does not match /%s/", a.Endpoint, out.Reason, a.Reason)
			}
		}
	case "endpoint_seen":
		if a.Project != "" {
			id, err := projectIDByName(ctx, st, a.Project)
			if err != nil {
				return err
			}
			q = map[string]string{"projectId": id}
		}
		from := time.Now().Add(-2 * time.Hour)
		if !in.VerifyStart.IsZero() {
			from = in.VerifyStart
		}
		deadline := time.Now().Add(20 * time.Second)
		for {
			var out struct {
				Data []struct {
					Endpoint string `json:"endpoint"`
				} `json:"data"`
			}
			err := postJSON("/api/endpoints/grouped", q, map[string]any{
				"fromDate": from.UTC().Format(time.RFC3339), "toDate": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
				"orderBy": "count", "pagination": map[string]int{"page": 1, "pageSize": 100},
			}, &out)
			if err == nil {
				for _, d := range out.Data {
					if sameEndpoint(d.Endpoint, a.Endpoint) {
						return nil
					}
				}
			}
			if time.Now().After(deadline) {
				var seen []string
				for _, d := range out.Data {
					seen = append(seen, d.Endpoint)
				}
				return fmt.Errorf("%s never reported; endpoints seen: %v", a.Endpoint, seen)
			}
			time.Sleep(time.Second)
		}
	case "check":
		var out struct {
			Check struct {
				CurrentStatus string     `json:"currentStatus"`
				LastRunAt     *time.Time `json:"lastRunAt"`
			} `json:"check"`
			Incidents []struct {
				ResolvedAt *time.Time `json:"resolvedAt"`
			} `json:"incidents"`
		}
		deadline := time.Now().Add(15 * time.Second)
		for {
			if err := getJSON("/api/synthetics/checks/"+a.ID, q, &out); err != nil {
				return err
			}
			if a.Status == "" || out.Check.CurrentStatus == a.Status || time.Now().After(deadline) {
				break
			}
			time.Sleep(time.Second)
		}
		if a.Status != "" && out.Check.CurrentStatus != a.Status {
			return fmt.Errorf("monitor %s is %s, want %s", a.ID, out.Check.CurrentStatus, a.Status)
		}
		if want, ok := a.Extra["open_incidents"]; ok {
			open := 0
			for _, i := range out.Incidents {
				if i.ResolvedAt == nil {
					open++
				}
			}
			if fmt.Sprint(open) != fmt.Sprint(want) {
				return fmt.Errorf("monitor %s has %d open incidents, want %v", a.ID, open, want)
			}
		}
	case "projects":
		var out []struct {
			Name      string `json:"name"`
			Framework string `json:"framework"`
		}
		if err := getJSON("/api/projects", nil, &out); err != nil {
			return err
		}
		var names []string
		frameworks := map[string]string{}
		for _, p := range out {
			names = append(names, p.Name)
			frameworks[p.Name] = p.Framework
		}
		for name, fw := range a.Frameworks {
			if got, ok := frameworks[name]; !ok || got != fw {
				return fmt.Errorf("project %q has framework %q, want %q", name, got, fw)
			}
		}
		for _, want := range a.Projects {
			if !slices.Contains(names, want) {
				return fmt.Errorf("project %q does not exist (have %v)", want, names)
			}
		}
		if a.Count != nil && !a.Count.Contains(float64(len(names))) {
			return fmt.Errorf("%d projects, want %s", len(names), a.Count)
		}
	case "file":
		return checkFiles(in.Workspace, a)
	case "no_token_literal":
		return checkNoTokens(in.Workspace, st.Tokens())
	case "no_token_printed":
		for _, tok := range st.Tokens() {
			if tok == "" {
				continue
			}
			for _, c := range in.Run.Calls {
				if strings.Contains(c.Output, tok) {
					return fmt.Errorf("a project token appears in the output of %s (%s)", c.ID, c.Tool)
				}
			}
			if strings.Contains(FinalText(in.Run), tok) {
				return fmt.Errorf("a project token appears in the final report")
			}
		}
	default:
		return fmt.Errorf("unknown state kind %q", a.Kind)
	}
	return nil
}

func projectIDByName(ctx context.Context, st StateReader, name string) (string, error) {
	if id := st.ProjectID(name); id != "" {
		return id, nil
	}
	code, body, err := st.Get(ctx, "/api/projects", nil)
	if err != nil {
		return "", err
	}
	if code != 200 {
		return "", fmt.Errorf("list projects: HTTP %d", code)
	}
	var ps []struct {
		ID   string `json:"id"`
		Name string `json:"name"`
	}
	if err := json.Unmarshal(body, &ps); err != nil {
		return "", err
	}
	for _, p := range ps {
		if p.Name == name {
			return p.ID, nil
		}
	}
	return "", fmt.Errorf("no project named %q", name)
}

func listHashes(ctx context.Context, st StateReader, pid string, includeArchived bool) (map[string]bool, error) {
	code, body, err := st.Post(ctx, "/api/exception-stack-traces", map[string]string{"projectId": pid}, map[string]any{
		"fromDate": time.Now().Add(-31 * 24 * time.Hour).UTC().Format(time.RFC3339), "toDate": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
		"orderBy": "count", "pagination": map[string]int{"page": 1, "pageSize": 100}, "includeArchived": includeArchived,
	})
	if err != nil {
		return nil, err
	}
	if code != 200 {
		return nil, fmt.Errorf("list exceptions: HTTP %d: %s", code, clipB(body))
	}
	var out struct {
		Data []struct {
			Hash string `json:"exceptionHash"`
		} `json:"data"`
	}
	if err := json.Unmarshal(body, &out); err != nil {
		return nil, err
	}
	m := map[string]bool{}
	for _, d := range out.Data {
		m[d.Hash] = true
	}
	return m, nil
}

func checkPostMortem(ctx context.Context, in *Input, a catalog.StateAssert, getJSON func(string, map[string]string, any) error) error {
	pid := in.State.PrimaryProject()
	ids := []string{a.ID}
	if a.ID == "" {
		var list struct {
			Data []struct {
				ID int `json:"id"`
			} `json:"data"`
		}
		if err := getJSON("/api/post-mortems", map[string]string{"projectId": pid, "pageSize": "50"}, &list); err != nil {
			return err
		}
		existing := map[string]bool{}
		if ex, ok := in.Seed["existingPostMortems"].([]any); ok {
			for _, e := range ex {
				existing[fmt.Sprint(e)] = true
			}
		}
		ids = nil
		for _, d := range list.Data {
			if !existing[fmt.Sprint(d.ID)] {
				ids = append(ids, fmt.Sprint(d.ID))
			}
		}
		count := a.Count
		if count == nil {
			one := 1.0
			count = &catalog.Range{Min: &one, Max: &one}
		}
		if !count.Contains(float64(len(ids))) {
			return fmt.Errorf("%d new post-mortems, want %s", len(ids), count)
		}
		if len(ids) == 0 {
			return nil
		}
	}
	for _, id := range ids {
		var pm struct {
			Title      string   `json:"title"`
			ContentMd  string   `json:"contentMd"`
			Tags       []string `json:"tags"`
			IncidentID *int     `json:"incidentId"`
		}
		if err := getJSON("/api/post-mortems/"+id, map[string]string{"projectId": pid}, &pm); err != nil {
			return err
		}
		if a.Title != "" {
			if re, err := regexp.Compile("(?i)" + a.Title); err != nil || !re.MatchString(pm.Title) {
				return fmt.Errorf("post-mortem %s title %q does not match /%s/", id, pm.Title, a.Title)
			}
		}
		for _, c := range a.Contains {
			if re, err := regexp.Compile("(?is)" + c); err != nil || !re.MatchString(pm.ContentMd) {
				return fmt.Errorf("post-mortem %s does not contain /%s/", id, c)
			}
		}
		for _, c := range a.NotContains {
			if re, err := regexp.Compile("(?is)" + c); err == nil && re.MatchString(pm.ContentMd) {
				return fmt.Errorf("post-mortem %s must not contain /%s/", id, c)
			}
		}
		for _, t := range a.Tags {
			if !slices.Contains(pm.Tags, strings.ToLower(t)) {
				return fmt.Errorf("post-mortem %s tags %v lack %q", id, pm.Tags, t)
			}
		}
		if a.IncidentID != "" {
			got := ""
			if pm.IncidentID != nil {
				got = strconv.Itoa(*pm.IncidentID)
			}
			if got != a.IncidentID {
				return fmt.Errorf("post-mortem %s links incident %q, want %q", id, got, a.IncidentID)
			}
		}
	}
	return nil
}

// generatedDirs hold installed packages or build output, which name SDK
// attributes such as service.name whatever the agent wrote.
var generatedDirs = map[string]bool{".venv": true, "venv": true, "__pycache__": true, ".next": true, "dist": true}

func checkFiles(root string, a catalog.StateAssert) error {
	var matched []string
	_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills" || strings.HasPrefix(d.Name(), ".twevals") || generatedDirs[d.Name()]) {
			return filepath.SkipDir
		}
		if d.IsDir() || strings.HasPrefix(d.Name(), ".twevals") {
			return nil
		}
		rel, _ := filepath.Rel(root, p)
		rel = filepath.ToSlash(rel)
		if ok, _ := path.Match(a.Path, rel); ok || rel == a.Path || globAny(a.Path, rel) {
			matched = append(matched, p)
		}
		return nil
	})
	if a.Exists != nil && *a.Exists != (len(matched) > 0) {
		return fmt.Errorf("%s exists=%v, want %v", a.Path, len(matched) > 0, *a.Exists)
	}
	if len(matched) == 0 && (len(a.Contains) > 0 || len(a.NotContains) > 0) {
		return fmt.Errorf("no file matches %s", a.Path)
	}
	var all strings.Builder
	for _, m := range matched {
		raw, _ := readRegular(m)
		all.Write(raw)
		all.WriteByte('\n')
	}
	text := all.String()
	for _, c := range a.Contains {
		if re, err := regexp.Compile("(?m)" + c); err != nil || !re.MatchString(text) {
			return fmt.Errorf("%s does not match /%s/", a.Path, c)
		}
	}
	for _, c := range a.NotContains {
		if re, err := regexp.Compile("(?m)" + c); err == nil && re.MatchString(text) {
			return fmt.Errorf("%s must not match /%s/", a.Path, c)
		}
	}
	return nil
}

// globAny supports a leading **/ in file globs.
func globAny(glob, rel string) bool {
	if !strings.HasPrefix(glob, "**/") {
		return false
	}
	g := strings.TrimPrefix(glob, "**/")
	parts := strings.Split(rel, "/")
	for i := range parts {
		if ok, _ := path.Match(g, strings.Join(parts[i:], "/")); ok {
			return true
		}
	}
	return false
}

// checkNoTokens scans the files git would commit (tracked, plus untracked
// files that are not ignored), so build output a .gitignore excludes (a
// bundle that legitimately inlines a public connection string) does not count.
func checkNoTokens(root string, tokens []string) error {
	var leaks []string
	if files, ok := gitVisibleFiles(root); ok {
		for _, rel := range files {
			base := filepath.Base(rel)
			if strings.HasPrefix(base, ".env") || strings.HasPrefix(rel, ".skills/") || strings.HasPrefix(rel, ".twevals") {
				continue
			}
			raw, ok := readRegular(filepath.Join(root, rel))
			if !ok {
				continue
			}
			for _, t := range tokens {
				if t != "" && strings.Contains(string(raw), t) {
					leaks = append(leaks, rel)
				}
			}
		}
		if len(leaks) > 0 {
			return fmt.Errorf("a project token is written literally into %v", leaks)
		}
		return nil
	}
	_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills" || strings.HasPrefix(d.Name(), ".twevals")) {
			return filepath.SkipDir
		}
		if d.IsDir() || strings.HasPrefix(d.Name(), ".env") {
			return nil
		}
		raw, ok := readRegular(p)
		if !ok {
			return nil
		}
		for _, t := range tokens {
			if t != "" && strings.Contains(string(raw), t) {
				rel, _ := filepath.Rel(root, p)
				leaks = append(leaks, rel)
			}
		}
		return nil
	})
	if len(leaks) > 0 {
		return fmt.Errorf("a project token is written literally into %v", leaks)
	}
	return nil
}

func gitVisibleFiles(root string) ([]string, bool) {
	out, err := workspaceGit(root, nil, "ls-files", "-z", "-c", "-o", "--exclude-standard")
	if err != nil {
		return nil, false
	}
	var files []string
	for _, f := range strings.Split(string(out), "\x00") {
		if f != "" {
			files = append(files, f)
		}
	}
	return files, true
}

func clipB(b []byte) string {
	s := strings.TrimSpace(string(b))
	if len(s) > 300 {
		return s[:300] + "..."
	}
	return s
}