Git designed for AI agents.
graft-resolve bench: report per language the answers the syntax check gave no verdict on
Federico Elia4883789c9750
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.goindex 1aefa18..e81ee29 100644--- a/tools/graft-agents/resolve/bench/bench_test.go+++ b/tools/graft-agents/resolve/bench/bench_test.go@@ -82,7 +82,7 @@ func TestReportByLanguageAndKind(t *testing.T) {if err := rep.WriteMarkdown(&b); err != nil {t.Fatal(err)}- for _, want := range []string{"| language | cases", "| rust (low n) | 1 |", "| kind | cases", "| content (low n) | 2 | 1 (0) | 1 |", "| modify-delete (low n) | 1 |"} {+ for _, want := range []string{"| language | cases", "| rust (low n) | 1 |", "| kind | cases", "| content (low n) | 2 | 1 (0) | 1 |", "| modify-delete (low n) | 1 |", "By language: java 0 of 1 accepted, rust 0 of 1 accepted."} {if !strings.Contains(b.String(), want) {t.Fatalf("markdown lacks %q:\n%s", want, b.String())}@@ -135,7 +135,7 @@ func TestReportCounts(t *testing.T) {t.Fatal(err)}if !strings.Contains(b.String(), "| total | 4 | 1 (1) | 1 | 1 | 1 |") || !strings.Contains(b.String(), "blank lines only (counted as wrong): 1 (go 1, python 0)") ||- !strings.Contains(b.String(), "does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): 1 (go 1, python 0)") {+ !strings.Contains(b.String(), "does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): 1 of 2.") {t.Fatalf("markdown:\n%s", b.String())}}diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex 6840ab3..6a2aa52 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -183,8 +183,21 @@ func (rep Report) WriteMarkdown(w io.Writer) error {}fmt.Fprintf(&b, "\nWrong results that differ from the human resolution in blank lines only (counted as wrong): %d (%s).\n",rep.Total.BlankLinesOnly, perFamily(func(c Counts) int { return c.BlankLinesOnly }))- fmt.Fprintf(&b, "\nAccepted results the syntax check gave no verdict on, because a side does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): %d (%s).\n",- rep.Total.SyntaxUnchecked, perFamily(func(c Counts) int { return c.SyntaxUnchecked }))+ languages := make([]string, 0, len(rep.Languages))+ for l := range rep.Languages {+ languages = append(languages, l)+ }+ sort.Strings(languages)+ var unchecked []string+ for _, l := range languages {+ c := rep.Languages[l]+ if l == "" {+ continue+ }+ unchecked = append(unchecked, fmt.Sprintf("%s %d of %d accepted", l, c.SyntaxUnchecked, c.Accepted()))+ }+ fmt.Fprintf(&b, "\nAccepted results the syntax check gave no verdict on, because a side does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): %d of %d. By language: %s.\n",+ rep.Total.SyntaxUnchecked, rep.Total.Accepted(), strings.Join(unchecked, ", "))if len(rep.NotSent) > 0 {reasons := make([]string, 0, len(rep.NotSent))for r := range rep.NotSent {