Skip to content

Git designed for AI agents.

graft-resolve bench: report per language the answers the syntax check gave no verdict on

Federico Elia4883789c9750
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.go
index 1aefa18..e81ee29 100644
--- a/tools/graft-agents/resolve/bench/bench_test.go
+++ b/tools/graft-agents/resolve/bench/bench_test.go
@@ -82,7 +82,7 @@ func TestReportByLanguageAndKind(t *testing.T) {
if err := rep.WriteMarkdown(&b); err != nil {
t.Fatal(err)
}
- for _, want := range []string{"| language | cases", "| rust (low n) | 1 |", "| kind | cases", "| content (low n) | 2 | 1 (0) | 1 |", "| modify-delete (low n) | 1 |"} {
+ for _, want := range []string{"| language | cases", "| rust (low n) | 1 |", "| kind | cases", "| content (low n) | 2 | 1 (0) | 1 |", "| modify-delete (low n) | 1 |", "By language: java 0 of 1 accepted, rust 0 of 1 accepted."} {
if !strings.Contains(b.String(), want) {
t.Fatalf("markdown lacks %q:\n%s", want, b.String())
}
@@ -135,7 +135,7 @@ func TestReportCounts(t *testing.T) {
t.Fatal(err)
}
if !strings.Contains(b.String(), "| total | 4 | 1 (1) | 1 | 1 | 1 |") || !strings.Contains(b.String(), "blank lines only (counted as wrong): 1 (go 1, python 0)") ||
- !strings.Contains(b.String(), "does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): 1 (go 1, python 0)") {
+ !strings.Contains(b.String(), "does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): 1 of 2.") {
t.Fatalf("markdown:\n%s", b.String())
}
}
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
index 6840ab3..6a2aa52 100644
--- a/tools/graft-agents/resolve/bench/report.go
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -183,8 +183,21 @@ func (rep Report) WriteMarkdown(w io.Writer) error {
}
fmt.Fprintf(&b, "\nWrong results that differ from the human resolution in blank lines only (counted as wrong): %d (%s).\n",
rep.Total.BlankLinesOnly, perFamily(func(c Counts) int { return c.BlankLinesOnly }))
- fmt.Fprintf(&b, "\nAccepted results the syntax check gave no verdict on, because a side does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): %d (%s).\n",
- rep.Total.SyntaxUnchecked, perFamily(func(c Counts) int { return c.SyntaxUnchecked }))
+ languages := make([]string, 0, len(rep.Languages))
+ for l := range rep.Languages {
+ languages = append(languages, l)
+ }
+ sort.Strings(languages)
+ var unchecked []string
+ for _, l := range languages {
+ c := rep.Languages[l]
+ if l == "" {
+ continue
+ }
+ unchecked = append(unchecked, fmt.Sprintf("%s %d of %d accepted", l, c.SyntaxUnchecked, c.Accepted()))
+ }
+ fmt.Fprintf(&b, "\nAccepted results the syntax check gave no verdict on, because a side does not parse with the grammar (a dialect such as Flow, or code the grammar does not cover): %d of %d. By language: %s.\n",
+ rep.Total.SyntaxUnchecked, rep.Total.Accepted(), strings.Join(unchecked, ", "))
if len(rep.NotSent) > 0 {
reasons := make([]string, 0, len(rep.NotSent))
for r := range rep.NotSent {