Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

test: the gotestsum guard scans single lines of raw source, so a quoted flag value or a backslash continuation hides the output summary unseen (PIN-172)

Luigi Agostieb5df5cfca26
diff --git a/internal/devtools/gotestsum_test.go b/internal/devtools/gotestsum_test.go
index 0002e47c..c0646f38 100644
--- a/internal/devtools/gotestsum_test.go
+++ b/internal/devtools/gotestsum_test.go
@@ -31,24 +31,23 @@ func TestNoGotestsumInvocationHidesTheOutputSummary(t *testing.T) {
}
return nil
}
- if !scannableForInvocations(path) {
+ if !d.Type().IsRegular() || !mayHoldInvocations(path) {
return nil
}
body, readErr := os.ReadFile(path) // #nosec G304 -- files walked from this repo's own tree.
if readErr != nil {
return readErr
}
+ if !scannableForInvocations(path, string(body)) {
+ return nil
+ }
rel, _ := filepath.Rel(root, path)
- for i, line := range strings.Split(string(body), "\n") {
- if !isGotestsumInvocation(line) {
- continue
- }
- invocations++
- if hidesOutputSummary(line) {
- offenders++
- t.Errorf("%s:%d hides the gotestsum output summary, so a failure there prints a bare test name with the cause nowhere in the log; the rule is one-sided — hiding other sections is fine, hiding output is not:\n\t%s",
- rel, i+1, strings.TrimSpace(line))
- }
+ found, offences := scanGotestsumCommands(string(body))
+ invocations += found
+ for _, offence := range offences {
+ offenders++
+ t.Errorf("%s:%d hides the gotestsum output summary, so a failure there prints a bare test name with the cause nowhere in the log; the rule is one-sided — hiding other sections is fine, hiding output is not:\n\t%s",
+ rel, offence.line, offence.command)
}
return nil
})
@@ -64,12 +63,94 @@ func TestNoGotestsumInvocationHidesTheOutputSummary(t *testing.T) {
}
}
+// A command, not a line. The guard reads source text while gotestsum reads argv, and the
+// shell sits between them: quotes are stripped before the tool sees a value, and a
+// trailing backslash means the flag lives on a line that never mentions the binary. Both
+// were live evasions of a per-line scan. Everything below therefore works on a LOGICAL
+// command — comments dropped, continuations joined, quotes removed — so the ban asks a
+// question about a command rather than about a line of text.
+//
+// WHAT THIS CANNOT SEE, stated because the previous version of this comment claimed a
+// completeness it did not have: it described the spellings gotestsum ACCEPTS, which is not
+// the same as what a source scan COVERS. A scan cannot follow indirection.
+// --hide-summary=$SECTIONS, a value arriving from a workflow matrix input, or an
+// invocation inside a composite action are all unreachable by any grep or AST over this
+// repo. That is inherent to scanning source rather than argv, not a gap left unclosed.
+//
+// STOP RULE. Three rounds have gone on this one guard — flag order, then quotes and
+// continuations. The hazard is a person editing a CI flag, so redding on the spellings a
+// person writes plus naming the residual is proportionate. The next evasion is met by
+// accepting the residual, not by a fourth predicate.
+type gotestsumOffence struct {
+ line int
+ command string
+}
+
+type logicalCommand struct {
+ line int
+ text string
+}
+
+// scanGotestsumCommands returns how many gotestsum invocations the body contains and
+// which of them hide the output summary. Both counts come from the same normalisation, so
+// the invocation floor cannot pass while the ban silently scans nothing.
+func scanGotestsumCommands(body string) (invocations int, offences []gotestsumOffence) {
+ for _, command := range logicalCommands(body) {
+ if !isGotestsumInvocation(command.text) {
+ continue
+ }
+ invocations++
+ if hidesOutputSummary(command.text) {
+ offences = append(offences, gotestsumOffence{line: command.line, command: command.text})
+ }
+ }
+ return invocations, offences
+}
+
+// logicalCommands joins trailing-backslash continuations into one command and reports the
+// line the command STARTS on, so a failure points at something the reader can open.
+//
+// Comments are dropped BEFORE joining, and a dropped comment also ends any pending
+// continuation. Joining first would splice a commented-out invocation into a live command;
+// gluing across a comment would build a command that never existed.
+func logicalCommands(body string) []logicalCommand {
+ var commands []logicalCommand
+ pending := ""
+ startLine := 0
+ flush := func() {
+ if pending != "" {
+ commands = append(commands, logicalCommand{line: startLine, text: strings.TrimSpace(pending)})
+ }
+ pending = ""
+ startLine = 0
+ }
+
+ for i, raw := range strings.Split(body, "\n") {
+ trimmed := strings.TrimSpace(raw)
+ if strings.HasPrefix(trimmed, "#") {
+ flush()
+ continue
+ }
+ if pending == "" {
+ startLine = i + 1
+ }
+ if continued := strings.HasSuffix(trimmed, "\\"); continued {
+ pending += strings.TrimSuffix(trimmed, "\\") + " "
+ continue
+ }
+ pending += trimmed
+ flush()
+ }
+ flush()
+ return commands
+}
+
// hidesOutputSummary parses --hide-summary as the comma-separated list gotestsum accepts,
-// so "skipped,output" is caught as surely as "output". gotestsum rejects every other
-// spelling — a space instead of =, a single dash, uppercase section names — so the fields
-// of the =-joined lowercase list are the whole class.
-func hidesOutputSummary(line string) bool {
- rest := line
+// so "skipped,output" is caught as surely as "output". Quote characters are stripped first
+// because the shell removes them before the tool sees the value: without that, the last
+// field of --hide-summary="skipped,output" carries a trailing quote and the compare fails.
+func hidesOutputSummary(command string) bool {
+ rest := unquote(command)
for {
at := strings.Index(rest, hideSummaryFlag)
if at < 0 {
@@ -88,22 +169,65 @@ func hidesOutputSummary(line string) bool {
}
}
-func isGotestsumInvocation(line string) bool {
- trimmed := strings.TrimSpace(line)
+func unquote(s string) string {
+ return strings.NewReplacer(`"`, "", `'`, "").Replace(s)
+}
+
+// isGotestsumInvocation asks whether the command RUNS the tool. A line that merely
+// mentions the flag — prose, or an echo explaining it — looks identical once quotes are
+// stripped, so an echoed mention is excluded explicitly.
+func isGotestsumInvocation(command string) bool {
+ normalised := unquote(command)
// Case-insensitive: scripts/test.sh invokes the tool through "$GOTESTSUM_BIN", which a
// lowercase-only match misses entirely.
- if strings.HasPrefix(trimmed, "#") || !strings.Contains(strings.ToLower(trimmed), gotestsumBinary) {
+ if !strings.Contains(strings.ToLower(normalised), gotestsumBinary) {
return false
}
- return strings.Contains(trimmed, "--format=") || strings.Contains(trimmed, hideSummaryFlag)
+ if !strings.Contains(normalised, "--format=") && !strings.Contains(normalised, hideSummaryFlag) {
+ return false
+ }
+ return !printsRatherThanRuns(normalised)
+}
+
+func printsRatherThanRuns(command string) bool {
+ for _, field := range strings.Fields(command) {
+ switch field {
+ case "echo", "printf":
+ return true
+ case gotestsumBinary:
+ return false
+ }
+ if strings.Contains(strings.ToLower(field), gotestsumBinary) {
+ return false
+ }
+ }
+ return false
+}
+
+// scannableForInvocations accepts a file by SHAPE rather than by name: the shell and
+// workflow extensions, plus anything carrying a shebang. The previous version special-cased
+// one extensionless runner by base name, which is itself the evidence that a second one is
+// writable — a Makefile-style runner or scripts/citest would have been invisible.
+func scannableForInvocations(path, body string) bool {
+ if hasScannableExtension(path) {
+ return true
+ }
+ return strings.HasPrefix(body, "#!")
}
-func scannableForInvocations(path string) bool {
+func hasScannableExtension(path string) bool {
switch filepath.Ext(path) {
case ".yml", ".yaml", ".sh":
return true
}
- return filepath.Base(path) == "dev"
+ return false
+}
+
+// mayHoldInvocations keeps the walk from reading every file in the tree: a shebang can only
+// be found by reading, so extensionless files are read, and everything with an extension we
+// do not scan is skipped unread.
+func mayHoldInvocations(path string) bool {
+ return hasScannableExtension(path) || filepath.Ext(path) == ""
}
func skipScanDir(name string) bool {
@@ -133,3 +257,109 @@ func repoRoot(t *testing.T) string {
dir = parent
}
}
+
+// TestTheGotestsumGuardReadsCommandsNotLines is the acceptance for the two evasions that
+// ran correctly while the per-line scan stayed green — a quoted value and a flag on a
+// backslash continuation — plus the shapes that must NOT red, which is the risk of asking
+// the question at a coarser level. The already-caught forms are here as controls, not as
+// evidence: they passed before this change too.
+func TestTheGotestsumGuardReadsCommandsNotLines(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ offending bool
+ wantLine int
+ body string
+ }{
+ {"double-quoted value", true, 1, `gotestsum --format=pkgname --hide-summary="skipped,output" -- ./...`},
+ {"single-quoted value", true, 1, `gotestsum --format=pkgname --hide-summary='output' -- ./...`},
+ {"backslash continuation in a YAML run block", true, 6, `jobs:
+ test:
+ steps:
+ - name: Test
+ run: |
+ gotestsum --format=pkgname \
+ --hide-summary=output \
+ -- -p 1 ./...`},
+ {"continuation in a shell script", true, 2, `#!/usr/bin/env bash
+"$GOTESTSUM_BIN" --format=pkgname \
+ --hide-summary=output \
+ --jsonfile "$UNIT_JSON" -- ./...`},
+ {"bare output", true, 1, `gotestsum --format=pkgname --hide-summary=output -- ./...`},
+ {"reordered list", true, 1, `gotestsum --format=pkgname --hide-summary=output,skipped -- ./...`},
+ {"two flags on one command", true, 1, `gotestsum --format=pkgname --hide-summary=skipped --hide-summary=output -- ./...`},
+ {"a legal invocation", false, 0, `gotestsum --format=pkgname --hide-summary=skipped -- ./...`},
+ {"no hide-summary at all", false, 0, `gotestsum --format=pkgname -- -p 1 ./...`},
+ {"commented-out invocation", false, 0, `# gotestsum --format=pkgname --hide-summary=output -- ./...`},
+ {"commented-out invocation spanning a continuation", false, 0, `# gotestsum --format=pkgname \
+# --hide-summary=output \
+# -- ./...`},
+ {"a comment ends a pending continuation", false, 0, `gotestsum --format=pkgname \
+# --hide-summary=output
+`},
+ {"an echoed mention", false, 0, `echo "gotestsum --format=pkgname --hide-summary=output is banned"`},
+ {"a printf mention", false, 0, `printf 'run gotestsum --hide-summary=output to hide it\n'`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, offences := scanGotestsumCommands(tc.body)
+ if tc.offending {
+ if len(offences) != 1 {
+ t.Fatalf("offences = %v, want exactly one: this form runs correctly and hides the failure cause", offences)
+ }
+ if offences[0].line != tc.wantLine {
+ t.Errorf("reported line %d, want %d — the report must point at the line the invocation STARTS on, not an index into the joined text", offences[0].line, tc.wantLine)
+ }
+ return
+ }
+ if len(offences) != 0 {
+ t.Fatalf("offences = %v, want none", offences)
+ }
+ })
+ }
+}
+
+// The invocation count feeds the vacuity floor, so a shape that stops being recognised as
+// an invocation would let the ban scan nothing while the floor still passed.
+func TestTheGotestsumGuardCountsBothRealInvocationShapes(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ want int
+ body string
+ }{
+ {"workflow run:", 1, ` run: gotestsum --format=pkgname -- -p 1 ./...`},
+ {"script through an uppercase variable", 1, ` if ! "$GOTESTSUM_BIN" --format=pkgname --hide-summary=skipped -- ./...; then`},
+ {"a mention is not an invocation", 0, ` echo "gotestsum --format=pkgname not found"`},
+ {"resolution helper without flags", 0, ` if command -v gotestsum >/dev/null 2>&1; then`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ invocations, _ := scanGotestsumCommands(tc.body)
+ if invocations != tc.want {
+ t.Errorf("invocations = %d, want %d", invocations, tc.want)
+ }
+ })
+ }
+}
+
+// The file-shape axis: a runner is recognised by carrying a shebang, not by being named
+// "dev". A second extensionless runner used to be invisible.
+func TestAnExtensionlessRunnerIsScannedByItsShebang(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ path string
+ body string
+ scannable bool
+ }{
+ {"the repo's own runner", "dev", "#!/usr/bin/env bash\n", true},
+ {"a future extensionless runner", "scripts/citest", "#!/bin/sh\n", true},
+ {"a Makefile-style runner with a shebang", "Makefile", "#!/usr/bin/make -f\n", true},
+ {"an extensionless file with no shebang", "LICENSE", "MIT License\n", false},
+ {"a shell script", "scripts/test.sh", "set -e\n", true},
+ {"a workflow", ".github/workflows/go.yml", "name: go\n", true},
+ {"documentation", "docs/guides/contributing.md", "# gotestsum --hide-summary=output\n", false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := scannableForInvocations(tc.path, tc.body); got != tc.scannable {
+ t.Errorf("scannableForInvocations(%q) = %v, want %v", tc.path, got, tc.scannable)
+ }
+ })
+ }
+}