Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

fix(khaos): the seven findings an outside audit could see and this project could not

CarterPerez-dev9c4d850c4bc1
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
index 69ea2fd0..fe04e0ce 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
@@ -9,6 +9,7 @@ import (
"fmt"
"os"
"os/signal"
+ "runtime/debug"
"strconv"
"strings"
"syscall"
@@ -36,6 +37,11 @@ const (
noColorFlagTrue = noColorFlag + "="
errorFormat = "%s: %v\n"
fixFormat = " fix: %s\n"
+ panicFormat = " cause: %v\n"
+)
+
+var internalFailure = errors.New(
+ "khaos hit a bug and stopped; every fault it had applied was reverted on the way out",
)
var errInterrupted = errors.New(
@@ -43,7 +49,34 @@ var errInterrupted = errors.New(
)
func main() {
- os.Exit(execute())
+ os.Exit(guarded(execute))
+}
+
+// guarded is the boundary that makes CodeInternal reachable. The taxonomy
+// published code 70 as "a recovered panic" and nothing in khaos recovered
+// one, so a bug crashed with a Go stack trace and whatever exit status the
+// runtime chose -- indistinguishable to CI from a tool that could not parse
+// its flags.
+//
+// Recovering HERE and nowhere deeper is deliberate: the panic still unwinds
+// every frame below, so every deferred revert, lease release and lock
+// handoff runs exactly as it does on any other error path. khaos only
+// decides what to say about it.
+func guarded(run func() int) (code int) {
+ defer func() {
+ recovered := recover()
+ if recovered == nil {
+ return
+ }
+
+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, internalFailure)
+ fmt.Fprintf(os.Stderr, panicFormat, recovered)
+ fmt.Fprint(os.Stderr, string(debug.Stack()))
+
+ code = cli.CodeInternal
+ }()
+
+ return run()
}
func execute() int {
@@ -110,36 +143,30 @@ func noColorRequested(args []string) bool {
// verdict outcomes is decided once where the run ends rather than re-derived
// by a chain here that grows with every epistemic value. The checks below it
// are for errors that never reached a verdict at all.
+// fail decides what the operator READS. What the process RETURNS is decided by
+// cli.ExitCode, which is the same classification the run record carries, so
+// the status and the artifact cannot disagree. This function used to carry its
+// own table, and a rollback failure exited 6 while run.json said 1.
func fail(err error) int {
- if errors.Is(err, run.ErrRevertFailed) {
- fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
- printFix(err)
- return exitRevertFailed
- }
-
- // Checked before the carried code as well as inside it: this is the one
- // outcome nothing may suppress, so it is matched on the sentinel too and
- // not only on a code some other layer computed.
- if errors.Is(err, cli.ErrNotRecovered) {
- fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
- return exitRevertFailed
- }
-
- var exit *cli.Exit
- if errors.As(err, &exit) {
- fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
- return exit.Code
- }
+ code := cli.ExitCode(err)
+ // Cancellation gets its own wording, because "context canceled" is Go's
+ // phrasing for a decision the operator made.
if errors.Is(err, context.Canceled) {
fmt.Fprintf(os.Stderr, errorFormat, version.Name, errInterrupted)
- return exitInterrupted
+
+ return code
}
fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
- printFix(err)
- return exitError
+ // The fix line is for a refusal that names one, which is every refusal
+ // predicate and every unrevertable fault.
+ if errors.Is(err, run.ErrRevertFailed) || code == exitError {
+ printFix(err)
+ }
+
+ return code
}
func printFix(err error) {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
index 48e0ce9c..d19c13fc 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
@@ -205,3 +205,51 @@ func TestRollbackFailureOutranksEveryOtherExitCode(t *testing.T) {
})
}
}
+
+// Exit code 70 was published in the taxonomy as "a recovered panic" and
+// nothing in khaos recovered one: `CodeInternal` appeared in its own
+// declaration and in a numeric assertion, and in no production path at all.
+// A bug therefore crashed with a Go stack trace and whatever status the
+// runtime chose, which CI cannot tell from a tool that could not parse its
+// flags.
+//
+// The deferred cleanup below is the half that matters more than the code: the
+// panic still unwinds every frame, so a fault khaos had applied is still
+// reverted on the way out.
+func TestAPanicBecomesTheInternalCodeWithCleanupIntact(t *testing.T) {
+ reverted := false
+
+ code := guarded(func() int {
+ defer func() { reverted = true }()
+
+ panic("a bug in khaos")
+ })
+
+ if code != cli.CodeInternal {
+ t.Errorf(
+ "a recovered panic exited %d, want %d: the taxonomy publishes 70 for exactly this",
+ code,
+ cli.CodeInternal,
+ )
+ }
+
+ if !reverted {
+ t.Error(
+ "a deferred revert did not run while the panic unwound, so khaos would leave the fault applied",
+ )
+ }
+}
+
+// And the boundary is not a blanket: an ordinary run's code passes through
+// untouched, or every exit in the taxonomy would collapse to 70.
+func TestTheGuardLeavesAnOrdinaryExitAlone(t *testing.T) {
+ for _, want := range []int{
+ cli.CodeOK,
+ cli.CodeDefect,
+ cli.CodeRollback,
+ } {
+ if got := guarded(func() int { return want }); got != want {
+ t.Errorf("guarded changed exit %d into %d", want, got)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go
index e95fab45..7517540d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go
@@ -4,12 +4,14 @@
package cli
import (
+ "context"
"errors"
"fmt"
"io"
"github.com/CarterPerez-dev/khaos/internal/experiment"
"github.com/CarterPerez-dev/khaos/internal/record"
+ "github.com/CarterPerez-dev/khaos/internal/run"
"github.com/CarterPerez-dev/khaos/internal/version"
"github.com/spf13/cobra"
)
@@ -124,9 +126,13 @@ func emitRecord(
return outcome
}
- // A verdict khaos reached outranks a file it could not write.
+ // A verdict khaos reached outranks a file it could not write. The warning
+ // goes to the DIAGNOSTIC stream: writing it to out appended prose to the
+ // JSON document under --output json, which breaks that contract on the one
+ // path where CI needs machine-readable evidence.
if outcome != nil {
- fmt.Fprintf(out, warnRecordWrite, writeErr)
+ fmt.Fprintf(emit.diagnostic(), warnRecordWrite, writeErr)
+
return outcome
}
@@ -163,15 +169,41 @@ func writeProjections(rec record.Record, into Projections) error {
// exitDetail reads the code and the outcome name back off a finished run, so
// the record can carry what khaos is about to return to the shell.
+//
+// It is the SAME classification the process boundary uses, because these were
+// two lists: this one mapped anything without a typed Exit to usage code 1
+// while the boundary recognised the rollback and cancellation sentinels on its
+// own, so a real rollback failure wrote exit.code 1 into run.json and exited 6.
func exitDetail(outcome error) (int, string) {
if outcome == nil {
return CodeOK, ""
}
+ // Matched on the sentinels BEFORE any carried code, because these are the
+ // outcomes nothing may suppress and they must not depend on some other
+ // layer having wrapped them correctly.
+ if errors.Is(outcome, run.ErrRevertFailed) ||
+ errors.Is(outcome, ErrNotRecovered) {
+ return CodeRollback, string(outcomeRollback)
+ }
+
var exit *Exit
if errors.As(outcome, &exit) {
return exit.Code, string(exit.Outcome)
}
+ if errors.Is(outcome, context.Canceled) {
+ return CodeInterrupted, string(OutcomeInterrupted)
+ }
+
return CodeUsage, ""
}
+
+// ExitCode is the one authority on what a finished run returns to the shell.
+// The process boundary reads it rather than carrying its own table, so the
+// status and the record cannot disagree.
+func ExitCode(outcome error) int {
+ code, _ := exitDetail(outcome)
+
+ return code
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go
index 30d03e2b..85dfdf8e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go
@@ -180,6 +180,7 @@ func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {
}
out := &strings.Builder{}
+ diagnostic := &strings.Builder{}
err := emitRecord(
out,
@@ -187,8 +188,9 @@ func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {
result,
defect,
Output{
- Shape: outputText,
- Record: filepath.Join("/proc", "khaos-cannot-write-here"),
+ Shape: outputText,
+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),
+ Diagnostic: diagnostic,
},
)
@@ -197,9 +199,21 @@ func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {
}
// And the failure is still SAID, because an operator who asked for a
- // record and did not get one must not have to guess.
- if !strings.Contains(out.String(), "could not write the run record") {
- t.Fatalf("the write failure was silent: %.80q", out.String())
+ // record and did not get one must not have to guess. On the DIAGNOSTIC
+ // stream, not the report stream: the same line on stdout appended prose to
+ // the JSON document under --output json.
+ if !strings.Contains(
+ diagnostic.String(),
+ "could not write the run record",
+ ) {
+ t.Fatalf("the write failure was silent: %.80q", diagnostic.String())
+ }
+
+ if strings.Contains(out.String(), "could not write") {
+ t.Fatalf(
+ "the write failure reached the report stream, which is what breaks --output json: %.80q",
+ out.String(),
+ )
}
}
@@ -432,3 +446,60 @@ func TestTheReportingFlagsAdvertiseWhatTheyTake(t *testing.T) {
}
}
}
+
+// --output json promises stdout carries the record and nothing else, so
+// `khaos run x --output json | jq .verdict` works. A write failure broke that
+// promise on exactly the path CI needs most: the JSON went to `out`, and so
+// did the prose warning about the artifact khaos could not write, so a defect
+// plus an invalid --record destination produced a JSON document followed by a
+// sentence. Diagnostics belong on stderr; the machine format gets the stream
+// it was asked for.
+func TestAnArtifactFailureNeverContaminatesTheJSONStream(t *testing.T) {
+ result, definition := emitted(t)
+
+ out := &strings.Builder{}
+ diagnostic := &strings.Builder{}
+
+ outcome := &Exit{
+ Code: CodeDefect,
+ Outcome: OutcomeDefect,
+ Err: ErrDefectFound,
+ }
+
+ err := emitRecord(
+ out,
+ definition,
+ result,
+ outcome,
+ Output{
+ Shape: outputJSON,
+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),
+ Diagnostic: diagnostic,
+ },
+ )
+
+ // The verdict outranks the bookkeeping: the defect still comes back.
+ if !errors.Is(err, ErrDefectFound) {
+ t.Fatalf("the defect was replaced by the write failure: %v", err)
+ }
+
+ body := out.String()
+
+ var parsed record.Record
+ if jsonErr := json.Unmarshal([]byte(body), &parsed); jsonErr != nil {
+ t.Fatalf(
+ "stdout is not a single decodable JSON document, so the --output json contract is broken on the path CI depends on: %v\nstdout was: %q",
+ jsonErr,
+ body,
+ )
+ }
+
+ // And the failure is not swallowed: in CI the operator believes they have
+ // an artifact, so the warning has to land somewhere they will see it.
+ if !strings.Contains(diagnostic.String(), "could not write") {
+ t.Fatalf(
+ "the artifact failure was not reported on the diagnostic stream: %q",
+ diagnostic.String(),
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go
index 78e21e0d..309c42b0 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go
@@ -4,9 +4,13 @@
package cli
import (
+ "context"
"errors"
+ "fmt"
+ "io"
"slices"
"testing"
+ "time"
"github.com/CarterPerez-dev/khaos/internal/experiment"
"github.com/CarterPerez-dev/khaos/internal/run"
@@ -441,3 +445,100 @@ func TestTheExitCodesAreTheirPublishedNumbers(t *testing.T) {
}
}
}
+
+// The code in the record and the code the process returns are the SAME
+// contract, and they were computed by two different lists.
+//
+// exitFor returned an execution error unclassified, exitDetail mapped anything
+// without a typed Exit to usage code 1, and the process boundary separately
+// recognised the rollback and cancellation sentinels. So a real rollback
+// failure wrote `exit.code: 1` into run.json while the process exited 6 -- the
+// artifact disagreeing with the status about the one outcome no --fail-on value
+// may suppress. The unit census never caught it because it called exitFor with
+// a nil execution error.
+func TestTheRecordedCodeIsTheCodeTheProcessReturns(t *testing.T) {
+ for name, tc := range map[string]struct {
+ err error
+ want int
+ }{
+ "a fault that could not be reverted": {
+ err: run.ErrRevertFailed,
+ want: CodeRollback,
+ },
+ "a target that never recovered": {
+ err: ErrNotRecovered,
+ want: CodeRollback,
+ },
+ "an interrupted run": {
+ err: context.Canceled,
+ want: CodeInterrupted,
+ },
+ "an ordinary execution error": {
+ err: errors.New("the daemon refused"),
+ want: CodeUsage,
+ },
+ "a wrapped rollback failure": {
+ err: fmt.Errorf("reverting: %w", run.ErrRevertFailed),
+ want: CodeRollback,
+ },
+
+ // errors.Join is where the precedence actually bites: the state khaos
+ // left the target in outranks the verdict about it, and the RECORD has
+ // to agree with that, not just the status.
+ "a rollback failure joined with a defect": {
+ err: errors.Join(ErrDefectFound, run.ErrRevertFailed),
+ want: CodeRollback,
+ },
+ "a rollback failure joined with cancellation": {
+ err: errors.Join(context.Canceled, run.ErrRevertFailed),
+ want: CodeRollback,
+ },
+ "a carried defect code joined with a rollback failure": {
+ err: errors.Join(
+ &Exit{Code: CodeDefect, Err: ErrDefectFound},
+ run.ErrRevertFailed,
+ ),
+ want: CodeRollback,
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ outcome := exitFor(held(), tc.err)
+
+ if got := ExitCode(outcome); got != tc.want {
+ t.Errorf(
+ "the process would exit %d",
+ got,
+ )
+ }
+
+ if got, _ := exitDetail(outcome); got != tc.want {
+ t.Errorf(
+ "the record would say exit.code %d, want %d: an artifact that disagrees with the status is worse than no artifact",
+ got,
+ tc.want,
+ )
+ }
+ })
+ }
+}
+
+// `reap --interval 0` reached time.NewTicker(0), which panics, and it got
+// there AFTER the workspace was opened and its lock taken -- so the operator
+// paid for a refusal that arrived as a Go stack trace. Validated before
+// anything is opened.
+func TestANonPositiveReapIntervalIsRefusedBeforeAnythingOpens(t *testing.T) {
+ for _, interval := range []time.Duration{
+ 0,
+ -time.Second,
+ } {
+ err := runReap(context.Background(), io.Discard, interval)
+
+ if !errors.Is(err, ErrIntervalNotPositive) {
+ t.Fatalf(
+ "reap --interval %s returned %v, want a refusal naming the interval",
+ interval,
+ err,
+ )
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index 0dc05dfd..fe133e63 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -339,6 +339,7 @@ func newRunExperimentCmd(definition experiment.Definition) *cobra.Command {
Shape: shape,
Record: into,
Projections: projections,
+ Diagnostic: cmd.ErrOrStderr(),
},
)
},
@@ -650,6 +651,13 @@ type Output struct {
Shape string
Record string
Projections Projections
+
+ // Diagnostic is where a failure that is NOT the verdict goes. It is
+ // separate from the report stream because a warning on stdout turns the
+ // --output json contract into a JSON document followed by a sentence, and
+ // it is separate from the report because under json the report is
+ // discarded while this must still be seen.
+ Diagnostic io.Writer
}
// human is where the terminal report goes, and under --output json that is
@@ -663,6 +671,16 @@ func (o Output) human(out io.Writer) io.Writer {
return out
}
+// diagnostic defaults to stderr rather than to the report stream, so a caller
+// that forgets to set it cannot silently reintroduce the contamination.
+func (o Output) diagnostic() io.Writer {
+ if o.Diagnostic == nil {
+ return os.Stderr
+ }
+
+ return o.Diagnostic
+}
+
func runExperiment(
ctx context.Context,
out io.Writer,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go
index 99b6575a..5a91887e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go
@@ -5,6 +5,7 @@ package cli
import (
"context"
+ "errors"
"fmt"
"io"
"time"
@@ -53,6 +54,15 @@ every other fault on the machine.`
msgWatching = "watching %s every %s, until you stop it"
msgStopped = "stopped watching, %d fault(s) reclaimed"
msgDamaged = "the journal has a damaged line; khaos acted on every record it could read, but cannot promise that is all of them"
+
+ errIntervalUsage = "%w: %s"
+)
+
+// A non-positive interval reached time.NewTicker, which panics, AFTER the
+// workspace had been opened and its lock taken. Refused by name instead, and
+// before anything is opened.
+var ErrIntervalNotPositive = errors.New(
+ "the reap interval has to be longer than zero, or there is no tick to check the journal on",
)
func newReclaimCmd() *cobra.Command {
@@ -125,6 +135,12 @@ func runReclaim(ctx context.Context, out io.Writer, expiredOnly bool) error {
}
func runReap(ctx context.Context, out io.Writer, interval time.Duration) error {
+ // Before the workspace, because a refusal that has already taken the lock
+ // is a refusal that cost the operator something.
+ if interval <= 0 {
+ return fmt.Errorf(errIntervalUsage, ErrIntervalNotPositive, interval)
+ }
+
work, err := workspace.OpenShared(ctx)
if err != nil {
return err
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
index 14588af1..782dd8bb 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go
@@ -616,3 +616,86 @@ func TestARequiredForgedStreamWithNoDecisionsCannotReportHeld(t *testing.T) {
verdict.Epistemic, Passed, strings.Join(verdict.Reasons, "; "))
}
}
+
+// The guard that requires this experiment's own stream to have been MEASURED
+// counted transport observations as decisions.
+//
+// `Countable()` excludes only khaos's own failures, so a timeout, an
+// unreachable target and a throttle all satisfied a check whose own name and
+// comment say "the target actually decided about it". Thirty forged probes that
+// timed out therefore cleared the floor, and a healthy valid/invalid pair then
+// produced a CONCLUSIVE hold about a credential the target was never observed
+// to judge. `Answered()` is the discriminator -- the same one that split
+// `Unavailable` from `Unreachable` -- and this was the fourth site reading the
+// wrong one.
+func TestATimedOutForgedStreamIsNotEvidenceTheTargetDecided(t *testing.T) {
+ for _, disposition := range []probe.Disposition{
+ probe.Unreachable,
+ probe.Unavailable,
+ } {
+ t.Run(string(disposition), func(t *testing.T) {
+ observation := forgeryObservation(
+ window(
+ probe.Accepted,
+ probe.Rejected,
+ probe.Accepted,
+ "",
+ MinimumPaired,
+ ),
+ window(
+ probe.Accepted,
+ probe.Rejected,
+ disposition,
+ "",
+ MinimumPaired,
+ ),
+ probe.ExpectAccepted,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic != ProbeError {
+ t.Fatalf(
+ "verdict = %s / %s conclusive=%v, want %s: every forged probe was a %s, so the target was never seen to decide about the credential this experiment is about (%s)",
+ verdict.Epistemic,
+ verdict.Direction,
+ verdict.Conclusive,
+ ProbeError,
+ disposition,
+ strings.Join(verdict.Reasons, "; "),
+ )
+ }
+ })
+ }
+}
+
+// The other half, because a guard that rejects everything is not a guard: a
+// forged stream the target actually DECIDED about still clears the floor.
+func TestADecidedForgedStreamStillClearsTheFloor(t *testing.T) {
+ observation := forgeryObservation(
+ window(
+ probe.Accepted,
+ probe.Rejected,
+ probe.Accepted,
+ "",
+ MinimumPaired,
+ ),
+ window(
+ probe.Accepted,
+ probe.Rejected,
+ probe.Rejected,
+ "",
+ MinimumPaired,
+ ),
+ probe.ExpectAccepted,
+ )
+
+ verdict := Judge(observation, idpPolarity)
+
+ if verdict.Epistemic == ProbeError {
+ t.Fatalf(
+ "a forged stream the target refused every time was reported unmeasurable: %s",
+ strings.Join(verdict.Reasons, "; "),
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
index 7e12d50b..258a2cfc 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
@@ -776,7 +776,11 @@ func requiredStreamThin(observation Observation) (string, bool) {
return "", false
}
- if decided := observation.Fault.Forged.Countable(); decided < MinimumPaired {
+ // Answered(), not Countable(): a timeout or an unreachable target is a
+ // transport observation and not a decision about the credential, and
+ // Countable() excludes only khaos's own failures. This is the same
+ // discriminator that splits Unavailable from Unreachable.
+ if decided := observation.Fault.Forged.Answered(); decided < MinimumPaired {
return sprintf(reasonThinForged, decided, MinimumPaired), true
}
@@ -1164,25 +1168,49 @@ func unanswered(
// recoveryOf requires a run of consecutive healthy pairs, because a state
// transition declared on one lucky sample is how a flaky recovery reads as a
// clean one. An empty window is UNOBSERVED, never recovered.
+//
+// The whole window is read, because this used to return the moment it found a
+// qualifying run: two healthy pairs followed by twenty broken ones was
+// RECOVERED, and a run whose last observed state was broken could exit 0.
+//
+// Recovery therefore needs two things -- it has to have been ESTABLISHED, and
+// it must not have collapsed for a sustained stretch afterwards. The same run
+// length governs both, because the reason a single healthy pair cannot declare
+// recovery is the reason a single failed one cannot withdraw it. A window of
+// fifteen pairs whose last sample happens to be a blip is still a recovery;
+// this verdict maps to the one exit code no --fail-on value can suppress, so
+// the cost of declaring it on noise lands on every later job.
func recoveryOf(recovery probe.Window, closed []probe.Disposition) Recovery {
if recovery.Paired() == 0 {
return Unobserved
}
- run := 0
+ healthy := 0
+ established := false
+ collapsed := 0
+
for i := 0; i < recovery.Valid.Len() && i < recovery.Invalid.Len(); i++ {
if recovery.Valid.Outcomes[i].Disposition == probe.Accepted &&
shutBy(
recovery.Invalid.Outcomes[i].Disposition,
decided(closed),
) {
- run++
- if run >= RecoveryRunLength {
- return Recovered
+ healthy++
+ collapsed = 0
+
+ if healthy >= RecoveryRunLength {
+ established = true
}
+
continue
}
- run = 0
+
+ healthy = 0
+ collapsed++
+ }
+
+ if established && collapsed < RecoveryRunLength {
+ return Recovered
}
return Degraded
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
index f621a863..a6f88c3c 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go
@@ -546,3 +546,111 @@ func TestADeclaredClosureNeverHidesARefusal(t *testing.T) {
)
}
}
+
+// recoveryOf returned the moment it found its run of consecutive healthy pairs
+// and never looked at the rest of the window, so a recovery that collapsed
+// after two good samples was reported RECOVERED. A run whose last observed
+// state was broken could then exit 0.
+//
+// The criterion is now the END of the window rather than the first qualifying
+// run: recovery has to still be established when khaos stops looking. A late
+// recovery still counts, because a window that ends with a full healthy run
+// ends healthy however it started.
+func TestRecoveryIsJudgedAtTheEndOfTheWindowNotTheFirstGoodRun(t *testing.T) {
+ healthy := func(n int) []probe.Disposition {
+ out := make([]probe.Disposition, n)
+ for i := range out {
+ out[i] = probe.Accepted
+ }
+ return out
+ }
+
+ for name, tc := range map[string]struct {
+ valid []probe.Disposition
+ want Recovery
+ }{
+ "healthy throughout": {
+ valid: healthy(6),
+ want: Recovered,
+ },
+ "two good pairs then collapse": {
+ valid: []probe.Disposition{
+ probe.Accepted,
+ probe.Accepted,
+ probe.Unreachable,
+ probe.Unreachable,
+ probe.Unreachable,
+ probe.Unreachable,
+ },
+ want: Degraded,
+ },
+ "broken first then a full healthy run": {
+ valid: []probe.Disposition{
+ probe.Unreachable,
+ probe.Unreachable,
+ probe.Accepted,
+ probe.Accepted,
+ probe.Accepted,
+ },
+ want: Recovered,
+ },
+ "alternating never settles": {
+ valid: []probe.Disposition{
+ probe.Accepted,
+ probe.Unreachable,
+ probe.Accepted,
+ probe.Unreachable,
+ probe.Accepted,
+ },
+ want: Degraded,
+ },
+ "recovers only on the very last pair": {
+ valid: []probe.Disposition{
+ probe.Unreachable,
+ probe.Unreachable,
+ probe.Unreachable,
+ probe.Accepted,
+ },
+ want: Degraded,
+ },
+ // The brittleness guard. A single trailing blip in a long healthy
+ // window must NOT withdraw a recovery: this verdict maps to the one
+ // exit code no --fail-on value can suppress, and declaring it on one
+ // unlucky last sample makes every later job on that runner suspect.
+ "one trailing blip after a long healthy run": {
+ valid: append(healthy(12), probe.Unreachable),
+ want: Recovered,
+ },
+ "two trailing blips are a sustained collapse": {
+ valid: append(
+ healthy(12),
+ probe.Unreachable,
+ probe.Unreachable,
+ ),
+ want: Degraded,
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ recovery := probe.Window{}
+
+ for _, disposition := range tc.valid {
+ at := time.Now().UTC()
+ recovery.Record(probe.Outcome{
+ At: at,
+ Credential: probe.Valid,
+ Disposition: disposition,
+ })
+ recovery.Record(probe.Outcome{
+ At: at,
+ Credential: probe.Invalid,
+ Disposition: probe.Rejected,
+ })
+ }
+
+ got := recoveryOf(recovery, nil)
+ if got != tc.want {
+ t.Fatalf("recovery = %s, want %s", got, tc.want)
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go
index 6840bec0..fceb86d1 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go
@@ -190,23 +190,44 @@ func warnings(from []experiment.Warning) []string {
return out
}
+// Phase is one measured window under the name the record calls it by.
+type Phase struct {
+ Name string
+ Window probe.Window
+}
+
+// Phases is the ONE enumeration of the measured windows, in the order the
+// engine measures them. The aggregate serializer and the raw-event serializer
+// each used to carry their own list and each listed four of the five: `shift`,
+// the second fault window the authorization tier needs and the verdict reads
+// at three sites, was in neither. A defect established from it was written to
+// a record that did not contain the observations establishing it, with an
+// events hash and count that were internally consistent about the wrong set.
+//
+// Both serializers read this, so a phase added to the engine cannot go missing
+// from one of them.
+func Phases(observation experiment.Observation) []Phase {
+ return []Phase{
+ {"baseline", observation.Baseline},
+ {"fault", observation.Fault},
+ {"shift", observation.Shift},
+ {"resumed", observation.Resumed},
+ {"recovery", observation.Recovery},
+ }
+}
+
// windows omits a window that was never measured rather than recording it as
// zero: "khaos did not get here" and "khaos measured nothing here" are
// different facts, and a reader cannot tell them apart from a zero.
func windows(observation experiment.Observation) map[string]Window {
out := map[string]Window{}
- for name, window := range map[string]probe.Window{
- "baseline": observation.Baseline,
- "fault": observation.Fault,
- "recovery": observation.Recovery,
- "resumed": observation.Resumed,
- } {
- if window.Len() == 0 {
+ for _, phase := range Phases(observation) {
+ if phase.Window.Len() == 0 {
continue
}
- out[name] = aggregate(window)
+ out[phase.Name] = aggregate(phase.Window)
}
return out
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go
index 7dabc7ab..97c4c2c1 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go
@@ -61,8 +61,17 @@ func finished() experiment.Result {
StartedAt: base,
Duration: 30 * time.Second,
Observation: experiment.Observation{
- Baseline: window(0, probe.Rejected),
- Fault: window(10*time.Second, probe.Accepted),
+ Baseline: window(0, probe.Rejected),
+ Fault: window(10*time.Second, probe.Accepted),
+
+ // Shift and Resumed are the two windows this fixture used to
+ // claim it carried and did not. Both are real measured phases --
+ // shift is the second fault window the authorization tier needs,
+ // and resumed is what the idleness discriminator reads -- and
+ // leaving them empty is why a serializer that dropped one passed.
+ Shift: window(15*time.Second, probe.Accepted),
+ Shifted: true,
+ Resumed: window(18*time.Second, probe.Rejected),
Recovery: window(20*time.Second, probe.Rejected),
FaultConfirmed: true,
ConfirmMethod: "a measured observable",
@@ -215,9 +224,14 @@ func TestEveryObservationReachesTheEventsFile(t *testing.T) {
// Requests, not Len: Len is Paired, and one paired sample is at least
// two requests. Counting pairs here would have asserted half the
// observations reached the file and called that complete.
- want := result.Observation.Baseline.Requests() +
- result.Observation.Fault.Requests() +
- result.Observation.Recovery.Requests()
+ //
+ // Summed over Phases rather than over a list written here, because a
+ // hardcoded list of three windows is what let the shift window go
+ // unnoticed in both serializers.
+ want := 0
+ for _, phase := range Phases(result.Observation) {
+ want += phase.Window.Requests()
+ }
if len(events) != want {
t.Fatalf("wrote %d events for %d observations", len(events), want)
@@ -516,3 +530,63 @@ func inodeOf(path string) (uint64, error) {
var errNoInode = errors.New(
"this platform does not expose an inode, so the rename cannot be distinguished from a truncation here",
)
+
+// The record is canonical and every other output is a projection of it, so a
+// window the VERDICT reads must survive serialization. The shift window did
+// not: both the aggregate and the raw-event serializers enumerated their
+// phases separately and both listed four of the five, so a defect established
+// from the second fault window was saved without the observations that
+// established it -- with an events hash and count that were internally
+// consistent about the wrong set.
+func TestEveryMeasuredPhaseSurvivesTheRecord(t *testing.T) {
+ result := finished()
+
+ for name, window := range map[string]probe.Window{
+ "baseline": result.Observation.Baseline,
+ "fault": result.Observation.Fault,
+ "shift": result.Observation.Shift,
+ "resumed": result.Observation.Resumed,
+ "recovery": result.Observation.Recovery,
+ } {
+ if window.Len() == 0 {
+ t.Fatalf(
+ "the fixture does not populate %q, so this test cannot see it dropped",
+ name,
+ )
+ }
+ }
+
+ rec := Of("1.2.3", result, definition(t), 2, "defect")
+
+ for _, name := range []string{
+ "baseline",
+ "fault",
+ "shift",
+ "resumed",
+ "recovery",
+ } {
+ if _, ok := rec.Windows[name]; !ok {
+ t.Errorf(
+ "the record has no %q window, so a verdict resting on it cannot be audited from the record",
+ name,
+ )
+ }
+ }
+
+ seen := map[string]int{}
+ for _, event := range EventsOf(result.Observation) {
+ seen[event.Window]++
+ }
+
+ for _, name := range []string{
+ "baseline",
+ "fault",
+ "shift",
+ "resumed",
+ "recovery",
+ } {
+ if seen[name] == 0 {
+ t.Errorf("events.jsonl carries no %q observation", name)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go
index db1ac666..c66ce763 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go
@@ -179,16 +179,8 @@ func writeAtomic(path string, body []byte) error {
func EventsOf(observation experiment.Observation) []Event {
out := []Event{}
- for _, named := range []struct {
- name string
- window probe.Window
- }{
- {"baseline", observation.Baseline},
- {"fault", observation.Fault},
- {"resumed", observation.Resumed},
- {"recovery", observation.Recovery},
- } {
- out = append(out, eventsOfWindow(named.name, named.window)...)
+ for _, phase := range Phases(observation) {
+ out = append(out, eventsOfWindow(phase.Name, phase.Window)...)
}
return out
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go
index dd51e4cb..026ddf72 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go
@@ -51,6 +51,8 @@ const (
defaultTruncate = 4
+ tempPrefix = ".khaos-holding-"
+
describeEmpty = "truncate %s to zero bytes in place, which is the one secret fault that is not an error at the syscall level: os.ReadFile returns no error and an empty slice, so every error-handling path in the target is bypassed"
describeTruncated = "cut %s down to a few bytes in place, which leaves a silently WRONG key rather than a missing one"
@@ -182,7 +184,7 @@ func (m Mutation) Apply() error {
return os.Truncate(m.Path, m.keep(info.Size()))
case Unreadable:
- if err := copyFile(m.Path, m.Aside, info.Mode()); err != nil {
+ if err := publishCopy(m.Path, m.Aside, info.Mode()); err != nil {
return err
}
@@ -200,8 +202,13 @@ func (m Mutation) Apply() error {
// hold copies the original aside and then makes the secret writable, in that
// order: the copy has to happen while the file can still be read.
+//
+// The backup is PUBLISHED atomically, so its existence means it is complete.
+// Restore cannot tell a truncated backup from a genuinely short secret by
+// looking at it, and a 5-byte secret is a legitimate secret, so the guarantee
+// has to be structural rather than validated.
func (m Mutation) hold(info os.FileInfo) error {
- if err := copyFile(m.Path, m.Aside, info.Mode()); err != nil {
+ if err := publishCopy(m.Path, m.Aside, info.Mode()); err != nil {
return err
}
@@ -251,7 +258,7 @@ func (m Mutation) Restore() error {
return err
}
- if err := copyFile(m.Aside, m.Path, held.Mode()); err != nil {
+ if err := copyInto(m.Aside, m.Path, held.Mode()); err != nil {
return err
}
@@ -262,7 +269,75 @@ func (m Mutation) Restore() error {
return os.Remove(m.Aside)
}
-func copyFile(from, to string, mode os.FileMode) error {
+// publishCopy writes the copy to a temporary file in the destination's own
+// directory, flushes it, and renames it into place. A failed copy leaves
+// NOTHING at the destination, which is what lets a later Restore treat the
+// backup's existence as proof that the whole original is in it.
+//
+// Same directory because rename is only atomic within a filesystem, and the
+// flush before it because a rename that beats the data to disk publishes a
+// complete-looking backup with no bytes in it.
+func publishCopy(from, to string, mode os.FileMode) error {
+ //nolint:gosec // the path is the fault's subject, guarded by the nonce
+ source, err := os.Open(from)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = source.Close() }()
+
+ if mode.Perm() == 0 {
+ mode = restoredMode
+ }
+
+ temp, err := os.CreateTemp(filepath.Dir(to), tempPrefix)
+ if err != nil {
+ return err
+ }
+
+ published := false
+
+ defer func() {
+ if !published {
+ _ = temp.Close()
+ //nolint:gosec // os.CreateTemp's own name, in the directory
+ // khaos derived from the destination; no caller supplies it
+ _ = os.Remove(temp.Name())
+ }
+ }()
+
+ if _, err := io.Copy(temp, source); err != nil {
+ return err
+ }
+
+ if err := temp.Sync(); err != nil {
+ return err
+ }
+
+ if err := temp.Chmod(mode.Perm()); err != nil {
+ return err
+ }
+
+ if err := temp.Close(); err != nil {
+ return err
+ }
+
+ //nolint:gosec // same: both paths are khaos's own, and the rename is what
+ // makes the backup's existence mean it is complete
+ if err := os.Rename(temp.Name(), to); err != nil {
+ return err
+ }
+
+ published = true
+
+ return nil
+}
+
+// copyInto overwrites the destination IN PLACE, keeping its inode. The secret
+// is restored this way rather than by rename because the whole reason this
+// tier runs in a helper container is that unlink-and-recreate is invisible to
+// a target holding the file open, and a restore that swapped the inode would
+// leave that target reading the fault forever.
+func copyInto(from, to string, mode os.FileMode) error {
//nolint:gosec // the path is the fault's subject, guarded by the nonce
source, err := os.Open(from)
if err != nil {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go
index 12629d9e..d3618268 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go
@@ -9,6 +9,7 @@ import (
"io/fs"
"os"
"path/filepath"
+ "strings"
"testing"
)
@@ -342,3 +343,133 @@ func TestVerifyRefusesAMalformedCredential(t *testing.T) {
}
}
}
+
+// The one outcome this package must never produce: khaos destroying the secret
+// it was asked to preserve.
+//
+// The backup used to be written at its FINAL path, so a copy that failed
+// partway left a short file where a complete original was expected, and a read
+// error left a zero-byte one. Restore read existence as proof of a complete
+// original, opened the untouched secret with O_TRUNC, and copied the stub over
+// it. A short write or a full filesystem was enough, and truncating the
+// original could itself free the space that let the destructive restore finish
+// and report success.
+//
+// Restore cannot tell a truncated backup from a genuinely short secret by
+// looking at it -- a 5-byte secret is a legitimate secret -- so the fix is
+// structural: the backup is published atomically, and a failed hold leaves
+// nothing behind for Restore to trust.
+func TestAFailedHoldLeavesTheOriginalIntactAndUnrestorable(t *testing.T) {
+ path, aside := seeded(t)
+
+ // /proc/self/mem opens cleanly and fails on the first read, so the copy
+ // fails with the destination already created under the old code.
+ failing := Mutation{Kind: Empty, Path: "/proc/self/mem", Aside: aside}
+
+ if err := failing.hold(modeOf(t, path)); err == nil {
+ t.Fatal("copying from an unreadable source reported success")
+ }
+
+ if _, err := os.Stat(aside); err == nil {
+ body, _ := os.ReadFile(aside)
+ t.Fatalf(
+ "a failed backup left %d bytes at the path Restore trusts as the whole original",
+ len(body),
+ )
+ }
+
+ // The chain that destroyed the secret: the revert that follows a failed
+ // apply must refuse, not restore from whatever is lying there.
+ mutation := Mutation{Kind: Empty, Path: path, Aside: aside}
+
+ if err := mutation.Restore(); err != nil {
+ t.Fatalf(
+ "an untouched secret with no backup should be already-restored, got: %v",
+ err,
+ )
+ }
+
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("the original is unreadable after the revert: %v", err)
+ }
+
+ if string(body) != original {
+ t.Fatalf(
+ "khaos destroyed the secret it was preserving: now %q (%d bytes), was %q (%d bytes)",
+ body,
+ len(body),
+ original,
+ len(original),
+ )
+ }
+}
+
+// Publishing atomically must not leave its temporary file behind either: the
+// secret's own directory is inside the target's mount, and a stray copy of a
+// credential there outlives the run.
+func TestAFailedHoldLeavesNoTemporaryCopyOfTheSecret(t *testing.T) {
+ path, aside := seeded(t)
+
+ failing := Mutation{Kind: Empty, Path: "/proc/self/mem", Aside: aside}
+ _ = failing.hold(modeOf(t, path))
+
+ entries, err := os.ReadDir(filepath.Dir(path))
+ if err != nil {
+ t.Fatalf("reading the secret's directory: %v", err)
+ }
+
+ for _, entry := range entries {
+ if strings.HasPrefix(entry.Name(), tempPrefix) {
+ t.Fatalf(
+ "a failed hold left %q beside the secret, which is a stray copy of a credential",
+ entry.Name(),
+ )
+ }
+ }
+}
+
+// The backup a SUCCESSFUL hold publishes has to be byte-identical and carry the
+// original's mode, or the restore puts back something the target cannot read.
+func TestThePublishedBackupIsTheWholeOriginal(t *testing.T) {
+ path, aside := seeded(t)
+
+ mutation := Mutation{Kind: Empty, Path: path, Aside: aside}
+
+ if err := mutation.hold(modeOf(t, path)); err != nil {
+ t.Fatalf("hold: %v", err)
+ }
+
+ body, err := os.ReadFile(aside)
+ if err != nil {
+ t.Fatalf("the published backup is unreadable: %v", err)
+ }
+
+ if string(body) != original {
+ t.Fatalf("the backup holds %q, want %q", body, original)
+ }
+
+ info, err := os.Stat(aside)
+ if err != nil {
+ t.Fatalf("stat the backup: %v", err)
+ }
+
+ if info.Mode().Perm() != secretPerm {
+ t.Fatalf(
+ "the backup is mode %#o, want the original's %#o",
+ info.Mode().Perm(),
+ secretPerm,
+ )
+ }
+}
+
+func modeOf(t *testing.T, path string) os.FileInfo {
+ t.Helper()
+
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatalf("stat: %v", err)
+ }
+
+ return info
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh
index e79150c5..87362b58 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh
@@ -302,8 +302,14 @@ echo "== the codes this gate cannot reach, named rather than implied =="
# internal/cli covers the mapping.
# 9 UNSUPPORTED driven by scripts/dns.sh and scripts/tls.sh, against the
# targets whose question genuinely does not apply.
-# 70 INTERNAL a recovered panic. Reachable only from a khaos bug, so
-# there is nothing honest to assert.
+# 70 INTERNAL a recovered panic, recovered at the process boundary in
+# cmd/khaos. This comment used to say there was "nothing
+# honest to assert" about it, which was itself the dishonest
+# part: no production path returned the code at all, and no
+# recover() existed anywhere, so the gate described a
+# mechanism khaos did not have. It is driven by a unit test
+# that panics through the real boundary and asserts both the
+# code and that deferred reverts still ran.
#
# A CENSUS rather than a comment. The paragraph above was an unconditional
# `pass`, which is documentation wearing an assertion's clothes: it could not
@@ -329,6 +335,35 @@ else
fail "the unit census is gone, so the codes this gate cannot drive are unverified"
fi
+# A code with no producer is a published contract khaos does not honour, which
+# is what 70 was: declared, counted by the census above, asserted by number,
+# and returned from nowhere. The census cannot see that, because counting
+# constants proves declaration and never reachability.
+if grep -q 'code = cli.CodeInternal' cmd/khaos/main.go &&
+ grep -q 'func TestAPanicBecomesTheInternalCodeWithCleanupIntact' cmd/khaos/main_test.go; then
+ pass "the internal code has a producer and a test that panics through it"
+else
+ fail "exit 70 is declared with no production path, so the taxonomy publishes a code khaos never returns"
+fi
+
+# The refusal that used to be a panic: a non-positive interval reached
+# time.NewTicker AFTER the workspace lock was taken.
+if ! out="$("$BIN" reap --interval 0 --no-color 2>&1)"; then
+ if grep -q "longer than zero" <<<"$out"; then
+ pass "a non-positive reap interval is refused by name instead of panicking"
+ else
+ fail "reap --interval 0 failed without naming the cause: ${out}"
+ fi
+else
+ fail "reap --interval 0 was accepted"
+fi
+
+if grep -q "panic" <<<"$out"; then
+ fail "reap --interval 0 still panics: ${out}"
+else
+ pass "reap --interval 0 produces no stack trace"
+fi
+
if [[ "$failures" -gt 0 ]]; then
echo "report: ${failures} assertion(s) failed"
exit 1