Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(khaos): the reporting surface, and the exit code that was a green build

CarterPerez-devba05463158b1
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
index 5885b7eb..69ea2fd0 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go
@@ -23,15 +23,14 @@ import (
)
const (
- exitOK = 0
- exitError = 1
- exitDefect = 2
+ exitOK = cli.CodeOK
- // An interrupted run is a kind of inconclusive one, so they share a code.
- exitInconclusive = 4
- exitInterrupted = 4
+ // The catch-all. A bad flag, an unreadable config and an error khaos has
+ // no taxonomy entry for are all the operator's to fix before measuring.
+ exitError = cli.CodeUsage
- exitRevertFailed = 6
+ exitInterrupted = cli.CodeInterrupted
+ exitRevertFailed = cli.CodeRollback
noColorFlag = "--" + cli.FlagNoColor
noColorFlagTrue = noColorFlag + "="
@@ -106,6 +105,11 @@ func noColorRequested(args []string) bool {
// fail maps an error to an exit code, and the ORDER is the contract: the state
// khaos left the target in outranks the verdict, and both outrank interruption.
// errors.Join carries several at once, so the first match decides.
+//
+// A finished run arrives carrying its own code, because the precedence among
+// verdict outcomes is decided once where the run ends rather than re-derived
+// by a chain here that grows with every epistemic value. The checks below it
+// are for errors that never reached a verdict at all.
func fail(err error) int {
if errors.Is(err, run.ErrRevertFailed) {
fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
@@ -113,11 +117,20 @@ func fail(err error) int {
return exitRevertFailed
}
+ // Checked before the carried code as well as inside it: this is the one
+ // outcome nothing may suppress, so it is matched on the sentinel too and
+ // not only on a code some other layer computed.
if errors.Is(err, cli.ErrNotRecovered) {
fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
return exitRevertFailed
}
+ var exit *cli.Exit
+ if errors.As(err, &exit) {
+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
+ return exit.Code
+ }
+
if errors.Is(err, context.Canceled) {
fmt.Fprintf(os.Stderr, errorFormat, version.Name, errInterrupted)
return exitInterrupted
@@ -126,14 +139,6 @@ func fail(err error) int {
fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)
printFix(err)
- if errors.Is(err, cli.ErrDefectFound) {
- return exitDefect
- }
-
- if errors.Is(err, cli.ErrInconclusive) {
- return exitInconclusive
- }
-
return exitError
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
index a21a56f0..48e0ce9c 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go
@@ -155,8 +155,45 @@ func TestRollbackFailureOutranksEveryOtherExitCode(t *testing.T) {
exitRevertFailed,
},
{"cancellation alone", context.Canceled, exitInterrupted},
- {"defect alone", cli.ErrDefectFound, exitDefect},
- {"inconclusive alone", cli.ErrInconclusive, exitInconclusive},
+
+ // A finished run arrives carrying its own code, which is the shape
+ // exitFor actually produces. The bare sentinels below it reach here
+ // only from paths that never got a verdict.
+ {
+ "a defect carrying its code",
+ &cli.Exit{
+ Code: cli.CodeDefect,
+ Outcome: cli.OutcomeDefect,
+ Err: cli.ErrDefectFound,
+ },
+ cli.CodeDefect,
+ },
+ {
+ "a rollback failure carrying its code",
+ &cli.Exit{
+ Code: cli.CodeRollback,
+ Err: run.ErrRevertFailed,
+ },
+ exitRevertFailed,
+ },
+ {
+ "an interrupted run carrying its code",
+ &cli.Exit{
+ Code: cli.CodeInterrupted,
+ Outcome: cli.OutcomeInterrupted,
+ Err: cli.ErrInconclusive,
+ },
+ cli.CodeInterrupted,
+ },
+ {
+ "a carried code joined with a rollback failure",
+ errors.Join(
+ &cli.Exit{Code: cli.CodeDefect, Err: cli.ErrDefectFound},
+ run.ErrRevertFailed,
+ ),
+ exitRevertFailed,
+ },
+ {"defect alone, with no code", cli.ErrDefectFound, exitError},
{"anything else", errors.New("something broke"), exitError},
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go
new file mode 100644
index 00000000..e95fab45
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go
@@ -0,0 +1,177 @@
+// ©AngelaMos | 2026
+// emit.go
+
+package cli
+
+import (
+ "errors"
+ "fmt"
+ "io"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/record"
+ "github.com/CarterPerez-dev/khaos/internal/version"
+ "github.com/spf13/cobra"
+)
+
+const (
+ errUnknownOutput = "%w: %q (khaos writes %s or %s)"
+ errRecordWrite = "%w: %v"
+ warnRecordWrite = "khaos could not write the run record: %v\n"
+)
+
+var ErrUnknownOutput = errors.New(
+ "that is not an output shape khaos can write",
+)
+
+var ErrRecordUnwritten = errors.New(
+ "the run finished and khaos could not write the record you asked for",
+)
+
+// outputShape refuses an unrecognised value rather than falling back to the
+// terminal report: a pipeline that asked for json and silently got a banner
+// fails downstream in a place that does not name the cause.
+func outputShape(cmd *cobra.Command) (string, error) {
+ shape, err := cmd.Flags().GetString(flagOutput)
+ if err != nil {
+ return "", err
+ }
+
+ switch shape {
+ case outputText, outputJSON:
+ return shape, nil
+ default:
+ return "", fmt.Errorf(
+ errUnknownOutput,
+ ErrUnknownOutput,
+ shape,
+ outputText,
+ outputJSON,
+ )
+ }
+}
+
+// Projections are the files that are derived from the record rather than
+// being it. Each is a path or empty.
+type Projections struct {
+ JUnit string
+ Metrics string
+}
+
+func projectionsOf(cmd *cobra.Command) (Projections, error) {
+ junit, err := cmd.Flags().GetString(flagJUnit)
+ if err != nil {
+ return Projections{}, err
+ }
+
+ metrics, err := cmd.Flags().GetString(flagMetricsOut)
+ if err != nil {
+ return Projections{}, err
+ }
+
+ return Projections{JUnit: junit, Metrics: metrics}, nil
+}
+
+// emitRecord writes the canonical record where it was asked for and returns
+// the run's own outcome.
+//
+// The ORDER of the two failures matters. The verdict outranks the
+// bookkeeping, so a record khaos could not write never masks a defect: a run
+// that found a bypass still exits 2 and says the record is missing. But a
+// record an operator explicitly asked for and did not get must not be
+// SILENT either, because in CI they believe they have an artifact and do
+// not. So a clean run that could not write its record fails on the write.
+func emitRecord(
+ out io.Writer,
+ definition experiment.Definition,
+ result experiment.Result,
+ outcome error,
+ emit Output,
+) error {
+ code, name := exitDetail(outcome)
+
+ rec := record.Of(version.Version(), result, definition, code, name)
+
+ var writeErr error
+
+ if emit.Record != "" {
+ rec, writeErr = record.Write(
+ emit.Record,
+ rec,
+ record.EventsOf(result.Observation),
+ )
+ }
+
+ if emit.Shape == outputJSON {
+ body, err := record.Encode(rec)
+ if err != nil {
+ return err
+ }
+
+ if _, err := out.Write(body); err != nil {
+ return err
+ }
+ }
+
+ // The projections are derived from the record AFTER it is complete, so
+ // they carry its events reference and its exit code and cannot describe
+ // a different run than the one on disk.
+ if err := writeProjections(rec, emit.Projections); err != nil {
+ writeErr = errors.Join(writeErr, err)
+ }
+
+ if writeErr == nil {
+ return outcome
+ }
+
+ // A verdict khaos reached outranks a file it could not write.
+ if outcome != nil {
+ fmt.Fprintf(out, warnRecordWrite, writeErr)
+ return outcome
+ }
+
+ return fmt.Errorf(errRecordWrite, ErrRecordUnwritten, writeErr)
+}
+
+// writeProjections writes each derived file that was asked for. They are
+// derived from the record alone, which is what makes them projections rather
+// than second opinions.
+func writeProjections(rec record.Record, into Projections) error {
+ var failures error
+
+ if into.JUnit != "" {
+ body, err := record.JUnit(rec)
+ if err != nil {
+ failures = errors.Join(failures, err)
+ } else {
+ failures = errors.Join(
+ failures,
+ record.WriteFile(into.JUnit, body),
+ )
+ }
+ }
+
+ if into.Metrics != "" {
+ failures = errors.Join(
+ failures,
+ record.WriteFile(into.Metrics, record.Metrics(rec)),
+ )
+ }
+
+ return failures
+}
+
+// exitDetail reads the code and the outcome name back off a finished run, so
+// the record can carry what khaos is about to return to the shell.
+func exitDetail(outcome error) (int, string) {
+ if outcome == nil {
+ return CodeOK, ""
+ }
+
+ var exit *Exit
+ if errors.As(outcome, &exit) {
+ return exit.Code, string(exit.Outcome)
+ }
+
+ return CodeUsage, ""
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go
new file mode 100644
index 00000000..6b3b9287
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go
@@ -0,0 +1,399 @@
+// ©AngelaMos | 2026
+// emit_test.go
+
+package cli
+
+import (
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/record"
+)
+
+func emitted(t *testing.T) (experiment.Result, experiment.Definition) {
+ t.Helper()
+
+ definition, err := experiment.Lookup("control.policy_store_down")
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ result := held()
+ result.Experiment = definition.Name
+ result.Family = definition.Family
+ result.Hypothesis = "a hypothesis"
+ result.Fault = experiment.Fault{
+ Kind: definition.FaultKind,
+ Target: "a-target",
+ }
+
+ return result, definition
+}
+
+// Under --output json, stdout carries the record and NOTHING else. A single
+// banner line makes `khaos run x --output json | jq .verdict` fail on the
+// first byte, and that failure names jq rather than khaos.
+func TestJSONOutputWritesOnlyTheRecord(t *testing.T) {
+ result, definition := emitted(t)
+
+ out := &strings.Builder{}
+
+ if err := emitRecord(
+ out,
+ definition,
+ result,
+ nil,
+ Output{Shape: outputJSON, Record: ""},
+ ); err != nil {
+ t.Fatalf("emit: %v", err)
+ }
+
+ body := out.String()
+
+ if !strings.HasPrefix(body, "{") {
+ t.Fatalf("stdout does not begin with the record: %.40q", body)
+ }
+
+ var parsed record.Record
+ if err := json.Unmarshal([]byte(body), &parsed); err != nil {
+ t.Fatalf("stdout is not a single JSON object: %v", err)
+ }
+
+ if parsed.Experiment.Name != definition.Name {
+ t.Fatalf("the record names %q", parsed.Experiment.Name)
+ }
+}
+
+// The text shape writes no record to stdout at all, or every existing
+// consumer of the terminal report gets a wall of JSON appended to it.
+func TestTextOutputWritesNoRecordToStdout(t *testing.T) {
+ result, definition := emitted(t)
+
+ out := &strings.Builder{}
+
+ if err := emitRecord(
+ out,
+ definition,
+ result,
+ nil,
+ Output{Shape: outputText, Record: ""},
+ ); err != nil {
+ t.Fatalf("emit: %v", err)
+ }
+
+ if out.String() != "" {
+ t.Fatalf("the text shape wrote to stdout: %.60q", out.String())
+ }
+}
+
+// The record carries the code khaos is about to return, because the exit
+// status is the only part of a run most CI ever sees.
+func TestTheRecordCarriesTheExitItIsAbout(t *testing.T) {
+ result, definition := emitted(t)
+
+ for _, row := range []struct {
+ name string
+ outcome error
+ code int
+ label string
+ }{
+ {"a clean run", nil, CodeOK, ""},
+ {
+ "a defect",
+ &Exit{
+ Code: CodeDefect,
+ Outcome: OutcomeDefect,
+ Err: ErrDefectFound,
+ },
+ CodeDefect,
+ string(OutcomeDefect),
+ },
+ {
+ "thin evidence",
+ &Exit{
+ Code: CodeInconclusive,
+ Outcome: OutcomeInconclusive,
+ Err: ErrInconclusive,
+ },
+ CodeInconclusive,
+ string(OutcomeInconclusive),
+ },
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ out := &strings.Builder{}
+
+ err := emitRecord(
+ out,
+ definition,
+ result,
+ row.outcome,
+ Output{Shape: outputJSON, Record: ""},
+ )
+
+ if !errors.Is(err, row.outcome) {
+ t.Fatalf("emit returned %v, want %v", err, row.outcome)
+ }
+
+ var parsed record.Record
+ if decodeErr := json.Unmarshal(
+ []byte(out.String()),
+ &parsed,
+ ); decodeErr != nil {
+ t.Fatalf("decode: %v", decodeErr)
+ }
+
+ if parsed.Exit.Code != row.code {
+ t.Errorf(
+ "the record says exit %d and khaos returns %d",
+ parsed.Exit.Code,
+ row.code,
+ )
+ }
+
+ if parsed.Exit.Outcome != row.label {
+ t.Errorf(
+ "the record names the outcome %q, want %q",
+ parsed.Exit.Outcome,
+ row.label,
+ )
+ }
+ })
+ }
+}
+
+// A verdict khaos reached outranks a file it could not write: a run that
+// found a bypass still exits on the bypass.
+func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {
+ result, definition := emitted(t)
+
+ defect := &Exit{
+ Code: CodeDefect,
+ Outcome: OutcomeDefect,
+ Err: ErrDefectFound,
+ }
+
+ out := &strings.Builder{}
+
+ err := emitRecord(
+ out,
+ definition,
+ result,
+ defect,
+ Output{
+ Shape: outputText,
+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),
+ },
+ )
+
+ if !errors.Is(err, ErrDefectFound) {
+ t.Fatalf("the defect was masked by the write failure: %v", err)
+ }
+
+ // And the failure is still SAID, because an operator who asked for a
+ // record and did not get one must not have to guess.
+ if !strings.Contains(out.String(), "could not write the run record") {
+ t.Fatalf("the write failure was silent: %.80q", out.String())
+ }
+}
+
+// But a CLEAN run that could not write the record the operator asked for
+// fails on the write: in CI they believe they have an artifact and do not.
+func TestACleanRunFailsWhenItsRecordCannotBeWritten(t *testing.T) {
+ result, definition := emitted(t)
+
+ err := emitRecord(
+ &strings.Builder{},
+ definition,
+ result,
+ nil,
+ Output{
+ Shape: outputText,
+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),
+ },
+ )
+
+ if !errors.Is(err, ErrRecordUnwritten) {
+ t.Fatalf("a missing record was silent on a clean run: %v", err)
+ }
+}
+
+// THE PROJECTION INVARIANT. The record is canonical and the stdout form is a
+// projection of it, so the bytes on stdout and the bytes on disk have to be
+// the same record. If they can differ, "the pretty output says one thing and
+// the JSON says another" is back.
+func TestTheRecordOnStdoutIsTheRecordOnDisk(t *testing.T) {
+ result, definition := emitted(t)
+ dir := t.TempDir()
+
+ out := &strings.Builder{}
+
+ if err := emitRecord(
+ out,
+ definition,
+ result,
+ nil,
+ Output{Shape: outputJSON, Record: dir},
+ ); err != nil {
+ t.Fatalf("emit: %v", err)
+ }
+
+ onDisk, err := os.ReadFile(filepath.Join(dir, record.RecordFile))
+ if err != nil {
+ t.Fatalf("read the record: %v", err)
+ }
+
+ if out.String() != string(onDisk) {
+ t.Fatal(
+ "the record on stdout and the record on disk are not the same bytes",
+ )
+ }
+}
+
+// An unrecognised --output is refused rather than falling back to the
+// terminal report: a pipeline that asked for json and silently got a banner
+// fails somewhere that does not name the cause.
+func TestAnUnknownOutputShapeIsRefused(t *testing.T) {
+ cmd := find(t, NewRoot(), "run")
+
+ sub := find(t, cmd, "control.policy_store_down")
+
+ if err := sub.Flags().Set(flagOutput, "yaml"); err != nil {
+ t.Fatalf("set: %v", err)
+ }
+
+ _, err := outputShape(sub)
+ if !errors.Is(err, ErrUnknownOutput) {
+ t.Fatalf("an unknown output shape was accepted: %v", err)
+ }
+}
+
+// Both shapes are accepted, so the refusal above cannot be passing because
+// outputShape refuses everything.
+func TestBothOutputShapesAreAccepted(t *testing.T) {
+ cmd := find(t, NewRoot(), "run")
+ sub := find(t, cmd, "control.policy_store_down")
+
+ for _, shape := range []string{outputText, outputJSON} {
+ if err := sub.Flags().Set(flagOutput, shape); err != nil {
+ t.Fatalf("set %q: %v", shape, err)
+ }
+
+ got, err := outputShape(sub)
+ if err != nil {
+ t.Fatalf("outputShape(%q): %v", shape, err)
+ }
+
+ if got != shape {
+ t.Fatalf("outputShape(%q) = %q", shape, got)
+ }
+ }
+}
+
+// Both projections are written from the FINISHED record, so they describe
+// the same run as the file on disk. Deriving either one before the record is
+// complete is how a JUnit suite ends up naming a different exit code than
+// the record beside it.
+func TestTheProjectionsDescribeTheSameRunAsTheRecord(t *testing.T) {
+ result, definition := emitted(t)
+ dir := t.TempDir()
+
+ defect := &Exit{
+ Code: CodeDefect,
+ Outcome: OutcomeDefect,
+ Err: ErrDefectFound,
+ }
+
+ junit := filepath.Join(dir, "khaos.xml")
+ metrics := filepath.Join(dir, "khaos.prom")
+
+ err := emitRecord(
+ &strings.Builder{},
+ definition,
+ result,
+ defect,
+ Output{
+ Shape: outputText,
+ Record: dir,
+ Projections: Projections{JUnit: junit, Metrics: metrics},
+ },
+ )
+
+ if !errors.Is(err, ErrDefectFound) {
+ t.Fatalf("emit: %v", err)
+ }
+
+ onDisk, err := os.ReadFile(filepath.Join(dir, record.RecordFile))
+ if err != nil {
+ t.Fatalf("read the record: %v", err)
+ }
+
+ var parsed record.Record
+ if err := json.Unmarshal(onDisk, &parsed); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+
+ xmlBody, err := os.ReadFile(junit)
+ if err != nil {
+ t.Fatalf("read the junit: %v", err)
+ }
+
+ promBody, err := os.ReadFile(metrics)
+ if err != nil {
+ t.Fatalf("read the metrics: %v", err)
+ }
+
+ // The run id ties all three together, and the exit code is the thing
+ // most likely to be computed twice and disagree.
+ for name, body := range map[string]string{
+ "junit": string(xmlBody),
+ "metrics": string(promBody),
+ } {
+ if !strings.Contains(body, parsed.RunID) {
+ t.Errorf("the %s does not carry the record's run id", name)
+ }
+
+ if !strings.Contains(body, strconv.Itoa(parsed.Exit.Code)) {
+ t.Errorf(
+ "the %s does not carry the record's exit code %d",
+ name,
+ parsed.Exit.Code,
+ )
+ }
+ }
+
+ // And the record's own events reference survived being written before
+ // the projections were derived from it.
+ if parsed.Events == nil || parsed.Events.SHA256 == "" {
+ t.Error("the record on disk carries no events reference")
+ }
+}
+
+// A projection khaos could not write follows the same rule as the record: a
+// verdict outranks it, and it is never silent.
+func TestAProjectionThatCannotBeWrittenIsNotSilent(t *testing.T) {
+ result, definition := emitted(t)
+
+ err := emitRecord(
+ &strings.Builder{},
+ definition,
+ result,
+ nil,
+ Output{
+ Shape: outputText,
+ Record: "",
+ Projections: Projections{
+ JUnit: filepath.Join("/proc", "nope", "khaos.xml"),
+ },
+ },
+ )
+
+ if !errors.Is(err, ErrRecordUnwritten) {
+ t.Fatalf("a projection khaos could not write was silent: %v", err)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit.go
new file mode 100644
index 00000000..b64a95dc
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit.go
@@ -0,0 +1,207 @@
+// ©AngelaMos | 2026
+// exit.go
+
+package cli
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+)
+
+// The exit contract. A single non-zero code would collapse "a control failed
+// open", "the experiment was invalid" and "khaos crashed" into one signal, and
+// those need three different human responses.
+const (
+ CodeOK = 0
+ CodeUsage = 1
+ CodeDefect = 2
+ CodeDeviated = 3
+ CodeInconclusive = 4
+ CodeProbe = 5
+ CodeRollback = 6
+ CodeVoid = 7
+ CodeBaseline = 8
+ CodeUnsupported = 9
+ CodeInterrupted = 10
+ CodeInternal = 70
+)
+
+// Outcome names an exit code for --fail-on. The names are the operator's
+// vocabulary, so they are the same words the verdict prints.
+type Outcome string
+
+const (
+ OutcomeDefect Outcome = "defect"
+ OutcomeDeviated Outcome = "deviated"
+ OutcomeInconclusive Outcome = "inconclusive"
+ OutcomeProbe Outcome = "probe-error"
+ OutcomeVoid Outcome = "void"
+ OutcomeBaseline Outcome = "baseline-not-met"
+ OutcomeUnsupported Outcome = "unsupported"
+ OutcomeInterrupted Outcome = "interrupted"
+
+ // outcomeRollback is deliberately unexported and absent from
+ // suppressible(): it names a code no gate may silence, so an operator
+ // must not be able to type it into --fail-on and believe it did
+ // something.
+ outcomeRollback Outcome = "rollback-failed"
+
+ // FailOnAny is the default: every outcome khaos declines to call a clean
+ // pass keeps its own non-zero code. FailOnNever reports every measurable
+ // outcome as 0 and leaves the detail to the run record.
+ FailOnAny = "any"
+ FailOnNever = "never"
+)
+
+// Exit carries the code beside the error, so the precedence is decided once
+// where the run finishes rather than re-derived by a growing chain of
+// errors.Is checks at the process boundary.
+type Exit struct {
+ Code int
+ Outcome Outcome
+ Err error
+}
+
+func (e *Exit) Error() string { return e.Err.Error() }
+func (e *Exit) Unwrap() error { return e.Err }
+
+// suppressible is every outcome --fail-on may silence. Rollback failure,
+// usage and internal errors are deliberately absent: an unreverted fault
+// leaves the environment dirty for every later job on that runner, and a tool
+// that crashed or could not read its own flags is not reporting a
+// measurement at all.
+func suppressible() []Outcome {
+ return []Outcome{
+ OutcomeDefect,
+ OutcomeDeviated,
+ OutcomeInconclusive,
+ OutcomeProbe,
+ OutcomeVoid,
+ OutcomeBaseline,
+ OutcomeUnsupported,
+ OutcomeInterrupted,
+ }
+}
+
+var ErrUnknownOutcome = errors.New(
+ "that is not an outcome khaos can gate on",
+)
+
+// Gate is the parsed --fail-on set.
+type Gate struct {
+ all bool
+ none bool
+ allowed []Outcome
+}
+
+// ParseGate refuses an unknown name rather than quietly gating on nothing,
+// because a typo that silenced the defect code would turn every run green.
+func ParseGate(value string) (Gate, error) {
+ trimmed := strings.TrimSpace(value)
+
+ switch trimmed {
+ case "", FailOnAny:
+ return Gate{all: true}, nil
+ case FailOnNever:
+ return Gate{none: true}, nil
+ }
+
+ gate := Gate{}
+ known := suppressible()
+
+ for _, name := range strings.Split(trimmed, ",") {
+ outcome := Outcome(strings.TrimSpace(name))
+
+ if !slices.Contains(known, outcome) {
+ return Gate{}, fmt.Errorf(
+ errUnknownOutcome,
+ ErrUnknownOutcome,
+ outcome,
+ joinOutcomes(known),
+ )
+ }
+
+ gate.allowed = append(gate.allowed, outcome)
+ }
+
+ return gate, nil
+}
+
+// Apply narrows a code to 0 when its outcome is outside the gate. An outcome
+// this gate cannot suppress passes through whatever the gate says.
+func (g Gate) Apply(exit *Exit) int {
+ if exit == nil {
+ return CodeOK
+ }
+
+ if !slices.Contains(suppressible(), exit.Outcome) {
+ return exit.Code
+ }
+
+ if g.all {
+ return exit.Code
+ }
+
+ if g.none {
+ return CodeOK
+ }
+
+ if slices.Contains(g.allowed, exit.Outcome) {
+ return exit.Code
+ }
+
+ return CodeOK
+}
+
+func joinOutcomes(outcomes []Outcome) string {
+ names := make([]string, 0, len(outcomes))
+ for _, outcome := range outcomes {
+ names = append(names, string(outcome))
+ }
+
+ return strings.Join(names, listSep)
+}
+
+// epistemicExit maps the one axis that is not about the target's direction.
+// Every value is reachable and assigned at a real site in the verdict engine,
+// so none of these is an inert code.
+func epistemicExit(epistemic experiment.Epistemic) (int, Outcome, bool) {
+ switch epistemic {
+ case experiment.ProbeError:
+ return CodeProbe, OutcomeProbe, true
+ case experiment.Aborted:
+ return CodeVoid, OutcomeVoid, true
+ case experiment.BaselineNotMet:
+ return CodeBaseline, OutcomeBaseline, true
+ case experiment.Unsupported:
+ return CodeUnsupported, OutcomeUnsupported, true
+ case experiment.Interrupted:
+ return CodeInterrupted, OutcomeInterrupted, true
+ default:
+ return 0, "", false
+ }
+}
+
+// gated narrows a run's own exit to what --fail-on allows. A nil result is a
+// clean pass; an error that is not an Exit came from somewhere the gate has
+// no opinion about and passes through untouched.
+func gated(gate Gate, err error) error {
+ if err == nil {
+ return nil
+ }
+
+ var exit *Exit
+ if !errors.As(err, &exit) {
+ return err
+ }
+
+ if gate.Apply(exit) == CodeOK {
+ return nil
+ }
+
+ return err
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go
new file mode 100644
index 00000000..78e21e0d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go
@@ -0,0 +1,443 @@
+// ©AngelaMos | 2026
+// exit_test.go
+
+package cli
+
+import (
+ "errors"
+ "slices"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/run"
+)
+
+// held is a finished run with nothing wrong with it, which every case below
+// mutates one field of. Building it here rather than per case is what keeps a
+// case honest: a case that passes because it forgot to set Conclusive is
+// testing the fixture, not the mapping.
+func held() experiment.Result {
+ return experiment.Result{
+ Verdict: experiment.Verdict{
+ Epistemic: experiment.Passed,
+ Direction: experiment.Held,
+ Recovery: experiment.Recovered,
+ Conclusive: true,
+ },
+ }
+}
+
+func codeOf(t *testing.T, result experiment.Result) int {
+ t.Helper()
+
+ err := exitFor(result, nil)
+ if err == nil {
+ return CodeOK
+ }
+
+ var exit *Exit
+ if !errors.As(err, &exit) {
+ t.Fatalf("exitFor returned an error carrying no code: %v", err)
+ }
+
+ return exit.Code
+}
+
+// Every epistemic value is reachable and assigned at a real site in the
+// verdict engine, so every one needs its own code. A single non-zero exit
+// would collapse "a control failed open", "the experiment was invalid" and
+// "khaos could not measure" into one signal, and those are three different
+// human responses.
+func TestEveryReachableOutcomeHasItsOwnExitCode(t *testing.T) {
+ for _, row := range []struct {
+ name string
+ shape func(*experiment.Result)
+ want int
+ }{
+ {
+ "a clean held run",
+ func(*experiment.Result) {},
+ CodeOK,
+ },
+ {
+ "the direction this experiment calls a defect",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.Deviated
+ r.Verdict.Direction = experiment.Permitted
+ r.Verdict.Defect = true
+ },
+ CodeDefect,
+ },
+ {
+ "deviated in a direction it does NOT call a defect",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.Deviated
+ r.Verdict.Direction = experiment.Denied
+ },
+ CodeDeviated,
+ },
+ {
+ "held on evidence khaos declined to call conclusive",
+ func(r *experiment.Result) {
+ r.Verdict.Conclusive = false
+ },
+ CodeInconclusive,
+ },
+ {
+ "khaos's own measurement failed",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.ProbeError
+ },
+ CodeProbe,
+ },
+ {
+ "the fault was never confirmed",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.Aborted
+ },
+ CodeVoid,
+ },
+ {
+ "the target was broken before khaos injected",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.BaselineNotMet
+ },
+ CodeBaseline,
+ },
+ {
+ "the question does not apply to this target",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.Unsupported
+ },
+ CodeUnsupported,
+ },
+ {
+ "the run was interrupted",
+ func(r *experiment.Result) {
+ r.Verdict.Epistemic = experiment.Interrupted
+ },
+ CodeInterrupted,
+ },
+ {
+ "the fault could not be reverted",
+ func(r *experiment.Result) {
+ r.Observation.RollbackFailed = true
+ },
+ CodeRollback,
+ },
+ {
+ "the target never came back",
+ func(r *experiment.Result) {
+ r.Verdict.Recovery = experiment.Degraded
+ },
+ CodeRollback,
+ },
+ {
+ "recovery was never observed",
+ func(r *experiment.Result) {
+ r.Verdict.Recovery = experiment.Unobserved
+ },
+ CodeInconclusive,
+ },
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ result := held()
+ row.shape(&result)
+
+ if got := codeOf(t, result); got != row.want {
+ t.Fatalf("exit code %d, want %d", got, row.want)
+ }
+ })
+ }
+}
+
+// No two outcomes may share a code, or CI cannot tell them apart, which is
+// the entire reason the taxonomy exists. Interrupted and inconclusive were
+// both literally 4 in the shipped binary.
+func TestNoTwoOutcomesShareACode(t *testing.T) {
+ codes := map[int]Outcome{}
+
+ for _, outcome := range append(suppressible(), outcomeRollback) {
+ code := codeFor(t, outcome)
+
+ if other, clash := codes[code]; clash {
+ t.Fatalf(
+ "%s and %s both exit %d, so nothing downstream can tell them apart",
+ outcome,
+ other,
+ code,
+ )
+ }
+
+ codes[code] = outcome
+ }
+}
+
+// codeFor walks the mapping the other way, through the states that produce
+// each outcome, so this census reads the real function rather than a table
+// written beside it.
+func codeFor(t *testing.T, outcome Outcome) int {
+ t.Helper()
+
+ result := held()
+
+ switch outcome {
+ case OutcomeDefect:
+ result.Verdict.Epistemic = experiment.Deviated
+ result.Verdict.Defect = true
+ case OutcomeDeviated:
+ result.Verdict.Epistemic = experiment.Deviated
+ case OutcomeInconclusive:
+ result.Verdict.Conclusive = false
+ case OutcomeProbe:
+ result.Verdict.Epistemic = experiment.ProbeError
+ case OutcomeVoid:
+ result.Verdict.Epistemic = experiment.Aborted
+ case OutcomeBaseline:
+ result.Verdict.Epistemic = experiment.BaselineNotMet
+ case OutcomeUnsupported:
+ result.Verdict.Epistemic = experiment.Unsupported
+ case OutcomeInterrupted:
+ result.Verdict.Epistemic = experiment.Interrupted
+ case outcomeRollback:
+ result.Observation.RollbackFailed = true
+ default:
+ t.Fatalf("no state produces the outcome %q", outcome)
+ }
+
+ return codeOf(t, result)
+}
+
+// A defect outranks the epistemic axis, because one leaked request is a proof
+// and khaos reports a proven bypass even on thin evidence.
+func TestADefectOutranksThinEvidence(t *testing.T) {
+ result := held()
+ result.Verdict.Epistemic = experiment.Deviated
+ result.Verdict.Defect = true
+ result.Verdict.Conclusive = false
+
+ if got := codeOf(t, result); got != CodeDefect {
+ t.Fatalf("exit code %d, want the defect code %d", got, CodeDefect)
+ }
+}
+
+// And the state khaos left the target in outranks the defect.
+func TestRollbackOutranksTheDefect(t *testing.T) {
+ result := held()
+ result.Verdict.Defect = true
+ result.Observation.RollbackFailed = true
+
+ if got := codeOf(t, result); got != CodeRollback {
+ t.Fatalf("exit code %d, want the rollback code %d", got, CodeRollback)
+ }
+}
+
+func TestTheDefaultGateNarrowsNothing(t *testing.T) {
+ gate, err := ParseGate("")
+ if err != nil {
+ t.Fatalf("the empty default was refused: %v", err)
+ }
+
+ for _, outcome := range suppressible() {
+ exit := &Exit{Code: codeFor(t, outcome), Outcome: outcome}
+
+ if got := gate.Apply(exit); got == CodeOK {
+ t.Fatalf(
+ "the default gate silenced %s; a run khaos declined to conclude from exiting 0 is a green build on no evidence",
+ outcome,
+ )
+ }
+ }
+}
+
+func TestGatingOnTheDefectAloneSilencesTheRest(t *testing.T) {
+ gate, err := ParseGate(string(OutcomeDefect))
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ for _, outcome := range suppressible() {
+ exit := &Exit{Code: codeFor(t, outcome), Outcome: outcome}
+ got := gate.Apply(exit)
+
+ if outcome == OutcomeDefect {
+ if got != CodeDefect {
+ t.Fatalf("the gated outcome was silenced: %d", got)
+ }
+
+ continue
+ }
+
+ if got != CodeOK {
+ t.Fatalf(
+ "%s survived a gate that does not name it: %d",
+ outcome,
+ got,
+ )
+ }
+ }
+}
+
+// The two codes no gate may touch, asserted against EVERY gate value rather
+// than against the default: a --fail-on that could suppress an unreverted
+// fault would leave the environment dirty for every later job on the runner.
+func TestNoGateCanSuppressARollbackFailure(t *testing.T) {
+ for _, value := range []string{
+ FailOnAny,
+ FailOnNever,
+ string(OutcomeDefect),
+ string(OutcomeInconclusive) + "," + string(OutcomeVoid),
+ } {
+ t.Run(value, func(t *testing.T) {
+ gate, err := ParseGate(value)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ exit := &Exit{
+ Code: CodeRollback,
+ Outcome: outcomeRollback,
+ Err: run.ErrRevertFailed,
+ }
+
+ if got := gate.Apply(exit); got != CodeRollback {
+ t.Fatalf(
+ "--fail-on %s silenced an unreverted fault, which leaves the environment dirty",
+ value,
+ )
+ }
+ })
+ }
+}
+
+// `never` is the one value that reports every measurable outcome as 0, and it
+// still cannot reach the rollback code above.
+func TestNeverSilencesEveryMeasurableOutcome(t *testing.T) {
+ gate, err := ParseGate(FailOnNever)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ for _, outcome := range suppressible() {
+ exit := &Exit{Code: codeFor(t, outcome), Outcome: outcome}
+
+ if got := gate.Apply(exit); got != CodeOK {
+ t.Fatalf("--fail-on never left %s at %d", outcome, got)
+ }
+ }
+}
+
+// A typo that silenced the defect code would turn every run green, so an
+// unknown name is refused rather than gating on nothing.
+func TestAnUnknownOutcomeIsRefusedByName(t *testing.T) {
+ for _, value := range []string{
+ "open",
+ "defect,typo",
+ "rollback-failed",
+ "DEFECT",
+ } {
+ t.Run(value, func(t *testing.T) {
+ _, err := ParseGate(value)
+ if !errors.Is(err, ErrUnknownOutcome) {
+ t.Fatalf("ParseGate(%q) did not refuse by name: %v", value, err)
+ }
+ })
+ }
+}
+
+// The gate's vocabulary is the verdict's own vocabulary, so an operator reads
+// an outcome in the report and can type it into the flag.
+func TestTheGateNamesMatchTheVerdictsOwnWords(t *testing.T) {
+ for _, row := range []struct {
+ outcome Outcome
+ epistemic experiment.Epistemic
+ }{
+ {OutcomeProbe, experiment.ProbeError},
+ {OutcomeVoid, experiment.Aborted},
+ {OutcomeBaseline, experiment.BaselineNotMet},
+ {OutcomeUnsupported, experiment.Unsupported},
+ {OutcomeInterrupted, experiment.Interrupted},
+ } {
+ // `void` is deliberately not `aborted`: the epistemic word describes
+ // what khaos did and the gate word describes what the operator got.
+ if row.outcome == OutcomeVoid {
+ continue
+ }
+
+ if string(row.outcome) != string(row.epistemic) {
+ t.Errorf(
+ "the gate calls it %q and the verdict prints %q",
+ row.outcome,
+ row.epistemic,
+ )
+ }
+ }
+}
+
+// gated is what turns a narrowed code into a nil error, so a suppressed
+// outcome genuinely exits 0 rather than printing a failure with code 0.
+func TestASuppressedOutcomeReturnsNoError(t *testing.T) {
+ gate, err := ParseGate(FailOnNever)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+
+ suppressed := gated(gate, &Exit{
+ Code: CodeInconclusive,
+ Outcome: OutcomeInconclusive,
+ Err: ErrInconclusive,
+ })
+
+ if suppressed != nil {
+ t.Fatalf("a suppressed outcome still returned an error: %v", suppressed)
+ }
+
+ // And an error the gate has no opinion about passes through untouched.
+ foreign := errors.New("the daemon refused the connection")
+ if got := gated(gate, foreign); !errors.Is(got, foreign) {
+ t.Fatalf("a non-verdict error was swallowed by the gate: %v", got)
+ }
+}
+
+// Nothing may quietly drop out of the gate's vocabulary: a code with no name
+// is a code no operator can gate on.
+func TestEverySuppressibleOutcomeIsParseable(t *testing.T) {
+ for _, outcome := range suppressible() {
+ gate, err := ParseGate(string(outcome))
+ if err != nil {
+ t.Fatalf("ParseGate(%q): %v", outcome, err)
+ }
+
+ if !slices.Contains(gate.allowed, outcome) {
+ t.Fatalf("%q parsed into a gate that does not contain it", outcome)
+ }
+ }
+}
+
+// The LITERAL numbers, because they are a published contract: a CI pipeline
+// keys off the integer, not off the name of the constant. Every other test
+// here compares a state against a named code, so all of them would survive a
+// wholesale renumbering together. This one would not.
+func TestTheExitCodesAreTheirPublishedNumbers(t *testing.T) {
+ for name, want := range map[string]struct {
+ got int
+ want int
+ }{
+ "OK": {CodeOK, 0},
+ "USAGE": {CodeUsage, 1},
+ "DEFECT": {CodeDefect, 2},
+ "DEVIATED": {CodeDeviated, 3},
+ "INCONCLUSIVE": {CodeInconclusive, 4},
+ "PROBE": {CodeProbe, 5},
+ "ROLLBACK": {CodeRollback, 6},
+ "VOID": {CodeVoid, 7},
+ "BASELINE": {CodeBaseline, 8},
+ "UNSUPPORTED": {CodeUnsupported, 9},
+ "INTERRUPTED": {CodeInterrupted, 10},
+ "INTERNAL": {CodeInternal, 70},
+ } {
+ if want.got != want.want {
+ t.Errorf("%s is %d, want %d", name, want.got, want.want)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index 3946b7a7..b5f1f8fb 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -42,6 +42,24 @@ const (
flagFaultFor = "fault-for"
flagRecoverFor = "recover-for"
flagSampleEvery = "sample-every"
+ flagFailOn = "fail-on"
+ flagRecord = "record"
+ flagOutput = "output"
+ flagJUnit = "junit"
+ flagMetricsOut = "metrics-out"
+
+ outputText = "text"
+ outputJSON = "json"
+
+ recordUsage = "a directory to write the canonical run record into, as `run.json` plus an `events.jsonl` beside it carrying one line per observation. The record is what every other output khaos produces is a projection of, and it is the only form that survives the terminal scrolling away"
+
+ outputUsage = "`text` for the terminal report, or `json` to write the run record to stdout and suppress every other line, so `khaos run x --output json | jq .verdict` works"
+
+ junitUsage = "a file to write JUnit XML to, for CI test reporters. A run khaos declined to conclude from becomes a `skipped` rather than a `failure`, which is what stops thin evidence breaking a build on noise; only a control that failed in the direction this experiment declares a defect becomes a `failure`"
+
+ metricsUsage = "a file to write Prometheus text exposition to, for node_exporter's textfile collector. Written atomically and consumed by pointing that collector at the directory; khaos stands up no server and opens no port"
+
+ failOnUsage = "which outcomes keep a non-zero exit code. `any` is the default and narrows nothing, because a run khaos declined to conclude from exiting 0 is a green build on no evidence; `never` reports every measurable outcome as 0 and leaves the detail to the run record; or name the outcomes to gate on, comma separated, where `defect` alone is the shape a pipeline that should only break on a security regression wants. A rollback failure, a usage error and an internal error ignore this flag, because an unreverted fault makes every later job on that runner untrustworthy"
catalogShort = "list the experiments khaos ships"
catalogLong = `Each experiment pairs a fault with two probes: one carrying a credential the
@@ -106,6 +124,7 @@ enforcing in the first place.`
errNoExperiment = "name the experiment to %s (known experiments: %s)"
errNoRangeFor = "%w by passing %s"
errInconclusive = "%w (%s)"
+ errUnknownOutcome = "%w: %q (khaos gates on: %s, or `any`, or `never`)"
verdictThinEvidence = "the sample cannot rule out a bypass"
verdictRecoveryUnobserved = "recovery was never observed"
errNoProxiedPath = "%w: this experiment injects through khaos's fault proxy, and the running range has no %s; bring the range down and up again to create it"
@@ -172,6 +191,10 @@ var ErrInconclusive = errors.New(
"this run did not reach a conclusion, so nothing can be concluded from it either way",
)
+var ErrDeviated = errors.New(
+ "the target broke in a direction this experiment does not declare a defect, which is the correct outcome with an availability cost khaos did not measure",
+)
+
func newCatalogCmd() *cobra.Command {
return &cobra.Command{
Use: "catalog",
@@ -277,18 +300,57 @@ func newRunExperimentCmd(definition experiment.Definition) *cobra.Command {
if err != nil {
return err
}
+
+ // Parsed BEFORE the run, so a typo that would have silenced the
+ // defect code is refused instead of discovered afterwards.
+ raw, err := cmd.Flags().GetString(flagFailOn)
+ if err != nil {
+ return err
+ }
+
+ gate, err := ParseGate(raw)
+ if err != nil {
+ return err
+ }
+
+ shape, err := outputShape(cmd)
+ if err != nil {
+ return err
+ }
+
+ into, err := cmd.Flags().GetString(flagRecord)
+ if err != nil {
+ return err
+ }
+
+ projections, err := projectionsOf(cmd)
+ if err != nil {
+ return err
+ }
+
return runExperiment(
cmd.Context(),
cmd.OutOrStdout(),
definition,
ttl,
given,
+ gate,
+ Output{
+ Shape: shape,
+ Record: into,
+ Projections: projections,
+ },
)
},
}
experimentFlags(cmd, definition)
cmd.Flags().Duration(flagTTL, lease.DefaultTTL, ttlUsage)
+ cmd.Flags().String(flagFailOn, FailOnAny, failOnUsage)
+ cmd.Flags().String(flagRecord, "", recordUsage)
+ cmd.Flags().String(flagOutput, outputText, outputUsage)
+ cmd.Flags().String(flagJUnit, "", junitUsage)
+ cmd.Flags().String(flagMetricsOut, "", metricsUsage)
return cmd
}
@@ -518,51 +580,116 @@ func describe(out io.Writer, spec experiment.Experiment, resolved string) {
fmt.Fprintln(out)
}
-func runExperiment(
+// prepared is everything a run needs resolved before anything is injected:
+// the workspace lock, the experiment with its parameters bound, and the
+// container whose consent khaos checked.
+type prepared struct {
+ work *workspace.Workspace
+ spec experiment.Experiment
+ accepted dockerx.Target
+ guard dockerx.Guard
+ project string
+}
+
+// prepareRun does every step that can still refuse. Separated from the run
+// itself because nothing here touches the target: if this returns an error,
+// khaos has injected nothing and has nothing to revert.
+//
+// The caller owns closing the workspace, because the lock has to outlive
+// this function and be released after the run rather than before it.
+func prepareRun(
ctx context.Context,
- out io.Writer,
+ human io.Writer,
definition experiment.Definition,
- ttl time.Duration,
given inputs,
-) error {
+) (prepared, error) {
work, err := workspace.OpenExclusive(ctx)
if err != nil {
- return err
+ return prepared{}, err
+ }
+
+ out := prepared{work: work, project: given.Project}
+ out.guard = dockerx.Guard{
+ Project: out.project,
+ SelfID: dockerx.SelfContainerID(),
}
- defer work.Close()
supplied, err := fillFromRange(
ctx,
- out,
+ human,
work.Docker,
definition,
given.Supplied,
)
if err != nil {
- return err
+ return out, err
}
- spec, err := experiment.Build(definition.Name, supplied, given.Timing)
+ out.spec, err = experiment.Build(
+ definition.Name,
+ supplied,
+ given.Timing,
+ )
if err != nil {
- return err
+ return out, err
}
- project := given.Project
- guard := dockerx.Guard{Project: project, SelfID: dockerx.SelfContainerID()}
-
- accepted, err := work.Docker.Accept(
+ out.accepted, err = work.Docker.Accept(
ctx,
- spec.Fault.Target,
- dockerx.Filter{Project: project},
- guard,
+ out.spec.Fault.Target,
+ dockerx.Filter{Project: out.project},
+ out.guard,
)
+
+ return out, err
+}
+
+// Output is where a run's results go. Grouped because they are one decision
+// -- what this invocation should emit -- rather than three.
+type Output struct {
+ Shape string
+ Record string
+ Projections Projections
+}
+
+// human is where the terminal report goes, and under --output json that is
+// nowhere: stdout has to carry the record and nothing else, or piping it
+// into jq fails on the first banner line.
+func (o Output) human(out io.Writer) io.Writer {
+ if o.Shape == outputJSON {
+ return io.Discard
+ }
+
+ return out
+}
+
+func runExperiment(
+ ctx context.Context,
+ out io.Writer,
+ definition experiment.Definition,
+ ttl time.Duration,
+ given inputs,
+ gate Gate,
+ emit Output,
+) error {
+ human := emit.human(out)
+
+ ready, err := prepareRun(ctx, human, definition, given)
+ if ready.work != nil {
+ defer ready.work.Close()
+ }
+
if err != nil {
return err
}
+ work := ready.work
+ spec, accepted := ready.spec, ready.accepted
+ project, guard := ready.project, ready.guard
+
env, closeTiers, err := standUpTiers(
ctx,
- out,
+ human,
work,
&spec,
accepted,
@@ -583,21 +710,28 @@ func runExperiment(
session := run.NewSession(env, work.Journal, ttl)
engine := wire(
- out,
+ human,
session,
work.Docker,
dockerx.Filter{Project: project},
guard,
)
- describe(out, spec, accepted.Name)
+ describe(human, spec, accepted.Name)
result, runErr := underLease(ctx, engine, spec, session)
result.RunID = session.RunID()
- reportVerdict(out, result)
- return exitFor(result, runErr)
+ // The exit is decided BEFORE anything is reported, because the record
+ // carries the code: the exit status is the only part of a run most CI
+ // ever sees, and a record that cannot explain it is not a record of the
+ // run.
+ outcome := gated(gate, exitFor(result, runErr))
+
+ reportVerdict(human, result)
+
+ return emitRecord(out, definition, result, outcome, emit)
}
func wire(
@@ -690,45 +824,87 @@ func exitFor(result experiment.Result, runErr error) error {
return runErr
}
+ // The state khaos left the target in outranks every verdict about it, and
+ // no --fail-on value may suppress either.
if result.Observation.RollbackFailed {
- return run.ErrRevertFailed
+ return &Exit{
+ Code: CodeRollback,
+ Outcome: outcomeRollback,
+ Err: run.ErrRevertFailed,
+ }
}
if result.Verdict.Recovery == experiment.Degraded {
- return ErrNotRecovered
+ return &Exit{
+ Code: CodeRollback,
+ Outcome: outcomeRollback,
+ Err: ErrNotRecovered,
+ }
}
+ // The DEFECT outranks the epistemic axis, because a proven bypass is
+ // reported even on thin evidence: one leaked request is a proof.
if result.Verdict.Defect {
- return ErrDefectFound
+ return &Exit{
+ Code: CodeDefect,
+ Outcome: OutcomeDefect,
+ Err: ErrDefectFound,
+ }
}
- if !conclusive(result.Verdict.Epistemic) {
- return fmt.Errorf(
- errInconclusive,
- ErrInconclusive,
- result.Verdict.Epistemic,
- )
+ if code, outcome, matched := epistemicExit(
+ result.Verdict.Epistemic,
+ ); matched {
+ return &Exit{
+ Code: code,
+ Outcome: outcome,
+ Err: fmt.Errorf(
+ errInconclusive,
+ ErrInconclusive,
+ result.Verdict.Epistemic,
+ ),
+ }
}
+ // Deviated in a direction this experiment's polarity does NOT call a
+ // defect: correct, with an availability cost khaos did not measure.
+ if result.Verdict.Epistemic == experiment.Deviated {
+ return &Exit{
+ Code: CodeDeviated,
+ Outcome: OutcomeDeviated,
+ Err: ErrDeviated,
+ }
+ }
+
+ // A HELD verdict khaos declined to call conclusive. Exiting 0 here would
+ // assert a control held on evidence khaos itself refused to stand behind.
if !result.Verdict.Conclusive {
- return fmt.Errorf(errInconclusive, ErrInconclusive, verdictThinEvidence)
+ return &Exit{
+ Code: CodeInconclusive,
+ Outcome: OutcomeInconclusive,
+ Err: fmt.Errorf(
+ errInconclusive,
+ ErrInconclusive,
+ verdictThinEvidence,
+ ),
+ }
}
if result.Verdict.Recovery == experiment.Unobserved {
- return fmt.Errorf(
- errInconclusive,
- ErrInconclusive,
- verdictRecoveryUnobserved,
- )
+ return &Exit{
+ Code: CodeInconclusive,
+ Outcome: OutcomeInconclusive,
+ Err: fmt.Errorf(
+ errInconclusive,
+ ErrInconclusive,
+ verdictRecoveryUnobserved,
+ ),
+ }
}
return nil
}
-func conclusive(epistemic experiment.Epistemic) bool {
- return epistemic == experiment.Passed || epistemic == experiment.Deviated
-}
-
func reportVerdict(out io.Writer, result experiment.Result) {
fmt.Fprintln(out)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
index 34a7ae6f..aad327f0 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -441,9 +441,23 @@ func TestEveryExperimentRegistersExactlyItsDeclaredParameters(t *testing.T) {
}
registered := stringFlags(cmd)
- want := append(
- []string{flagProject},
- names(definition.Params)...)
+
+ // These three belong to `run` alone: `plan` executes
+ // nothing, so it has no outcome to gate and no record to
+ // write, and a flag there would do nothing.
+ extra := []string{flagProject}
+ if verb == "run" {
+ extra = append(
+ extra,
+ flagFailOn,
+ flagRecord,
+ flagOutput,
+ flagJUnit,
+ flagMetricsOut,
+ )
+ }
+
+ want := slices.Concat(extra, names(definition.Params))
slices.Sort(registered)
slices.Sort(want)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
index ffadf81b..a12f8e1f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
@@ -231,7 +231,9 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
timing := spec.Timing.withDefaults()
result := Result{
Experiment: spec.Name,
+ Family: spec.Family,
Hypothesis: spec.Hypothesis.String(),
+ Fault: spec.Fault,
StartedAt: e.now(),
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
index 4e70a26f..31537e00 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -200,9 +200,16 @@ type Experiment struct {
}
type Result struct {
- Experiment string `json:"experiment"`
- RunID string `json:"run_id"`
- Hypothesis string `json:"hypothesis"`
+ Experiment string `json:"experiment"`
+ Family string `json:"family"`
+ RunID string `json:"run_id"`
+ Hypothesis string `json:"hypothesis"`
+
+ // Fault is carried on the RESULT rather than left on the spec, because
+ // the record is the product: a record that cannot say what was injected
+ // or where is not a record of the run.
+ Fault Fault `json:"fault"`
+
StartedAt time.Time `json:"started_at"`
Duration time.Duration `json:"duration"`
Observation Observation `json:"observation"`
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/junit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/junit.go
new file mode 100644
index 00000000..fec604de
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/junit.go
@@ -0,0 +1,260 @@
+// ©AngelaMos | 2026
+// junit.go
+
+package record
+
+import (
+ "encoding/xml"
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+// khaos's declared JUnit flavor. There is no normative specification for this
+// format -- it is what Apache Ant emitted in 2002, extended by Maven Surefire
+// and copied by every runner since -- so the flavor is written down rather
+// than inferred from whatever a parser happened to accept.
+//
+// root <testsuites>, even for one suite. The bare <testsuite>
+// root is the Maven-only shape; more parsers accept this one.
+// pass a <testcase> with NO child element. That is the whole
+// semantic: the outcome is which child is present.
+// failure a security regression, which fails the build.
+// skipped a run khaos declined to conclude from. This is the
+// mechanism that stops thin evidence failing a build.
+// error something wrong with the RUN rather than with the target,
+// visible as a problem without being a security finding.
+const (
+ suiteName = "khaos"
+
+ // Two testcases at most. The first is always the security question; the
+ // second appears only when the run itself had a problem, which is what
+ // gives a reporter the flake-versus-regression split.
+ caseVerdict = "verdict"
+ caseValidity = "experiment validity"
+
+ propRunID = "run_id"
+ propVersion = "khaos_version"
+ propFault = "fault"
+ propTarget = "target"
+ propExit = "exit_code"
+
+ typeDefect = "security-regression"
+ typeRollback = "rollback-failed"
+ typeVoid = "experiment-void"
+
+ messageDefect = "this control failed in the direction %s declares a defect"
+
+ messageRollback = "khaos could not revert the fault; the target may still be degraded"
+
+ messageVoid = "%s: nothing measured during this run can be attributed to the fault"
+
+ messageSkipped = "%s: khaos declined to draw a conclusion from this run"
+
+ // The availability finding rides in system-out rather than in a failure:
+ // a control that refused to serve while its dependency was gone did its
+ // job, and failing a build on correct behaviour trains people to stop
+ // reading the results.
+ noteDeviated = "this control held, and refusing to serve carried an availability cost khaos did not measure"
+)
+
+type testsuites struct {
+ XMLName xml.Name `xml:"testsuites"`
+ Name string `xml:"name,attr"`
+ Tests int `xml:"tests,attr"`
+ Failures int `xml:"failures,attr"`
+ Errors int `xml:"errors,attr"`
+ Skipped int `xml:"skipped,attr"`
+ Time string `xml:"time,attr"`
+ Suites []testsuite `xml:"testsuite"`
+}
+
+type testsuite struct {
+ Name string `xml:"name,attr"`
+ Tests int `xml:"tests,attr"`
+ Failures int `xml:"failures,attr"`
+ Errors int `xml:"errors,attr"`
+ Skipped int `xml:"skipped,attr"`
+ Time string `xml:"time,attr"`
+ Timestamp string `xml:"timestamp,attr"`
+ Props properties `xml:"properties"`
+ Cases []testcase `xml:"testcase"`
+}
+
+type properties struct {
+ Props []property `xml:"property"`
+}
+
+type property struct {
+ Name string `xml:"name,attr"`
+ Value string `xml:"value,attr"`
+}
+
+type testcase struct {
+ Name string `xml:"name,attr"`
+ Classname string `xml:"classname,attr"`
+ Time string `xml:"time,attr"`
+ Failure *detail `xml:"failure,omitempty"`
+ Error *detail `xml:"error,omitempty"`
+ Skipped *skipped `xml:"skipped,omitempty"`
+ SystemOut string `xml:"system-out,omitempty"`
+}
+
+type detail struct {
+ Message string `xml:"message,attr"`
+ Type string `xml:"type,attr"`
+ Body string `xml:",chardata"`
+}
+
+type skipped struct {
+ Message string `xml:"message,attr"`
+}
+
+// JUnit projects the record onto khaos's declared flavor. It takes the record
+// rather than the run, so it cannot see anything the record does not carry.
+func JUnit(rec Record) ([]byte, error) {
+ cases := []testcase{verdictCase(rec)}
+
+ if validity, ok := validityCase(rec); ok {
+ cases = append(cases, validity)
+ }
+
+ suite := testsuite{
+ Name: rec.Experiment.Name,
+ Tests: len(cases),
+ Time: seconds(rec.DurationMS),
+ Timestamp: rec.Started,
+ Props: propertiesOf(rec),
+ Cases: cases,
+ }
+
+ // The counts are DERIVED from the cases rather than tracked alongside
+ // them. A hand-maintained count that disagrees with the body is the
+ // defect this format invites, and a reporter trusts the attribute.
+ for _, each := range cases {
+ switch {
+ case each.Failure != nil:
+ suite.Failures++
+ case each.Error != nil:
+ suite.Errors++
+ case each.Skipped != nil:
+ suite.Skipped++
+ }
+ }
+
+ root := testsuites{
+ Name: suiteName,
+ Tests: suite.Tests,
+ Failures: suite.Failures,
+ Errors: suite.Errors,
+ Skipped: suite.Skipped,
+ Time: suite.Time,
+ Suites: []testsuite{suite},
+ }
+
+ body, err := xml.MarshalIndent(root, "", " ")
+ if err != nil {
+ return nil, err
+ }
+
+ return append(
+ []byte(xml.Header),
+ append(body, '\n')...,
+ ), nil
+}
+
+// verdictCase is the security question. A pass carries no child element,
+// which is the whole semantic of the format.
+func verdictCase(rec Record) testcase {
+ out := testcase{
+ Name: caseVerdict,
+ Classname: rec.Experiment.Name,
+ Time: seconds(rec.DurationMS),
+ }
+
+ reasons := strings.Join(rec.Verdict.Reasons, "\n")
+
+ switch {
+ case rec.Verdict.Defect:
+ out.Failure = &detail{
+ Message: fmt.Sprintf(messageDefect, rec.Experiment.Name),
+ Type: typeDefect,
+ Body: reasons,
+ }
+
+ case concluded(rec):
+ // A deviation the polarity does not call a defect, and a clean hold,
+ // both pass. The availability note distinguishes them for a reader
+ // without failing anybody's build.
+ if rec.Verdict.Direction != "HELD" {
+ out.SystemOut = noteDeviated + "\n" + reasons
+ } else {
+ out.SystemOut = reasons
+ }
+
+ default:
+ out.Skipped = &skipped{
+ Message: fmt.Sprintf(messageSkipped, rec.Verdict.Epistemic),
+ }
+ out.SystemOut = reasons
+ }
+
+ return out
+}
+
+// validityCase appears only when something was wrong with the RUN rather
+// than with the target. Without it a void run is a bare skip, and a skip
+// reads as "nobody got to this" rather than "this could not be concluded".
+func validityCase(rec Record) (testcase, bool) {
+ out := testcase{
+ Name: caseValidity,
+ Classname: rec.Experiment.Name,
+ Time: "0",
+ }
+
+ switch {
+ case rec.Fault.RollbackFailed:
+ out.Error = &detail{
+ Message: messageRollback,
+ Type: typeRollback,
+ Body: strings.Join(rec.Verdict.Reasons, "\n"),
+ }
+
+ case !concluded(rec) && !rec.Verdict.Defect:
+ out.Error = &detail{
+ Message: fmt.Sprintf(messageVoid, rec.Verdict.Epistemic),
+ Type: typeVoid,
+ Body: strings.Join(rec.Verdict.Reasons, "\n"),
+ }
+
+ default:
+ return testcase{}, false
+ }
+
+ return out, true
+}
+
+// concluded reads the record rather than the exit code, because the exit code
+// can be narrowed by --fail-on and the XML must describe the run.
+func concluded(rec Record) bool {
+ if !rec.Verdict.Conclusive {
+ return false
+ }
+
+ return rec.Verdict.Epistemic == "passed" ||
+ rec.Verdict.Epistemic == "deviated"
+}
+
+func propertiesOf(rec Record) properties {
+ return properties{Props: []property{
+ {Name: propRunID, Value: rec.RunID},
+ {Name: propVersion, Value: rec.Khaos},
+ {Name: propFault, Value: rec.Fault.Kind},
+ {Name: propTarget, Value: rec.Fault.Target},
+ {Name: propExit, Value: strconv.Itoa(rec.Exit.Code)},
+ }}
+}
+
+func seconds(ms int64) string {
+ return strconv.FormatFloat(float64(ms)/1000, 'f', 3, 64)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/metrics.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/metrics.go
new file mode 100644
index 00000000..0b8207a8
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/metrics.go
@@ -0,0 +1,207 @@
+// ©AngelaMos | 2026
+// metrics.go
+
+package record
+
+import (
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+)
+
+// Prometheus text exposition, written to a FILE for node_exporter's textfile
+// collector. No client library, no server, no port, no lifecycle.
+//
+// Two rules from the format's own reference, both load-bearing and both
+// omitted by the research that specified this:
+//
+// the last line must end with \n, or the file is a parse failure and a
+// parse failure drops the WHOLE file;
+//
+// only one HELP and one TYPE line may exist per metric name, TYPE must
+// precede the first sample for that name, and all lines for a metric must
+// be one contiguous group.
+//
+// Label values escape backslash, double quote and line feed. HELP text
+// escapes backslash and line feed.
+const (
+ metricPrefix = "khaos_"
+
+ helpInfo = "# HELP " + metricPrefix +
+ "run_info the run this file describes, as labels on a constant 1\n"
+ typeInfo = "# TYPE " + metricPrefix + "run_info gauge\n"
+
+ helpExit = "# HELP " + metricPrefix +
+ "run_exit_code the code khaos returned to the shell\n"
+ typeExit = "# TYPE " + metricPrefix + "run_exit_code gauge\n"
+
+ helpMetricDefect = "# HELP " + metricPrefix +
+ "run_defect 1 when the control failed in the direction this experiment declares a defect\n"
+ typeMetricDefect = "# TYPE " + metricPrefix + "run_defect gauge\n"
+
+ helpConclusive = "# HELP " + metricPrefix +
+ "run_conclusive 1 when khaos stands behind this verdict as conclusive\n"
+ typeConclusive = "# TYPE " + metricPrefix + "run_conclusive gauge\n"
+
+ helpConfirmed = "# HELP " + metricPrefix +
+ "run_fault_confirmed 1 when khaos measured the fault actually taking effect\n"
+ typeConfirmed = "# TYPE " + metricPrefix + "run_fault_confirmed gauge\n"
+
+ helpDuration = "# HELP " + metricPrefix +
+ "run_duration_seconds how long the run took\n"
+ typeDuration = "# TYPE " + metricPrefix + "run_duration_seconds gauge\n"
+
+ helpObservations = "# HELP " + metricPrefix +
+ "observations the probe observations khaos recorded, by window, stream and disposition\n"
+ typeObservations = "# TYPE " + metricPrefix + "observations gauge\n"
+)
+
+// Metrics projects the record onto the text exposition format. It takes the
+// record, so it cannot report anything the record does not carry.
+func Metrics(rec Record) []byte {
+ out := &strings.Builder{}
+
+ // run_info carries the identity as LABELS on a constant 1, which is the
+ // conventional shape: putting a run id or a verdict word in a metric
+ // NAME makes every run a new series.
+ out.WriteString(helpInfo)
+ out.WriteString(typeInfo)
+ fmt.Fprintf(
+ out,
+ "%srun_info{experiment=%s,family=%s,run_id=%s,version=%s,epistemic=%s,direction=%s,cause=%s,outcome=%s} 1\n",
+ metricPrefix,
+ quote(rec.Experiment.Name),
+ quote(rec.Experiment.Family),
+ quote(rec.RunID),
+ quote(rec.Khaos),
+ quote(rec.Verdict.Epistemic),
+ quote(rec.Verdict.Direction),
+ quote(rec.Verdict.Cause),
+ quote(rec.Exit.Outcome),
+ )
+
+ gauge(out, helpExit, typeExit, "run_exit_code", float64(rec.Exit.Code))
+ gauge(
+ out,
+ helpMetricDefect,
+ typeMetricDefect,
+ "run_defect",
+ flag(rec.Verdict.Defect),
+ )
+ gauge(
+ out,
+ helpConclusive,
+ typeConclusive,
+ "run_conclusive",
+ flag(rec.Verdict.Conclusive),
+ )
+ gauge(
+ out,
+ helpConfirmed,
+ typeConfirmed,
+ "run_fault_confirmed",
+ flag(rec.Fault.Confirmed),
+ )
+ gauge(
+ out,
+ helpDuration,
+ typeDuration,
+ "run_duration_seconds",
+ float64(rec.DurationMS)/1000,
+ )
+
+ writeObservations(out, rec)
+
+ return []byte(out.String())
+}
+
+// writeObservations emits one contiguous group for the one series that has
+// labels varying per sample, because the format requires every line for a
+// metric name to be grouped together and each name-plus-labels combination
+// to be unique.
+func writeObservations(out *strings.Builder, rec Record) {
+ type sample struct {
+ key string
+ value int
+ }
+
+ samples := []sample{}
+
+ for window, w := range rec.Windows {
+ for stream, s := range w.Streams {
+ for disposition, count := range s.Distribution {
+ samples = append(samples, sample{
+ key: fmt.Sprintf(
+ "{window=%s,stream=%s,disposition=%s}",
+ quote(window),
+ quote(stream),
+ quote(disposition),
+ ),
+ value: count,
+ })
+ }
+ }
+ }
+
+ if len(samples) == 0 {
+ return
+ }
+
+ // Sorted, because Go map iteration is random and a file whose line order
+ // changes between runs is a file nobody can diff.
+ sort.Slice(samples, func(i, j int) bool {
+ return samples[i].key < samples[j].key
+ })
+
+ out.WriteString(helpObservations)
+ out.WriteString(typeObservations)
+
+ for _, each := range samples {
+ fmt.Fprintf(
+ out,
+ "%sobservations%s %d\n",
+ metricPrefix,
+ each.key,
+ each.value,
+ )
+ }
+}
+
+func gauge(out *strings.Builder, help, typed, name string, value float64) {
+ out.WriteString(help)
+ out.WriteString(typed)
+ fmt.Fprintf(out, "%s%s %s\n", metricPrefix, name, number(value))
+}
+
+// number spells the non-finite values the way the format does, rather than
+// Go's, because `+Inf` and `NaN` are what a parser accepts.
+func number(value float64) string {
+ switch {
+ case value != value:
+ return "NaN"
+ case value > 1e308:
+ return "+Inf"
+ case value < -1e308:
+ return "-Inf"
+ default:
+ return strconv.FormatFloat(value, 'f', -1, 64)
+ }
+}
+
+func flag(set bool) float64 {
+ if set {
+ return 1
+ }
+ return 0
+}
+
+// quote escapes exactly what the format says a label value escapes: the
+// backslash first, or escaping the quote would then double its own slash.
+func quote(value string) string {
+ value = strings.ReplaceAll(value, `\`, `\\`)
+ value = strings.ReplaceAll(value, `"`, `\"`)
+ value = strings.ReplaceAll(value, "\n", `\n`)
+
+ return `"` + value + `"`
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/projection_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/projection_test.go
new file mode 100644
index 00000000..efbbd5de
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/projection_test.go
@@ -0,0 +1,487 @@
+// ©AngelaMos | 2026
+// projection_test.go
+
+package record
+
+import (
+ "encoding/xml"
+ "strings"
+ "testing"
+)
+
+// shaped builds a record in one outcome, so each test states the one thing
+// it varies rather than rebuilding the world.
+func shaped(
+ epistemic, direction string,
+ defect, conclusive bool,
+) Record {
+ return Record{
+ Schema: Version,
+ Khaos: "1.2.3",
+ RunID: "01KRUN0000000000000000000",
+ Started: "2026-10-21T09:00:00Z",
+ Ended: "2026-10-21T09:00:30Z",
+ DurationMS: 30_000,
+ Experiment: Experiment{
+ Name: "control.policy_store_down",
+ Family: "control",
+ },
+ Fault: Fault{
+ Kind: "proxy-reset",
+ Target: "khaos-range-gateway-control",
+ Confirmed: true,
+ Method: "a measured observable",
+ },
+ Windows: map[string]Window{
+ "fault": {
+ Paired: 30,
+ Requests: 60,
+ Streams: map[string]Stream{
+ "invalid": {
+ Countable: 30,
+ Answered: 30,
+ Distribution: map[string]int{"accepted": 30},
+ },
+ },
+ },
+ },
+ Verdict: Verdict{
+ Epistemic: epistemic,
+ Direction: direction,
+ Cause: "UNDIAGNOSED",
+ Recovery: "recovered",
+ Conclusive: conclusive,
+ Defect: defect,
+ Reasons: []string{"a reason"},
+ },
+ Exit: Exit{Code: 2, Outcome: "defect"},
+ }
+}
+
+func parseSuites(t *testing.T, body []byte) testsuites {
+ t.Helper()
+
+ var root testsuites
+ if err := xml.Unmarshal(body, &root); err != nil {
+ t.Fatalf("the XML khaos wrote does not parse: %v", err)
+ }
+
+ return root
+}
+
+// The declared flavor: <testsuites> root even for one suite, because the
+// bare <testsuite> root is the Maven-only shape and fewer parsers take it.
+func TestTheJUnitRootIsTestsuitesEvenForOneSuite(t *testing.T) {
+ body, err := JUnit(shaped("deviated", "PERMITTED", true, true))
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ if !strings.Contains(string(body), "<?xml") {
+ t.Error("the file carries no XML declaration")
+ }
+
+ if !strings.Contains(string(body), "<testsuites") {
+ t.Fatalf("the root is not <testsuites>: %.120q", body)
+ }
+
+ root := parseSuites(t, body)
+ if len(root.Suites) != 1 {
+ t.Fatalf("got %d suites, want 1", len(root.Suites))
+ }
+}
+
+// The whole semantic of the format: an outcome is WHICH CHILD is present,
+// and a pass has none.
+func TestTheOutcomeIsWhichChildElementIsPresent(t *testing.T) {
+ for _, row := range []struct {
+ name string
+ epistemic string
+ direction string
+ defect bool
+ conclusive bool
+ failure bool
+ skip bool
+ }{
+ {
+ "a defect fails the build",
+ "deviated", "PERMITTED", true, true,
+ true, false,
+ },
+ {
+ "a clean hold passes with no child",
+ "passed", "HELD", false, true,
+ false, false,
+ },
+ {
+ "a non-defect deviation passes with no child",
+ "deviated", "DENIED", false, true,
+ false, false,
+ },
+ {
+ "thin evidence is skipped, never failed",
+ "passed", "HELD", false, false,
+ false, true,
+ },
+ {
+ "a void run is skipped, never failed",
+ "aborted", "UNDETERMINED", false, false,
+ false, true,
+ },
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ body, err := JUnit(shaped(
+ row.epistemic,
+ row.direction,
+ row.defect,
+ row.conclusive,
+ ))
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ verdict := parseSuites(t, body).Suites[0].Cases[0]
+
+ if (verdict.Failure != nil) != row.failure {
+ t.Errorf(
+ "failure present = %v, want %v",
+ verdict.Failure != nil,
+ row.failure,
+ )
+ }
+
+ if (verdict.Skipped != nil) != row.skip {
+ t.Errorf(
+ "skipped present = %v, want %v",
+ verdict.Skipped != nil,
+ row.skip,
+ )
+ }
+
+ // A pass is the ABSENCE of all three, which is the one thing a
+ // reporter reads.
+ if !row.failure && !row.skip {
+ if verdict.Failure != nil ||
+ verdict.Skipped != nil ||
+ verdict.Error != nil {
+ t.Error("a passing testcase carries a child element")
+ }
+ }
+ })
+ }
+}
+
+// Thin evidence must never be a <failure>: that is the specific mechanism
+// that stops a run khaos declined to conclude from breaking a build.
+func TestThinEvidenceIsSkippedRatherThanFailed(t *testing.T) {
+ body, err := JUnit(shaped("passed", "HELD", false, false))
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ root := parseSuites(t, body)
+
+ if root.Failures != 0 {
+ t.Fatalf("a thin-evidence run reports %d failures", root.Failures)
+ }
+
+ if root.Skipped != 1 {
+ t.Fatalf("a thin-evidence run reports %d skips, want 1", root.Skipped)
+ }
+}
+
+// A rollback failure is an <error>: visible as a problem without being
+// reported as a security regression, which it is not.
+func TestARollbackFailureIsAnErrorRatherThanAFailure(t *testing.T) {
+ rec := shaped("passed", "HELD", false, true)
+ rec.Fault.RollbackFailed = true
+
+ body, err := JUnit(rec)
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ root := parseSuites(t, body)
+
+ if root.Errors != 1 {
+ t.Fatalf("got %d errors, want 1", root.Errors)
+ }
+
+ if root.Failures != 0 {
+ t.Fatalf("a rollback failure was reported as a security regression")
+ }
+}
+
+// The counts on the suite are derived from the cases. A hand-maintained
+// count that disagrees with the body is the defect this format invites, and
+// a reporter trusts the attribute over the elements.
+func TestTheSuiteCountsMatchTheCasesItCarries(t *testing.T) {
+ for _, rec := range []Record{
+ shaped("deviated", "PERMITTED", true, true),
+ shaped("passed", "HELD", false, true),
+ shaped("aborted", "UNDETERMINED", false, false),
+ } {
+ body, err := JUnit(rec)
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ root := parseSuites(t, body)
+ suite := root.Suites[0]
+
+ failures, errs, skips := 0, 0, 0
+ for _, each := range suite.Cases {
+ switch {
+ case each.Failure != nil:
+ failures++
+ case each.Error != nil:
+ errs++
+ case each.Skipped != nil:
+ skips++
+ }
+ }
+
+ if suite.Tests != len(suite.Cases) {
+ t.Errorf(
+ "suite says %d tests and carries %d cases",
+ suite.Tests,
+ len(suite.Cases),
+ )
+ }
+
+ if suite.Failures != failures ||
+ suite.Errors != errs ||
+ suite.Skipped != skips {
+ t.Errorf(
+ "suite says %d/%d/%d failures/errors/skips and carries %d/%d/%d",
+ suite.Failures,
+ suite.Errors,
+ suite.Skipped,
+ failures,
+ errs,
+ skips,
+ )
+ }
+ }
+}
+
+// A JUnit-only consumer has to be able to get back to the record.
+func TestThePropertiesCarryTheWayBackToTheRecord(t *testing.T) {
+ body, err := JUnit(shaped("deviated", "PERMITTED", true, true))
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ props := map[string]string{}
+ for _, each := range parseSuites(t, body).Suites[0].Props.Props {
+ props[each.Name] = each.Value
+ }
+
+ for _, name := range []string{
+ propRunID,
+ propVersion,
+ propFault,
+ propTarget,
+ propExit,
+ } {
+ if props[name] == "" {
+ t.Errorf("the suite carries no %s property", name)
+ }
+ }
+}
+
+// XML has to survive content that would otherwise break it, and a verdict
+// reason is free text khaos did not write.
+func TestTheXMLSurvivesHostileReasonText(t *testing.T) {
+ rec := shaped("deviated", "PERMITTED", true, true)
+ rec.Verdict.Reasons = []string{
+ `a reason with <angle> & "quotes" and ]]> in it`,
+ }
+
+ body, err := JUnit(rec)
+ if err != nil {
+ t.Fatalf("junit: %v", err)
+ }
+
+ root := parseSuites(t, body)
+
+ if !strings.Contains(
+ root.Suites[0].Cases[0].Failure.Body,
+ "<angle>",
+ ) {
+ t.Fatalf(
+ "the reason did not survive a round trip: %q",
+ root.Suites[0].Cases[0].Failure.Body,
+ )
+ }
+}
+
+// The exposition format's normative requirement: the last line must end with
+// a line feed, and without it the WHOLE file is a parse failure.
+func TestTheMetricsFileEndsWithALineFeed(t *testing.T) {
+ body := Metrics(shaped("deviated", "PERMITTED", true, true))
+
+ if len(body) == 0 {
+ t.Fatal("the metrics file is empty")
+ }
+
+ if body[len(body)-1] != '\n' {
+ t.Fatalf("the metrics file does not end with a line feed: %q",
+ string(body[max(0, len(body)-20):]))
+ }
+
+ // EVERY line, not just the last byte. Checking only the tail cannot see
+ // an unterminated interior line: dropping the newline from the gauge
+ // helper left the file still ending in one, because the observation
+ // group is written afterwards and terminates itself.
+ text := string(body)
+ for _, line := range strings.Split(strings.TrimRight(text, "\n"), "\n") {
+ if line == "" {
+ t.Fatal("the metrics file carries an empty line")
+ }
+
+ fields := strings.Fields(line)
+
+ if strings.HasPrefix(line, "#") {
+ continue
+ }
+
+ // A sample is exactly `name[{labels}] value`. A line carrying two
+ // samples is what a dropped newline actually produces, and it
+ // parses as a value with trailing junk rather than as an error.
+ if len(fields) != 2 {
+ t.Fatalf(
+ "a sample line does not have exactly a name and a value, so a newline was dropped: %q",
+ line,
+ )
+ }
+ }
+}
+
+// Only one HELP and one TYPE per metric name, and TYPE before the first
+// sample for that name.
+func TestEveryMetricIsDeclaredOnceAndBeforeItsFirstSample(t *testing.T) {
+ body := string(Metrics(shaped("deviated", "PERMITTED", true, true)))
+
+ helps := map[string]int{}
+ types := map[string]int{}
+ declared := map[string]bool{}
+
+ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {
+ fields := strings.Fields(line)
+
+ switch {
+ case strings.HasPrefix(line, "# HELP"):
+ helps[fields[2]]++
+ case strings.HasPrefix(line, "# TYPE"):
+ types[fields[2]]++
+ declared[fields[2]] = true
+ case strings.HasPrefix(line, "#"):
+ default:
+ name := strings.SplitN(fields[0], "{", 2)[0]
+ if !declared[name] {
+ t.Errorf("%s is sampled before its TYPE line", name)
+ }
+ }
+ }
+
+ for name, count := range helps {
+ if count != 1 {
+ t.Errorf("%s has %d HELP lines, want 1", name, count)
+ }
+ }
+
+ for name, count := range types {
+ if count != 1 {
+ t.Errorf("%s has %d TYPE lines, want 1", name, count)
+ }
+ }
+}
+
+// Each name-plus-labels combination must be unique, or the format says the
+// ingestion behaviour is undefined.
+func TestEverySampleLineIsUnique(t *testing.T) {
+ body := string(Metrics(shaped("deviated", "PERMITTED", true, true)))
+
+ seen := map[string]bool{}
+
+ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {
+ if strings.HasPrefix(line, "#") || line == "" {
+ continue
+ }
+
+ key := strings.Fields(line)[0]
+ if seen[key] {
+ t.Errorf("%s is sampled twice", key)
+ }
+
+ seen[key] = true
+ }
+}
+
+// All lines for one metric name have to be one contiguous group.
+func TestSamplesForOneMetricAreContiguous(t *testing.T) {
+ body := string(Metrics(shaped("deviated", "PERMITTED", true, true)))
+
+ order := []string{}
+ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {
+ if strings.HasPrefix(line, "#") || line == "" {
+ continue
+ }
+
+ name := strings.SplitN(strings.Fields(line)[0], "{", 2)[0]
+ if len(order) == 0 || order[len(order)-1] != name {
+ order = append(order, name)
+ }
+ }
+
+ seen := map[string]bool{}
+ for _, name := range order {
+ if seen[name] {
+ t.Errorf("%s appears in two separate groups", name)
+ }
+ seen[name] = true
+ }
+}
+
+// A label value khaos did not write has to be escaped, or one quote in a
+// reason makes the file unparseable.
+func TestLabelValuesAreEscaped(t *testing.T) {
+ rec := shaped("deviated", "PERMITTED", true, true)
+ rec.Experiment.Name = `a"name\with` + "\n" + "breaks"
+
+ body := string(Metrics(rec))
+
+ if strings.Contains(body, `a"name`) {
+ t.Fatal("an unescaped double quote reached a label value")
+ }
+
+ if !strings.Contains(body, `a\"name\\with\nbreaks`) {
+ t.Fatalf(
+ "the label was not escaped as the format requires: %.200q",
+ body,
+ )
+ }
+
+ // And the file still has one line per sample: an unescaped newline would
+ // have split one into two.
+ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {
+ if line == "" {
+ t.Fatal("an unescaped line feed split a sample line")
+ }
+ }
+}
+
+// The sample order is stable, because Go map iteration is random and a file
+// whose line order changes between identical runs is a file nobody can diff.
+func TestTheMetricsFileIsByteStableAcrossRuns(t *testing.T) {
+ rec := shaped("deviated", "PERMITTED", true, true)
+
+ first := string(Metrics(rec))
+
+ for range 8 {
+ if got := string(Metrics(rec)); got != first {
+ t.Fatal("two renders of one record produced different bytes")
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go
new file mode 100644
index 00000000..6840bec0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go
@@ -0,0 +1,248 @@
+// ©AngelaMos | 2026
+// record.go
+
+package record
+
+import (
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ // Version is the record's own schema version, independent of khaos's. A
+ // consumer that parses this file needs to know when its shape changed,
+ // and the binary's version moves for reasons that do not touch it.
+ Version = 1
+
+ // EventsFile sits beside the record rather than inside it. A sixty-second
+ // run at 300ms over four streams is roughly 800 observations: embedded
+ // they make the record impossible to read or paste into a ticket, and
+ // discarded they make the verdict unauditable.
+ RecordFile = "run.json"
+ EventsFile = "events.jsonl"
+
+ // RFC 3339 with nanoseconds, so events at a 300ms interval sort
+ // lexicographically as well as chronologically.
+ timeLayout = time.RFC3339Nano
+)
+
+// Record is the canonical artifact. Every other output khaos emits is a
+// projection of it, which is asserted rather than asserted-about: if the
+// terminal can show something this struct does not carry, the record is
+// incomplete and a test fails.
+//
+// It carries ONLY what khaos measured. Research 08's schema also lists host
+// details, compose-file hashes, image digests, confidence intervals and a
+// probe footprint; khaos measures none of those, and an empty key is a claim
+// that something was looked at. The gap is documented, not papered over.
+type Record struct {
+ Schema int `json:"schema_version"`
+ Khaos string `json:"khaos_version"`
+ RunID string `json:"run_id"`
+ Started string `json:"started_at"`
+ Ended string `json:"ended_at"`
+
+ DurationMS int64 `json:"duration_ms"`
+
+ Experiment Experiment `json:"experiment"`
+ Fault Fault `json:"fault"`
+
+ Windows map[string]Window `json:"windows"`
+
+ Verdict Verdict `json:"verdict"`
+
+ // Exit is what khaos returned to the shell, in the record, because the
+ // exit code is the only part of a run most CI ever sees and a record that
+ // cannot explain it is not a record of the run.
+ Exit Exit `json:"exit"`
+
+ // Events names the file beside this one and the hash of its bytes, so a
+ // record cannot be silently separated from the observations it summarises.
+ Events *Events `json:"events,omitempty"`
+}
+
+type Experiment struct {
+ Name string `json:"name"`
+ Family string `json:"family"`
+ Hypothesis string `json:"hypothesis"`
+
+ // Defects is the directions THIS experiment declares a defect, which is
+ // what makes the verdict's direction readable: the same direction is a
+ // finding for one control and correct for another.
+ Defects []string `json:"declared_defects"`
+ Rationale string `json:"polarity_rationale,omitempty"`
+}
+
+type Fault struct {
+ Kind string `json:"kind"`
+ Target string `json:"target"`
+
+ // Confirmed is REQUIRED for any verdict khaos will stand behind: an
+ // injection the API accepted is not a fault that took effect, and a run
+ // measuring no fault at all reads as a pass.
+ Confirmed bool `json:"confirmed"`
+ Method string `json:"confirm_method,omitempty"`
+
+ HeldFor int64 `json:"held_for_ms,omitempty"`
+
+ RollbackFailed bool `json:"rollback_failed"`
+}
+
+// Window is one measurement window, aggregated. The raw per-observation data
+// is in the events file.
+type Window struct {
+ Paired int `json:"paired"`
+ Requests int `json:"requests"`
+ Streams map[string]Stream `json:"streams"`
+}
+
+type Stream struct {
+ Countable int `json:"countable"`
+ Answered int `json:"answered"`
+ Distribution map[string]int `json:"distribution"`
+}
+
+type Verdict struct {
+ Epistemic string `json:"epistemic"`
+ Direction string `json:"direction"`
+ Cause string `json:"cause"`
+ Recovery string `json:"recovery"`
+ Conclusive bool `json:"conclusive"`
+ Defect bool `json:"defect"`
+ Reasons []string `json:"reasons"`
+ Warnings []string `json:"warnings,omitempty"`
+}
+
+type Exit struct {
+ Code int `json:"code"`
+ Outcome string `json:"outcome,omitempty"`
+}
+
+type Events struct {
+ File string `json:"file"`
+ SHA256 string `json:"sha256"`
+ Count int `json:"count"`
+}
+
+// Of projects a finished run onto the record. It is the ONE place the shape
+// is decided, so the stdout form and the files cannot drift from each other.
+func Of(
+ version string,
+ result experiment.Result,
+ definition experiment.Definition,
+ code int,
+ outcome string,
+) Record {
+ ended := result.StartedAt.Add(result.Duration)
+
+ return Record{
+ Schema: Version,
+ Khaos: version,
+ RunID: result.RunID,
+ Started: result.StartedAt.UTC().Format(time.RFC3339Nano),
+ Ended: ended.UTC().Format(time.RFC3339Nano),
+ DurationMS: result.Duration.Milliseconds(),
+ Experiment: Experiment{
+ Name: result.Experiment,
+ Family: result.Family,
+ Hypothesis: result.Hypothesis,
+ Defects: directions(definition.Polarity.Defects),
+ Rationale: definition.Polarity.Rationale,
+ },
+ Fault: Fault{
+ Kind: result.Fault.Kind,
+ Target: result.Fault.Target,
+ Confirmed: result.Observation.FaultConfirmed,
+ Method: result.Observation.ConfirmMethod,
+ HeldFor: result.Observation.FaultFor.Milliseconds(),
+ RollbackFailed: result.Observation.RollbackFailed,
+ },
+ Windows: windows(result.Observation),
+ Verdict: Verdict{
+ Epistemic: string(result.Verdict.Epistemic),
+ Direction: string(result.Verdict.Direction),
+ Cause: string(result.Verdict.Cause),
+ Recovery: string(result.Verdict.Recovery),
+ Conclusive: result.Verdict.Conclusive,
+ Defect: result.Verdict.Defect,
+ Reasons: result.Verdict.Reasons,
+ Warnings: warnings(result.Verdict.Warnings),
+ },
+ Exit: Exit{Code: code, Outcome: outcome},
+ }
+}
+
+func directions(from []experiment.Direction) []string {
+ out := make([]string, 0, len(from))
+ for _, direction := range from {
+ out = append(out, string(direction))
+ }
+ return out
+}
+
+func warnings(from []experiment.Warning) []string {
+ out := make([]string, 0, len(from))
+ for _, warning := range from {
+ out = append(out, string(warning))
+ }
+ return out
+}
+
+// windows omits a window that was never measured rather than recording it as
+// zero: "khaos did not get here" and "khaos measured nothing here" are
+// different facts, and a reader cannot tell them apart from a zero.
+func windows(observation experiment.Observation) map[string]Window {
+ out := map[string]Window{}
+
+ for name, window := range map[string]probe.Window{
+ "baseline": observation.Baseline,
+ "fault": observation.Fault,
+ "recovery": observation.Recovery,
+ "resumed": observation.Resumed,
+ } {
+ if window.Len() == 0 {
+ continue
+ }
+
+ out[name] = aggregate(window)
+ }
+
+ return out
+}
+
+func aggregate(window probe.Window) Window {
+ out := Window{
+ Paired: window.Paired(),
+ Requests: window.Requests(),
+ Streams: map[string]Stream{},
+ }
+
+ for name, stream := range map[string]probe.Stream{
+ "valid": window.Valid,
+ "invalid": window.Invalid,
+ "revoked": window.Revoked,
+ "forged": window.Forged,
+ } {
+ if stream.Len() == 0 {
+ continue
+ }
+
+ out.Streams[name] = Stream{
+ Countable: stream.Countable(),
+ Answered: stream.Answered(),
+ Distribution: distribution(stream),
+ }
+ }
+
+ return out
+}
+
+func distribution(stream probe.Stream) map[string]int {
+ out := map[string]int{}
+ for disposition, count := range stream.Distribution() {
+ out[string(disposition)] = count
+ }
+ return out
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go
new file mode 100644
index 00000000..7dabc7ab
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go
@@ -0,0 +1,518 @@
+// ©AngelaMos | 2026
+// record_test.go
+
+package record
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+func outcome(
+ at time.Time,
+ credential probe.Credential,
+ disposition probe.Disposition,
+) probe.Outcome {
+ return probe.Outcome{
+ At: at,
+ Credential: credential,
+ Disposition: disposition,
+ LatencyMS: 7,
+ }
+}
+
+// finished is a run with every window populated, so a test that drops a
+// window or a stream fails instead of quietly measuring a smaller shape.
+func finished() experiment.Result {
+ base := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+
+ window := func(offset time.Duration, invalid probe.Disposition) probe.Window {
+ out := probe.Window{}
+
+ for i := range 3 {
+ at := base.Add(offset + time.Duration(i)*time.Second)
+ out.Record(outcome(at, probe.Valid, probe.Accepted))
+ out.Record(outcome(at, probe.Invalid, invalid))
+ }
+
+ return out
+ }
+
+ return experiment.Result{
+ Experiment: "control.policy_store_down",
+ Family: "control",
+ RunID: "01KRUN0000000000000000000",
+ Hypothesis: "a hypothesis sentence",
+ Fault: experiment.Fault{
+ Kind: "proxy-reset",
+ Target: "khaos-range-gateway-control",
+ },
+ StartedAt: base,
+ Duration: 30 * time.Second,
+ Observation: experiment.Observation{
+ Baseline: window(0, probe.Rejected),
+ Fault: window(10*time.Second, probe.Accepted),
+ Recovery: window(20*time.Second, probe.Rejected),
+ FaultConfirmed: true,
+ ConfirmMethod: "a measured observable",
+ FaultFor: 20 * time.Second,
+ },
+ Verdict: experiment.Verdict{
+ Epistemic: experiment.Deviated,
+ Direction: experiment.Permitted,
+ Recovery: experiment.Recovered,
+ Conclusive: true,
+ Defect: true,
+ Reasons: []string{"a reason"},
+ },
+ }
+}
+
+func definition(t *testing.T) experiment.Definition {
+ t.Helper()
+
+ found, err := experiment.Lookup("control.policy_store_down")
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ return found
+}
+
+// The record has to carry what the terminal shows. These are the fields a
+// reader cannot reconstruct from anything else, and a record missing one is
+// a record that cannot answer why khaos said what it said.
+func TestTheRecordCarriesWhatTheVerdictRestsOn(t *testing.T) {
+ rec := Of("1.2.3", finished(), definition(t), 2, "defect")
+
+ for name, got := range map[string]any{
+ "schema version": rec.Schema,
+ "khaos version": rec.Khaos,
+ "run id": rec.RunID,
+ "started at": rec.Started,
+ "ended at": rec.Ended,
+ "experiment name": rec.Experiment.Name,
+ "family": rec.Experiment.Family,
+ "hypothesis": rec.Experiment.Hypothesis,
+ "fault kind": rec.Fault.Kind,
+ "fault target": rec.Fault.Target,
+ "confirm method": rec.Fault.Method,
+ "epistemic": rec.Verdict.Epistemic,
+ "direction": rec.Verdict.Direction,
+ "recovery": rec.Verdict.Recovery,
+ "exit code": rec.Exit.Code,
+ "exit outcome": rec.Exit.Outcome,
+ } {
+ switch value := got.(type) {
+ case string:
+ if value == "" {
+ t.Errorf("the record carries no %s", name)
+ }
+ case int:
+ if value == 0 && name != "exit code" {
+ t.Errorf("the record carries no %s", name)
+ }
+ }
+ }
+
+ if !rec.Fault.Confirmed {
+ t.Error("the record does not say the fault was confirmed")
+ }
+
+ if !rec.Verdict.Defect {
+ t.Error("the record does not carry the defect flag")
+ }
+
+ // The DECLARED defects, because the same direction is a finding for one
+ // control and correct for another: a reader with the direction alone
+ // cannot tell which they are looking at.
+ if len(rec.Experiment.Defects) == 0 {
+ t.Error(
+ "the record does not say which directions this experiment calls a defect",
+ )
+ }
+
+ if rec.Experiment.Rationale == "" {
+ t.Error("the record does not carry the polarity's rationale")
+ }
+}
+
+// A window khaos never reached is ABSENT, not zero. "khaos did not get here"
+// and "khaos measured nothing here" are different facts and a reader cannot
+// tell them apart from a zero.
+func TestAnUnmeasuredWindowIsAbsentRatherThanZero(t *testing.T) {
+ result := finished()
+ result.Observation.Recovery = probe.Window{}
+
+ rec := Of("1.2.3", result, definition(t), 2, "defect")
+
+ if _, present := rec.Windows["recovery"]; present {
+ t.Fatal("a window that was never measured is recorded as zero")
+ }
+
+ for _, name := range []string{"baseline", "fault"} {
+ if _, present := rec.Windows[name]; !present {
+ t.Fatalf("the %s window is missing from the record", name)
+ }
+ }
+
+ // And a stream nothing drove is absent for the same reason.
+ if _, present := rec.Windows["baseline"].Streams["forged"]; present {
+ t.Fatal("a stream that never fired is recorded as zero")
+ }
+}
+
+func TestTheWindowAggregatesMatchTheObservations(t *testing.T) {
+ rec := Of("1.2.3", finished(), definition(t), 2, "defect")
+
+ fault := rec.Windows["fault"]
+
+ if fault.Paired != 3 {
+ t.Errorf("fault paired = %d, want 3", fault.Paired)
+ }
+
+ // Six requests over two streams, which is the number the probe footprint
+ // is read from and the number a reader checks the verdict against.
+ if fault.Requests != 6 {
+ t.Errorf("fault requests = %d, want 6", fault.Requests)
+ }
+
+ invalid := fault.Streams["invalid"]
+ if invalid.Distribution["accepted"] != 3 {
+ t.Errorf(
+ "the fault window's invalid stream shows %v, want 3 accepted",
+ invalid.Distribution,
+ )
+ }
+
+ baseline := rec.Windows["baseline"].Streams["invalid"]
+ if baseline.Distribution["rejected"] != 3 {
+ t.Errorf(
+ "the baseline's invalid stream shows %v, want 3 rejected",
+ baseline.Distribution,
+ )
+ }
+}
+
+// Every observation reaches the events file. A record that summarises 800
+// observations and ships 600 of them is unauditable in the way that matters:
+// the missing ones are invisible.
+func TestEveryObservationReachesTheEventsFile(t *testing.T) {
+ result := finished()
+ events := EventsOf(result.Observation)
+
+ // Requests, not Len: Len is Paired, and one paired sample is at least
+ // two requests. Counting pairs here would have asserted half the
+ // observations reached the file and called that complete.
+ want := result.Observation.Baseline.Requests() +
+ result.Observation.Fault.Requests() +
+ result.Observation.Recovery.Requests()
+
+ if len(events) != want {
+ t.Fatalf("wrote %d events for %d observations", len(events), want)
+ }
+
+ windows := map[string]int{}
+ for _, event := range events {
+ windows[event.Window]++
+
+ if event.At == "" || event.Stream == "" || event.Disposition == "" {
+ t.Fatalf("an event is missing a field: %+v", event)
+ }
+ }
+
+ for _, name := range []string{"baseline", "fault", "recovery"} {
+ if windows[name] == 0 {
+ t.Errorf("no events carry the window %q", name)
+ }
+ }
+}
+
+// Within a window the streams are recorded one after another, so a naive
+// concatenation reads as four separate runs rather than one interleaved one.
+func TestEventsWithinAWindowAreTimeOrdered(t *testing.T) {
+ events := EventsOf(finished().Observation)
+
+ for i := 1; i < len(events); i++ {
+ if events[i-1].Window != events[i].Window {
+ continue
+ }
+
+ if events[i-1].At > events[i].At {
+ t.Fatalf(
+ "events in the %s window are out of order: %s then %s",
+ events[i].Window,
+ events[i-1].At,
+ events[i].At,
+ )
+ }
+ }
+}
+
+// The hash is what stops a record being silently separated from its data, so
+// it has to be the hash of the bytes that actually landed.
+func TestTheRecordHashesTheEventsItActuallyWrote(t *testing.T) {
+ dir := t.TempDir()
+ result := finished()
+
+ written, err := Write(
+ dir,
+ Of("1.2.3", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ )
+ if err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if written.Events == nil {
+ t.Fatal("the record carries no events reference")
+ }
+
+ body, err := os.ReadFile(filepath.Join(dir, EventsFile))
+ if err != nil {
+ t.Fatalf("read the events file: %v", err)
+ }
+
+ sum := sha256.Sum256(body)
+ if got := hex.EncodeToString(sum[:]); got != written.Events.SHA256 {
+ t.Fatalf(
+ "the record claims sha256 %s and the file on disk hashes to %s",
+ written.Events.SHA256,
+ got,
+ )
+ }
+
+ lines := strings.Count(strings.TrimRight(string(body), "\n"), "\n") + 1
+ if lines != written.Events.Count {
+ t.Fatalf(
+ "the record claims %d events and the file has %d lines",
+ written.Events.Count,
+ lines,
+ )
+ }
+}
+
+// The record on disk must parse back into the same shape, or every
+// projection built on it is built on a guess.
+func TestTheWrittenRecordParsesBackIdentically(t *testing.T) {
+ dir := t.TempDir()
+ result := finished()
+
+ written, err := Write(
+ dir,
+ Of("1.2.3", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ )
+ if err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ body, err := os.ReadFile(filepath.Join(dir, RecordFile))
+ if err != nil {
+ t.Fatalf("read the record: %v", err)
+ }
+
+ var parsed Record
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ t.Fatalf("the record khaos wrote is not valid JSON: %v", err)
+ }
+
+ first, err := Encode(written)
+ if err != nil {
+ t.Fatalf("encode: %v", err)
+ }
+
+ second, err := Encode(parsed)
+ if err != nil {
+ t.Fatalf("re-encode: %v", err)
+ }
+
+ if string(first) != string(second) {
+ t.Fatal("the record does not survive a parse-and-re-encode round trip")
+ }
+}
+
+// Every events line is its own JSON object. A single malformed line makes
+// the whole file useless to a streaming reader, which is the only kind of
+// reader a 1,200-line file gets.
+func TestEveryEventsLineIsItsOwnObject(t *testing.T) {
+ dir := t.TempDir()
+ result := finished()
+
+ if _, err := Write(
+ dir,
+ Of("1.2.3", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ ); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ body, err := os.ReadFile(filepath.Join(dir, EventsFile))
+ if err != nil {
+ t.Fatalf("read: %v", err)
+ }
+
+ if !strings.HasSuffix(string(body), "\n") {
+ t.Fatal(
+ "the events file does not end in a newline, so its last line is unterminated",
+ )
+ }
+
+ for i, line := range strings.Split(strings.TrimRight(string(body), "\n"), "\n") {
+ var event Event
+ if err := json.Unmarshal([]byte(line), &event); err != nil {
+ t.Fatalf("events line %d is not an object: %v", i+1, err)
+ }
+ }
+}
+
+// Atomicity is the requirement that fails silently: a reader that opens a
+// half-written file gets no error, just a truncated record. Proven by there
+// being no leftover temporary file and by the destination never existing in
+// a partial state, which is what rename-into-place buys.
+func TestTheWriteLeavesNoPartialFileBehind(t *testing.T) {
+ dir := t.TempDir()
+ result := finished()
+
+ if _, err := Write(
+ dir,
+ Of("1.2.3", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ ); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatalf("read the directory: %v", err)
+ }
+
+ names := []string{}
+ for _, entry := range entries {
+ names = append(names, entry.Name())
+
+ if strings.HasPrefix(entry.Name(), ".khaos-") {
+ t.Errorf(
+ "a temporary file survived the write: %s",
+ entry.Name(),
+ )
+ }
+ }
+
+ if len(names) != 2 {
+ t.Fatalf(
+ "the directory holds %v, want exactly the record and its events",
+ names,
+ )
+ }
+}
+
+// A second run into the same directory replaces the pair rather than failing
+// or appending, because re-running one experiment is the normal thing to do.
+func TestWritingTwiceReplacesTheRecord(t *testing.T) {
+ dir := t.TempDir()
+ result := finished()
+
+ for range 2 {
+ if _, err := Write(
+ dir,
+ Of("1.2.3", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ ); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+ }
+
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatalf("read: %v", err)
+ }
+
+ if len(entries) != 2 {
+ t.Fatalf("a second write left %d files, want 2", len(entries))
+ }
+}
+
+// Rename-into-place versus truncate-in-place, discriminated by the INODE.
+// This is the half of atomicity a single-threaded test can actually prove: a
+// truncating write reuses the destination's inode and is visible to a reader
+// in a partial state, while a rename installs a different inode whose
+// contents were complete before it became reachable under that name.
+func TestTheRecordIsRenamedIntoPlaceRatherThanTruncated(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, RecordFile)
+ result := finished()
+
+ // A first write, so the destination already exists and a truncating
+ // implementation has something to truncate.
+ if _, err := Write(
+ dir,
+ Of("1.2.3", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ ); err != nil {
+ t.Fatalf("first write: %v", err)
+ }
+
+ before, err := inodeOf(path)
+ if err != nil {
+ t.Fatalf("stat before: %v", err)
+ }
+
+ if _, err := Write(
+ dir,
+ Of("9.9.9", result, definition(t), 2, "defect"),
+ EventsOf(result.Observation),
+ ); err != nil {
+ t.Fatalf("second write: %v", err)
+ }
+
+ after, err := inodeOf(path)
+ if err != nil {
+ t.Fatalf("stat after: %v", err)
+ }
+
+ if before == after {
+ t.Fatal(
+ "the record kept its inode across a rewrite, so it was written in place; a reader can see it half-written and gets no error saying so",
+ )
+ }
+
+ // And the new content is actually there, or "it moved" is meaningless.
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read: %v", err)
+ }
+
+ if !strings.Contains(string(body), "9.9.9") {
+ t.Fatal("the second write did not replace the record's contents")
+ }
+}
+
+func inodeOf(path string) (uint64, error) {
+ info, err := os.Stat(path)
+ if err != nil {
+ return 0, err
+ }
+
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return 0, errNoInode
+ }
+
+ return stat.Ino, nil
+}
+
+var errNoInode = errors.New(
+ "this platform does not expose an inode, so the rename cannot be distinguished from a truncation here",
+)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go
new file mode 100644
index 00000000..db1ac666
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go
@@ -0,0 +1,248 @@
+// ©AngelaMos | 2026
+// write.go
+
+package record
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+
+ "github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const (
+ dirMode = 0o755
+
+ tempPattern = ".khaos-*"
+
+ errWriteDir = "create the directory for the run record: %w"
+ errWriteFile = "write %s: %w"
+ errEncode = "encode the run record: %w"
+)
+
+// Event is one line of events.jsonl. The window is on the line rather than
+// implied by position, so a consumer can filter without tracking state, and
+// `grep '"window":"fault"' events.jsonl` is a useful thing to be able to do.
+type Event struct {
+ At string `json:"at"`
+ Window string `json:"window"`
+ Stream string `json:"stream"`
+ Credential string `json:"credential"`
+ Disposition string `json:"disposition"`
+ Status int `json:"status,omitempty"`
+ LatencyMS int64 `json:"latency_ms"`
+ Detail string `json:"detail,omitempty"`
+}
+
+// Write puts the record and its events in dir, and fills the record's events
+// reference with the hash of what it actually wrote.
+//
+// The events go down FIRST: the record names their hash, so writing the
+// record before the bytes it hashes would leave a record pointing at a file
+// that does not exist yet, which is the window a reader is most likely to
+// land in when a run is killed.
+func Write(dir string, rec Record, events []Event) (Record, error) {
+ if err := os.MkdirAll(dir, dirMode); err != nil {
+ return rec, fmt.Errorf(errWriteDir, err)
+ }
+
+ encoded, err := encodeEvents(events)
+ if err != nil {
+ return rec, err
+ }
+
+ if err := writeAtomic(
+ filepath.Join(dir, EventsFile),
+ encoded,
+ ); err != nil {
+ return rec, fmt.Errorf(errWriteFile, EventsFile, err)
+ }
+
+ sum := sha256.Sum256(encoded)
+
+ rec.Events = &Events{
+ File: EventsFile,
+ SHA256: hex.EncodeToString(sum[:]),
+ Count: len(events),
+ }
+
+ body, err := Encode(rec)
+ if err != nil {
+ return rec, err
+ }
+
+ if err := writeAtomic(
+ filepath.Join(dir, RecordFile),
+ body,
+ ); err != nil {
+ return rec, fmt.Errorf(errWriteFile, RecordFile, err)
+ }
+
+ return rec, nil
+}
+
+// Encode is the one JSON form, used for the files and for stdout alike, so
+// the two cannot differ. Indented because a record a human will not open is
+// a record nobody checks, and HTML escaping off because a URL in a reason
+// should read as a URL.
+func Encode(rec Record) ([]byte, error) {
+ buffer := &writeBuffer{}
+
+ encoder := json.NewEncoder(buffer)
+ encoder.SetIndent("", " ")
+ encoder.SetEscapeHTML(false)
+
+ if err := encoder.Encode(rec); err != nil {
+ return nil, fmt.Errorf(errEncode, err)
+ }
+
+ return buffer.bytes, nil
+}
+
+type writeBuffer struct {
+ bytes []byte
+}
+
+func (b *writeBuffer) Write(p []byte) (int, error) {
+ b.bytes = append(b.bytes, p...)
+ return len(p), nil
+}
+
+func encodeEvents(events []Event) ([]byte, error) {
+ buffer := &writeBuffer{}
+
+ encoder := json.NewEncoder(buffer)
+ encoder.SetEscapeHTML(false)
+
+ for _, event := range events {
+ if err := encoder.Encode(event); err != nil {
+ return nil, fmt.Errorf(errEncode, err)
+ }
+ }
+
+ return buffer.bytes, nil
+}
+
+// writeAtomic writes a temporary file in the SAME directory and renames it
+// into place. Same directory because rename is only atomic within one
+// filesystem, and atomic because a reader that opens a half-written record
+// gets no error, just a truncated one.
+func writeAtomic(path string, body []byte) error {
+ path = filepath.Clean(path)
+ dir := filepath.Dir(path)
+
+ temp, err := os.CreateTemp(dir, tempPattern)
+ if err != nil {
+ return err
+ }
+
+ name := temp.Name()
+
+ // Removed on every failure path below, so a run that could not finish
+ // writing does not leave litter next to the record it failed to produce.
+ defer func() { _ = os.Remove(name) }()
+
+ if _, err := temp.Write(body); err != nil {
+ _ = temp.Close()
+ return err
+ }
+
+ // Durability before visibility: a rename that beats the data to disk
+ // leaves a correctly named, empty file after a crash.
+ if err := temp.Sync(); err != nil {
+ _ = temp.Close()
+ return err
+ }
+
+ if err := temp.Close(); err != nil {
+ return err
+ }
+
+ // Left at the 0600 os.CreateTemp gives it, deliberately. A probe
+ // outcome's Detail carries whatever the target answered, so the record
+ // is not a file to widen by default, and the operator khaos runs as is
+ // the operator CI runs as.
+ //nolint:gosec // the operator's own --record destination, cleaned above;
+ // khaos holds no privilege they do not, so there is no boundary here to
+ // traverse
+ return os.Rename(name, path)
+}
+
+// Events flattens every window's streams into one time-ordered sequence,
+// which is the order they were observed in and the order a reader expects.
+func EventsOf(observation experiment.Observation) []Event {
+ out := []Event{}
+
+ for _, named := range []struct {
+ name string
+ window probe.Window
+ }{
+ {"baseline", observation.Baseline},
+ {"fault", observation.Fault},
+ {"resumed", observation.Resumed},
+ {"recovery", observation.Recovery},
+ } {
+ out = append(out, eventsOfWindow(named.name, named.window)...)
+ }
+
+ return out
+}
+
+func eventsOfWindow(window string, from probe.Window) []Event {
+ out := []Event{}
+
+ for _, named := range []struct {
+ name string
+ stream probe.Stream
+ }{
+ {"valid", from.Valid},
+ {"invalid", from.Invalid},
+ {"revoked", from.Revoked},
+ {"forged", from.Forged},
+ } {
+ for _, outcome := range named.stream.Outcomes {
+ out = append(out, Event{
+ At: outcome.At.UTC().Format(timeLayout),
+ Window: window,
+ Stream: named.name,
+ Credential: string(outcome.Credential),
+ Disposition: string(outcome.Disposition),
+ Status: outcome.Status,
+ LatencyMS: outcome.LatencyMS,
+ Detail: outcome.Detail,
+ })
+ }
+ }
+
+ // Within a window the streams are recorded one after another, so a naive
+ // concatenation interleaves nothing and reads as four separate runs.
+ sort.SliceStable(out, func(i, j int) bool {
+ return out[i].At < out[j].At
+ })
+
+ return out
+}
+
+// WriteFile puts one projection where it was asked for, with the same
+// atomic rename the record gets: node_exporter reading a half-written
+// metrics file ships corrupted data rather than an error, and a CI reporter
+// parsing a half-written XML file reports a broken suite.
+func WriteFile(path string, body []byte) error {
+ dir := filepath.Dir(filepath.Clean(path))
+
+ if err := os.MkdirAll(dir, dirMode); err != nil {
+ return fmt.Errorf(errWriteDir, err)
+ }
+
+ if err := writeAtomic(path, body); err != nil {
+ return fmt.Errorf(errWriteFile, filepath.Base(path), err)
+ }
+
+ return nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/justfile b/PROJECTS/intermediate/chaos-engineering-tool/justfile
index eb426091..0bc1fbc4 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/justfile
+++ b/PROJECTS/intermediate/chaos-engineering-tool/justfile
@@ -97,7 +97,7 @@ fix:
check: lint tidy-check test colors
-gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gate control-gate
+gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gate control-gate report-gate
colors: build
./scripts/colors.sh
@@ -132,5 +132,8 @@ credential-gate: build
control-gate: build
./scripts/control.sh
+report-gate: build
+ ./scripts/report.sh
+
clean:
rm -f {{binary}}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh
index 25b08686..85df6ec7 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh
@@ -73,8 +73,13 @@ start_range() {
[[ "$with_db" == "yes" ]] && args+=(--with-db)
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" "${args[@]}" >/dev/null 2>&1; then
- fail "range up --fail-mode $fail_mode (with-db=$with_db)"
+
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" "${args[@]}" 2>&1)"; then
+ fail "range up --fail-mode $fail_mode (with-db=$with_db): $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -198,7 +203,12 @@ assert_denied() {
assert_report_has \
"${label} is not called a defect" \
"without calling it a defect"
- assert_exit "$label" 0
+
+ # 3, not 0. M10 gave a deviation in a direction the polarity does NOT
+ # call a defect its own exit code: the control did change under the
+ # fault, and reporting that as a clean pass hid a real availability
+ # finding behind a green build.
+ assert_exit "$label" 3
}
echo "== the three default-range controls, fail-open =="
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
index 627e5b81..e4f8039a 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
@@ -136,6 +136,39 @@ assert_verdict() {
fi
}
+# exit_for_verdict maps an expected verdict onto khaos's exit contract.
+#
+# It takes the experiment's DECLARED DEFECT DIRECTIONS as its second argument,
+# and that is not decoration: the exit code is a function of (verdict,
+# polarity), never of the verdict alone. `passed / HELD` is 0 for most of the
+# catalogue and 2 for the pooled family, where poolPolarity declares HELD the
+# defect -- a lifetime that converts a rotation into a revocation is the thing
+# those experiments hunt. An earlier version of this helper read the verdict
+# string by itself and reported four pooled assertions as exit 0 when khaos
+# correctly returned 2.
+#
+# The directions are passed per section rather than looked up, so an inverted
+# polarity is visible at the call site instead of buried in a helper.
+exit_for_verdict() {
+ local verdict="$1" defects="${2:-PERMITTED}" direction
+
+ direction="${verdict##*/ }"
+
+ if [[ " ${defects} " == *" ${direction} "* ]]; then
+ echo 2
+ return
+ fi
+
+ case "$verdict" in
+ "passed / "*) echo 0 ;;
+ "deviated / "*) echo 3 ;;
+ *)
+ echo "exit_for_verdict: no code for the verdict '$verdict'" >&2
+ echo 99
+ ;;
+ esac
+}
+
assert_exit() {
local label="$1" want="$2"
if [[ "$last_code" == "$want" ]]; then
@@ -249,9 +282,13 @@ start_range() {
local secret="${2:-unchecked}"
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --with-db --pool-mode "$pool" \
- --secret-mode "$secret" --no-color --quiet >/dev/null 2>&1; then
- fail "range up --with-db --pool-mode $pool --secret-mode $secret"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --with-db --pool-mode "$pool" \
+ --secret-mode "$secret" --no-color --quiet 2>&1)"; then
+ fail "range up --with-db --pool-mode $pool --secret-mode $secret: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -364,12 +401,6 @@ declare -A WANT_VERDICT=(
[held]="passed / HELD"
)
-declare -A WANT_EXIT=(
- [unbounded]=2
- [bounded]=0
- [held]=2
-)
-
# What khaos DID is carried by the HELD CAVEAT for these two, not by the defect
# phrase: that phrase is part of a leak reason and a leak is exactly what a
# HELD finding does not have. Asserting the defect phrase here failed six times
@@ -390,7 +421,8 @@ for pool in unbounded bounded held; do
label="${experiment#credential.} on a ${pool} pool"
assert_verdict "$label" "${WANT_VERDICT[$pool]}"
- assert_exit "$label" "${WANT_EXIT[$pool]}"
+ assert_exit "$label" \
+ "$(exit_for_verdict "${WANT_VERDICT[$pool]}" "HELD PERMITTED")"
assert_confirmed "$label"
assert_control_held "$label"
@@ -446,13 +478,6 @@ declare -A WANT_SECRET_UNCHECKED=(
[credential.secret_file_unreadable]="deviated / DENIED"
)
-declare -A WANT_SECRET_EXIT=(
- [credential.secret_file_empty]=2
- [credential.secret_file_truncated]=0
- [credential.secret_file_missing]=0
- [credential.secret_file_unreadable]=0
-)
-
if start_range unbounded unchecked; then
declare -A SECRET_OBSERVABLE=()
@@ -462,7 +487,7 @@ if start_range unbounded unchecked; then
label="${experiment#credential.} against a verifier that does not check the length"
assert_verdict "$label" "${WANT_SECRET_UNCHECKED[$experiment]}"
- assert_exit "$label" "${WANT_SECRET_EXIT[$experiment]}"
+ assert_exit "$label" "$(exit_for_verdict "${WANT_SECRET_UNCHECKED[$experiment]}")"
assert_confirmed "$label"
assert_control_held "$label"
@@ -514,7 +539,7 @@ if start_range unbounded validated; then
"secret_file_empty against a verifier that checks the length" \
"deviated / DENIED"
assert_exit \
- "secret_file_empty against a verifier that checks the length" 0
+ "secret_file_empty against a verifier that checks the length" 3
assert_confirmed \
"secret_file_empty against a verifier that checks the length"
assert_report_lacks \
@@ -582,11 +607,6 @@ declare -A WANT_ENV=(
[validated]="deviated / DENIED"
)
-declare -A WANT_ENV_EXIT=(
- [unchecked]=2
- [validated]=0
-)
-
for secret in unchecked validated; do
if ! start_range unbounded "$secret"; then
continue
@@ -598,7 +618,7 @@ for secret in unchecked validated; do
label="${experiment#credential.} against a ${secret} verifier"
assert_verdict "$label" "${WANT_ENV[$secret]}"
- assert_exit "$label" "${WANT_ENV_EXIT[$secret]}"
+ assert_exit "$label" "$(exit_for_verdict "${WANT_ENV[$secret]}")"
assert_confirmed "$label"
assert_control_held "$label"
done
@@ -870,13 +890,6 @@ declare -A WANT_AUTHZ=(
[cached]="passed / HELD"
)
-declare -A WANT_AUTHZ_EXIT=(
- [collapses]=2
- [distinguishes]=0
- [retries]=0
- [cached]=0
-)
-
for mode in collapses distinguishes retries cached; do
if ! start_authz_range "$mode"; then
continue
@@ -887,7 +900,8 @@ for mode in collapses distinguishes retries cached; do
label="apikey_revoked against a ${mode} caller"
assert_verdict "$label" "${WANT_AUTHZ[$mode]}"
- assert_exit "$label" "${WANT_AUTHZ_EXIT[$mode]}"
+ assert_exit "$label" \
+ "$(exit_for_verdict "${WANT_AUTHZ[$mode]}" "PERMITTED DENIED")"
assert_confirmed "$label"
assert_control_held "$label" "the caller authenticates credentials"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
index 080876c6..65a3dda4 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
@@ -72,9 +72,13 @@ start_range() {
local dns_mode="${2:-reresolve}"
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --fail-mode "$1" --dns-mode "$dns_mode" \
- --no-color --quiet >/dev/null 2>&1; then
- fail "range up --fail-mode $1 --dns-mode $dns_mode"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --fail-mode "$1" --dns-mode "$dns_mode" \
+ --no-color --quiet 2>&1)"; then
+ fail "range up --fail-mode $1 --dns-mode $dns_mode: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -354,7 +358,11 @@ if start_range open; then
for fault in ttl_zero ttl_long; do
run_experiment "dns.${fault}"
assert_verdict "${fault} vs an uncached target" "unsupported"
- assert_exit "${fault} vs an uncached target" 4
+
+ # 9, not 4. M10 gave UNSUPPORTED its own code: a question that does
+ # not apply to this target is not a measurement khaos failed to make,
+ # and it was indistinguishable from one while both exited 4.
+ assert_exit "${fault} vs an uncached target" 9
# Anchored on the LIFETIME khaos served, which is the measurement the
# verdict is entitled to make. The target's lookups do not scale with
@@ -394,7 +402,9 @@ if start_range closed; then
for fault in nxdomain servfail; do
run_experiment "dns.${fault}"
assert_verdict "${fault} vs fail-closed" "deviated / DENIED"
- assert_exit "${fault} vs fail-closed" 0
+
+ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.
+ assert_exit "${fault} vs fail-closed" 3
assert_confirmed "${fault}"
done
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
index 720e6eab..0d76202d 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh
@@ -106,8 +106,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }
start_range() {
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --fail-mode "$1" --quiet --no-color >/dev/null 2>&1; then
- fail "range up --fail-mode $1"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --fail-mode "$1" --quiet --no-color 2>&1)"; then
+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -134,6 +138,35 @@ assert_verdict() {
fi
}
+# exit_for_verdict maps an expected verdict onto khaos's exit contract.
+#
+# The second argument is the experiment's DECLARED DEFECT DIRECTIONS, because
+# the code is a function of (verdict, polarity) and never of the verdict
+# alone: `passed / HELD` is 0 ordinarily and 2 wherever a polarity declares
+# HELD the defect. The loop below passes the default because the one inverted
+# experiment in this family, identity.key_rotation, is held out as INVERTED
+# and asserted separately -- but the helper takes the argument so that a
+# second inversion cannot pass silently.
+exit_for_verdict() {
+ local verdict="$1" defects="${2:-PERMITTED}" direction
+
+ direction="${verdict##*/ }"
+
+ if [[ " ${defects} " == *" ${direction} "* ]]; then
+ echo 2
+ return
+ fi
+
+ case "$verdict" in
+ "passed / "*) echo 0 ;;
+ "deviated / "*) echo 3 ;;
+ *)
+ echo "exit_for_verdict: no code for the verdict '$verdict'" >&2
+ echo 99
+ ;;
+ esac
+}
+
assert_exit() {
local label="$1" got="$2" want="$3"
if [[ "$got" == "$want" ]]; then
@@ -389,9 +422,17 @@ if start_range closed; then
for experiment in "${parts[@]:1}"; do
label="${experiment} vs a correct verifier"
+ want_verdict="${CORRECT_VERDICT[$experiment]:-passed / HELD}"
+
run_experiment "$experiment"
- assert_exit "$label" "$last_code" 0
- assert_verdict "$label" "${CORRECT_VERDICT[$experiment]:-passed / HELD}"
+
+ # DERIVED from the expected verdict rather than written beside it.
+ # M10 gave a deviation in a non-defect direction its own exit
+ # code, so four of these seventeen now exit 3 and the rest exit 0
+ # -- and a hand-maintained second table would be one more thing
+ # that can disagree with the first.
+ assert_exit "$label" "$last_code" "$(exit_for_verdict "$want_verdict")"
+ assert_verdict "$label" "$want_verdict"
if grep -qE "^ verdict .*PERMITTED" <<<"$report"; then
fail "${label} found a bypass in a correct verifier: $(verdict_line)"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh
index bbb22bc2..3aab5f86 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh
@@ -38,8 +38,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }
start_range() {
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --fail-mode "$1" --cache-ttl "$CACHE_TTL" --quiet --no-color >/dev/null 2>&1; then
- fail "range up --fail-mode $1"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --fail-mode "$1" --cache-ttl "$CACHE_TTL" --quiet --no-color 2>&1)"; then
+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -180,7 +184,9 @@ fi
if start_range closed; then
run_experiment
assert_verdict "fail-closed" "deviated / DENIED"
- assert_exit "fail-closed" "$last_code" 0
+
+ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.
+ assert_exit "fail-closed" "$last_code" 3
assert_no_cause "fail-closed"
if grep -q "acceptable for this control" <<<"$report"; then
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh
index fec7b6c6..ba4610e4 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh
@@ -65,8 +65,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }
start_range() {
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --fail-mode "$1" --quiet --no-color >/dev/null 2>&1; then
- fail "range up --fail-mode $1"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --fail-mode "$1" --quiet --no-color 2>&1)"; then
+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -75,9 +79,21 @@ start_range() {
# Sets the globals `report` and `last_code`. Never call it through a command
# substitution: that runs it in a subshell and the report is lost while the exit
# code still comes back, which looks like a passing gate with no evidence.
+# Extra arguments REPLACE the default timing rather than appending to it: two
+# copies of --fault-for on one command line is an ambiguity, and a helper that
+# silently dropped an override would make a section measure the default window
+# while claiming to measure a short one.
run_experiment() {
+ local experiment="$1"
+ shift
+
+ local -a timing=("${TIMING[@]}")
+ if [[ $# -gt 0 ]]; then
+ timing=("$@")
+ fi
+
set +e
- report="$("$BIN" run "$1" --no-color "${TIMING[@]}" 2>&1)"
+ report="$("$BIN" run "$experiment" --no-color "${timing[@]}" 2>&1)"
last_code=$?
set -e
}
@@ -175,7 +191,27 @@ assert_an_unconfirmable_fault_is_not_success() {
return
fi
- assert_exit "an unconfirmable fault" "$last_code" 4
+ # The code is DERIVED from the epistemic word the run actually reported,
+ # not hardcoded. M10 split the one inconclusive code into three, and this
+ # check accepts three states: asserting a single code against a
+ # three-state regex is a flake that fires whenever the run lands on one
+ # of the other two.
+ local epistemic
+ epistemic="$(grep -E '^ verdict' <<<"$report" |
+ sed -E 's/^ verdict +([a-z-]+).*/\1/' | head -1 || true)"
+
+ local want
+ case "$epistemic" in
+ aborted) want=7 ;;
+ probe-error) want=5 ;;
+ baseline-not-met) want=8 ;;
+ *)
+ fail "could not read an epistemic word out of the verdict: ${epistemic}"
+ return
+ ;;
+ esac
+
+ assert_exit "an unconfirmable fault (${epistemic})" "$last_code" "$want"
}
assert_ran_with_no_flags() {
@@ -319,7 +355,11 @@ if start_range closed; then
fi
assert_verdict "${experiment} fail-closed" "deviated / DENIED"
- assert_exit "${experiment} fail-closed" "$last_code" 0
+
+ # 3, not 0: M10 moved a deviation in a direction the polarity does
+ # NOT call a defect onto its own code. The target did change under the
+ # fault, and reporting that as a clean pass hid it.
+ assert_exit "${experiment} fail-closed" "$last_code" 3
if grep -q "acceptable for this control" <<<"$report"; then
pass "${experiment} fail-closed polarity spared the safe direction"
@@ -329,6 +369,41 @@ if start_range closed; then
done
fi
+# M10's most dangerous exit code, and the one no other gate exercises. A HELD
+# verdict khaos declined to call conclusive used to exit 0: khaos asserting a
+# control held on evidence it refused to stand behind, as a green build.
+#
+# Asserted as a PAIR. "exit 4 on a short window" alone is satisfied by any
+# failure; the same experiment with a full window exiting 0 is what proves the
+# 4 came from the sample floor and not from something else going wrong.
+if start_range closed; then
+ run_experiment "$EOF_DEPENDENT_FAULT" \
+ --baseline-for 4s --fault-for 3s --recover-for 2s --sample-every 300ms
+
+ assert_verdict "a thin HELD window" "passed / HELD"
+ assert_exit "a thin HELD window" "$last_code" 4
+
+ # The SHORTFALL, named with both numbers. Without this the assertion
+ # cannot tell a sample floor from any other inconclusive cause.
+ if grep -qE 'paired samples cannot rule one out' <<<"$report"; then
+ pass "a thin HELD window says how many samples it had and how many it needed"
+ else
+ fail "a thin HELD window did not name the sample shortfall: $(grep -E '^ why' <<<"$report" | head -2)"
+ fi
+
+ # The negative control: the same fault, a full window, exit 0.
+ run_experiment "$EOF_DEPENDENT_FAULT" "${TIMING[@]}"
+
+ assert_verdict "a full HELD window" "passed / HELD"
+ assert_exit "a full HELD window" "$last_code" 0
+
+ if grep -qE 'cannot rule one out' <<<"$report"; then
+ fail "a full window still claimed its evidence was thin"
+ else
+ pass "a full HELD window claims its conclusion without a caveat"
+ fi
+fi
+
if [[ $failures -ne 0 ]]; then
echo "${failures} proxy assertion(s) failed"
exit 1
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh
index 72c77f74..304a6f2c 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh
@@ -26,8 +26,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }
start_range() {
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --fail-mode "$1" --cache-ttl "${2:-2s}" --quiet --no-color >/dev/null 2>&1; then
- fail "range up --fail-mode $1 --cache-ttl ${2:-2s}"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --fail-mode "$1" --cache-ttl "${2:-2s}" --quiet --no-color 2>&1)"; then
+ fail "range up --fail-mode $1 --cache-ttl ${2:-2s}: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -196,7 +200,9 @@ if start_range closed; then
run_experiment
assert_zero_flags "fail-closed"
assert_verdict "fail-closed" "deviated / DENIED"
- assert_exit "fail-closed" "$last_code" 0
+
+ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.
+ assert_exit "fail-closed" "$last_code" 3
if grep -q "acceptable for this control" <<<"$report"; then
pass "fail-closed the deviation was recorded without calling it a defect"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh
new file mode 100755
index 00000000..e79150c5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh
@@ -0,0 +1,337 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# report.sh
+
+set -euo pipefail
+
+# M10's exit gate. Five things have to be true of khaos's outputs.
+#
+# 1. The EXIT CODE TAXONOMY is exercised end to end against a live range.
+# A single non-zero code would collapse "a control failed open", "the
+# experiment was invalid" and "khaos could not measure" into one signal,
+# and those need three different human responses. Every code this gate
+# can reach is driven and asserted; the four it cannot are named below
+# with the reason, rather than left to look covered.
+#
+# 2. `--fail-on` NARROWS and never widens. The default keeps every non-pass
+# non-zero, because a run khaos declined to conclude from exiting 0 is a
+# green build on no evidence. No value may suppress a rollback failure.
+#
+# 3. The record is the canonical artifact and the events file is linked to
+# it BY HASH, so a record cannot be silently separated from the
+# observations it summarises.
+#
+# 4. The projections are projections. The JUnit XML and the metrics file are
+# derived from the record, so they must name the same run and the same
+# exit code, and a real parser has to accept each of them: `promtool`
+# for the metrics and an XML parser for the suite.
+#
+# 5. A record the operator ASKED FOR and did not get is never silent, and
+# never masks a verdict either.
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+WORK="$(mktemp -d)"
+
+# Three sample bands, because the verdict's own thresholds are what separate
+# three different exit codes: under MinimumPaired (5) khaos cannot measure,
+# under MinimumToClaimClosed (30) it will not call a hold conclusive, and
+# above it a hold is a hold.
+FULL=(--baseline-for 5s --fault-for 20s --recover-for 3s --sample-every 300ms)
+THIN=(--baseline-for 4s --fault-for 3s --recover-for 2s --sample-every 300ms)
+TINY=(--baseline-for 4s --fault-for 600ms --recover-for 2s --sample-every 300ms)
+
+failures=0
+report=""
+last_code=0
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR" "$WORK"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+start_range() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --fail-mode "$1" --no-color --quiet 2>&1)"; then
+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"
+ return 1
+ fi
+ return 0
+}
+
+# Never through a command substitution: that runs in a subshell and the
+# report is lost while the exit code still comes back, which reads as a
+# passing gate with no evidence.
+run_experiment() {
+ local experiment="$1"
+ shift
+ set +e
+ report="$("$BIN" run "$experiment" --no-color "$@" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+assert_exit() {
+ local label="$1" want="$2"
+ if [[ "$last_code" == "$want" ]]; then
+ pass "${label} exit ${want}"
+ else
+ fail "${label} exit ${last_code}, want ${want}"
+ fi
+}
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ fi
+}
+
+echo "== the exit codes, driven against a live range =="
+
+if start_range open; then
+ # 2, the gate. The only code a pipeline should break on by default.
+ run_experiment control.policy_store_down "${FULL[@]}"
+ assert_verdict "a defect" "deviated / PERMITTED"
+ assert_exit "a defect" 2
+
+ # 4. A HELD verdict khaos declined to call conclusive. This used to exit
+ # 0: khaos asserting a control held on evidence it refused to stand
+ # behind, as a green build.
+ run_experiment network.dependency_slow_close "${THIN[@]}"
+ assert_verdict "a thin hold" "passed / HELD"
+ assert_exit "a thin hold" 4
+
+ # 5. Fewer paired samples than khaos needs to measure anything at all,
+ # which is a different problem from a hold it will not call conclusive.
+ run_experiment network.dependency_slow_close "${TINY[@]}"
+ assert_exit "a window too short to measure" 5
+
+ # 7. A fault khaos could not confirm. Measuring nothing is not passing.
+ run_experiment network.dependency_partial "${FULL[@]}" --bytes 999999
+ assert_exit "a fault that could not be confirmed" 7
+
+ # 1. Usage. Refused before anything is injected.
+ run_experiment control.policy_store_down --fail-on not-an-outcome
+ assert_exit "an outcome khaos cannot gate on" 1
+
+ if grep -q 'is not an outcome khaos can gate on' <<<"$report"; then
+ pass "the refusal names the cause rather than just failing"
+ else
+ fail "the usage refusal did not name the cause"
+ fi
+fi
+
+if start_range closed; then
+ # 0 and 3 are the pair that makes the directional axis visible: the same
+ # fault, the same correct target, and khaos separates "nothing moved"
+ # from "it refused to serve, which is correct and costs availability".
+ run_experiment network.dependency_slow_close "${FULL[@]}"
+ assert_verdict "a clean hold" "passed / HELD"
+ assert_exit "a clean hold" 0
+
+ run_experiment network.dependency_reset "${FULL[@]}"
+ assert_verdict "a non-defect deviation" "deviated / DENIED"
+ assert_exit "a non-defect deviation" 3
+fi
+
+echo "== --fail-on narrows, and never past the two it cannot touch =="
+
+if start_range open; then
+ run_experiment control.policy_store_down "${FULL[@]}" --fail-on never
+ assert_exit "a defect under --fail-on never" 0
+
+ run_experiment control.policy_store_down "${FULL[@]}" --fail-on inconclusive
+ assert_exit "a defect under a gate that does not name it" 0
+
+ run_experiment control.policy_store_down "${FULL[@]}" --fail-on defect
+ assert_exit "a defect under a gate that names it" 2
+fi
+
+echo "== the record is canonical and its events are linked by hash =="
+
+if start_range open; then
+ into="${WORK}/record"
+
+ run_experiment control.policy_store_down "${FULL[@]}" \
+ --record "$into" \
+ --junit "${WORK}/khaos.xml" \
+ --metrics-out "${WORK}/khaos.prom"
+
+ assert_exit "a recorded run" 2
+
+ for file in run.json events.jsonl; do
+ if [[ -s "${into}/${file}" ]]; then
+ pass "the run wrote ${file}"
+ else
+ fail "the run wrote no ${file}"
+ fi
+ done
+
+ claimed="$(python3 -c "
+import json
+print(json.load(open('${into}/run.json'))['events']['sha256'])
+" 2>/dev/null || true)"
+ actual="$(sha256sum "${into}/events.jsonl" | cut -d' ' -f1)"
+
+ if [[ -n "$claimed" && "$claimed" == "$actual" ]]; then
+ pass "the record's hash matches the events file beside it"
+ else
+ fail "the record claims ${claimed:-<none>} and the file hashes to ${actual}"
+ fi
+
+ # The record has to explain the code khaos returned, because the exit
+ # status is the only part of a run most CI ever sees.
+ recorded="$(python3 -c "
+import json
+print(json.load(open('${into}/run.json'))['exit']['code'])
+" 2>/dev/null || true)"
+
+ if [[ "$recorded" == 2 ]]; then
+ pass "the record carries the exit code khaos returned"
+ else
+ fail "the record says exit ${recorded:-<none>} and khaos returned 2"
+ fi
+fi
+
+echo "== the projections are accepted by real parsers =="
+
+# promtool is the format's own linter, and it rejects every malformed shape
+# this gate cares about: a missing trailing newline is `unexpected end of
+# input stream`, a repeated TYPE line and an unescaped label quote are both
+# parse errors. Verified against promtool 3.15.0.
+if [[ ! -s "${WORK}/khaos.prom" ]]; then
+ fail "no metrics file was written, so this section cannot fail"
+elif ! command -v promtool >/dev/null 2>&1; then
+ fail "promtool is not installed, so khaos's exposition format is unverified"
+else
+ if promtool check metrics <"${WORK}/khaos.prom" >/dev/null 2>&1; then
+ pass "promtool accepts khaos's exposition format"
+ else
+ fail "promtool rejects khaos's metrics: $(promtool check metrics <"${WORK}/khaos.prom" 2>&1 | head -2)"
+ fi
+
+ # The NEGATIVE control on the linter itself: a checker that always says
+ # yes cannot detect a malformed file, and the trailing newline is the
+ # rule that fails silently.
+ if printf '# TYPE x_m gauge\nx_m 1' |
+ promtool check metrics >/dev/null 2>&1; then
+ fail "promtool accepted a file with no trailing newline, so its approval above proves nothing"
+ else
+ pass "promtool rejects a file with no trailing newline"
+ fi
+fi
+
+if [[ ! -s "${WORK}/khaos.xml" ]]; then
+ fail "no JUnit file was written, so this section cannot fail"
+else
+ if python3 -c "
+import sys, xml.etree.ElementTree as ET
+root = ET.parse('${WORK}/khaos.xml').getroot()
+assert root.tag == 'testsuites', root.tag
+suite = root.find('testsuite')
+assert suite is not None
+case = suite.find('testcase')
+assert case is not None
+assert case.find('failure') is not None, 'a defect is not a failure'
+assert int(root.get('failures')) == 1, root.get('failures')
+" 2>/dev/null; then
+ pass "the JUnit suite parses and reports the defect as a failure"
+ else
+ fail "the JUnit suite is malformed or does not report the defect"
+ fi
+
+ # The way back to the record, for a consumer that only has the XML.
+ if grep -q 'name="run_id"' "${WORK}/khaos.xml"; then
+ pass "the JUnit suite carries the run id back to the record"
+ else
+ fail "the JUnit suite carries no run id"
+ fi
+fi
+
+echo "== a record khaos could not write is never silent =="
+
+if start_range open; then
+ # The verdict outranks the bookkeeping: a run that found a bypass still
+ # exits on the bypass, and still says the record is missing.
+ run_experiment control.policy_store_down "${FULL[@]}" \
+ --record /proc/khaos-cannot-write-here
+
+ assert_exit "a defect whose record could not be written" 2
+
+ if grep -q 'could not write the run record' <<<"$report"; then
+ pass "the write failure is reported rather than swallowed"
+ else
+ fail "a record khaos could not write was silent"
+ fi
+
+ # But a clean run that could not write the record the operator asked for
+ # fails on the write: in CI they believe they have an artifact and do not.
+ run_experiment control.policy_store_down "${FULL[@]}" \
+ --fail-on never \
+ --record /proc/khaos-cannot-write-here
+
+ assert_exit "a clean run whose record could not be written" 1
+fi
+
+echo "== the codes this gate cannot reach, named rather than implied =="
+
+# Four codes are deliberately not driven here, and saying so is the point: a
+# gate that looked complete while skipping them would be the inert kind.
+#
+# 6 ROLLBACK needs khaos to fail to revert a fault it applied, which
+# means breaking the daemon mid-run. `scripts/survive.sh`
+# owns the rollback paths; forcing the failure here would
+# leave the operator's range dirty, which is the one thing
+# this tool must not do.
+# 8 BASELINE needs a target outside steady state BEFORE injection. The
+# range is healthy by construction, and the unit census in
+# internal/cli covers the mapping.
+# 9 UNSUPPORTED driven by scripts/dns.sh and scripts/tls.sh, against the
+# targets whose question genuinely does not apply.
+# 70 INTERNAL a recovered panic. Reachable only from a khaos bug, so
+# there is nothing honest to assert.
+#
+# A CENSUS rather than a comment. The paragraph above was an unconditional
+# `pass`, which is documentation wearing an assertion's clothes: it could not
+# fail, so it could not notice a thirteenth code arriving with nothing
+# covering it. This counts the taxonomy in the source and holds it to the
+# number this gate and the unit census between them account for.
+# [[:space:]] rather than \t: grep -E treats a backslash-t as a literal
+# `t`, so the tab form matched nothing and this census reported 0 under
+# bash while appearing to work when the same line was pasted into zsh.
+declared="$(grep -cE '^[[:space:]]+Code[A-Za-z]+ += ' internal/cli/exit.go || true)"
+
+if [[ "$declared" == 12 ]]; then
+ pass "the taxonomy is twelve codes, which is what this gate and the unit census cover"
+else
+ fail "the source declares ${declared} exit codes and this gate accounts for 12; a new code needs a home here or a named reason it cannot have one"
+fi
+
+# And the unit census that covers what a live range cannot reach has to
+# exist, or the four codes named above are covered by nothing at all.
+if grep -q 'func TestNoTwoOutcomesShareACode' internal/cli/exit_test.go; then
+ pass "the unit census that pins every code is present"
+else
+ fail "the unit census is gone, so the codes this gate cannot drive are unverified"
+fi
+
+if [[ "$failures" -gt 0 ]]; then
+ echo "report: ${failures} assertion(s) failed"
+ exit 1
+fi
+
+echo "report: all assertions passed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh
index 460f59f8..e3809bdc 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh
@@ -50,8 +50,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }
# reconfigured mid-run would let khaos change the target it is measuring.
start_range() {
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --tls-mode "$1" --no-color --quiet >/dev/null 2>&1; then
- fail "range up --tls-mode $1"
+ # The OUTPUT, not just the fact. A bare "range up failed" is what a
+ # transient startup error looks like in a two-hour gate run, and it
+ # leaves nothing to diagnose from.
+ local started
+ if ! started="$("$BIN" range up --tls-mode "$1" --no-color --quiet 2>&1)"; then
+ fail "range up --tls-mode $1: $(tail -3 <<<"$started")"
return 1
fi
return 0
@@ -111,7 +115,9 @@ if start_range closed; then
# this family declares it acceptable, so it must not be called a defect.
run_experiment tls.leaf_expired
assert_verdict "leaf-expired vs a correct client" "deviated / DENIED"
- assert_exit "leaf-expired vs a correct client" 0
+
+ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.
+ assert_exit "leaf-expired vs a correct client" 3
assert_confirmed "leaf-expired"
# THE CONTRAST, and the reason the unsupported verdict is honest rather
@@ -119,7 +125,7 @@ if start_range closed; then
# certificate is a finding it can actually have.
run_experiment tls.revoked_via_crl
assert_verdict "revoked vs a client that checks" "deviated / DENIED"
- assert_exit "revoked vs a client that checks" 0
+ assert_exit "revoked vs a client that checks" 3
if grep -qE '^ verdict .*unsupported' <<<"$report"; then
fail "revoked was called unsupported against a client that fetches the CRL, which excuses a real defect"
@@ -150,9 +156,13 @@ if start_range no-revocation-check; then
run_experiment tls.revoked_via_crl
assert_verdict "revoked vs a client that never asks" "unsupported"
- # NOT zero. A revoked certificate in active use must never exit 0, and
- # every inconclusive epistemic outcome exits 4.
- assert_exit "revoked vs a client that never asks" 4
+ # NOT zero. A revoked certificate in active use must never exit 0.
+ #
+ # 9 rather than 4 as of M10: "every inconclusive epistemic outcome exits
+ # 4" was true when this was written and is not any more. UNSUPPORTED has
+ # its own code, because a question that does not apply to this target is
+ # a different thing from evidence khaos could not gather.
+ assert_exit "revoked vs a client that never asks" 9
if grep -qE '^ verdict .*passed' <<<"$report"; then
fail "a revoked certificate in active use was reported as a pass"
@@ -275,7 +285,7 @@ if start_range closed; then
for experiment in tls.mtls_client_expired tls.mtls_client_absent; do
run_experiment "$experiment"
assert_verdict "${experiment} vs a client that reads" "deviated / DENIED"
- assert_exit "${experiment} vs a client that reads" 0
+ assert_exit "${experiment} vs a client that reads" 3
done
fi
@@ -307,7 +317,7 @@ if start_range closed; then
# revocation, so the hole is the HOOK rather than resumption itself.
run_experiment tls.resumption_after_revocation
assert_verdict "resumption vs a per-connection policy" "deviated / DENIED"
- assert_exit "resumption vs a per-connection policy" 0
+ assert_exit "resumption vs a per-connection policy" 3
fi
if [[ "$failures" -gt 0 ]]; then