Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(khaos): the reporting surface, and the exit code that was a green build
CarterPerez-devba05463158b1
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.goindex 5885b7eb..69ea2fd0 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go@@ -23,15 +23,14 @@ import ()const (- exitOK = 0- exitError = 1- exitDefect = 2+ exitOK = cli.CodeOK- // An interrupted run is a kind of inconclusive one, so they share a code.- exitInconclusive = 4- exitInterrupted = 4+ // The catch-all. A bad flag, an unreadable config and an error khaos has+ // no taxonomy entry for are all the operator's to fix before measuring.+ exitError = cli.CodeUsage- exitRevertFailed = 6+ exitInterrupted = cli.CodeInterrupted+ exitRevertFailed = cli.CodeRollbacknoColorFlag = "--" + cli.FlagNoColornoColorFlagTrue = noColorFlag + "="@@ -106,6 +105,11 @@ func noColorRequested(args []string) bool {// fail maps an error to an exit code, and the ORDER is the contract: the state// khaos left the target in outranks the verdict, and both outrank interruption.// errors.Join carries several at once, so the first match decides.+//+// A finished run arrives carrying its own code, because the precedence among+// verdict outcomes is decided once where the run ends rather than re-derived+// by a chain here that grows with every epistemic value. The checks below it+// are for errors that never reached a verdict at all.func fail(err error) int {if errors.Is(err, run.ErrRevertFailed) {fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)@@ -113,11 +117,20 @@ func fail(err error) int {return exitRevertFailed}+ // Checked before the carried code as well as inside it: this is the one+ // outcome nothing may suppress, so it is matched on the sentinel too and+ // not only on a code some other layer computed.if errors.Is(err, cli.ErrNotRecovered) {fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)return exitRevertFailed}+ var exit *cli.Exit+ if errors.As(err, &exit) {+ fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)+ return exit.Code+ }+if errors.Is(err, context.Canceled) {fmt.Fprintf(os.Stderr, errorFormat, version.Name, errInterrupted)return exitInterrupted@@ -126,14 +139,6 @@ func fail(err error) int {fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)printFix(err)- if errors.Is(err, cli.ErrDefectFound) {- return exitDefect- }-- if errors.Is(err, cli.ErrInconclusive) {- return exitInconclusive- }-return exitError}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.goindex a21a56f0..48e0ce9c 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go@@ -155,8 +155,45 @@ func TestRollbackFailureOutranksEveryOtherExitCode(t *testing.T) {exitRevertFailed,},{"cancellation alone", context.Canceled, exitInterrupted},- {"defect alone", cli.ErrDefectFound, exitDefect},- {"inconclusive alone", cli.ErrInconclusive, exitInconclusive},++ // A finished run arrives carrying its own code, which is the shape+ // exitFor actually produces. The bare sentinels below it reach here+ // only from paths that never got a verdict.+ {+ "a defect carrying its code",+ &cli.Exit{+ Code: cli.CodeDefect,+ Outcome: cli.OutcomeDefect,+ Err: cli.ErrDefectFound,+ },+ cli.CodeDefect,+ },+ {+ "a rollback failure carrying its code",+ &cli.Exit{+ Code: cli.CodeRollback,+ Err: run.ErrRevertFailed,+ },+ exitRevertFailed,+ },+ {+ "an interrupted run carrying its code",+ &cli.Exit{+ Code: cli.CodeInterrupted,+ Outcome: cli.OutcomeInterrupted,+ Err: cli.ErrInconclusive,+ },+ cli.CodeInterrupted,+ },+ {+ "a carried code joined with a rollback failure",+ errors.Join(+ &cli.Exit{Code: cli.CodeDefect, Err: cli.ErrDefectFound},+ run.ErrRevertFailed,+ ),+ exitRevertFailed,+ },+ {"defect alone, with no code", cli.ErrDefectFound, exitError},{"anything else", errors.New("something broke"), exitError},}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.gonew file mode 100644index 00000000..e95fab45--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go@@ -0,0 +1,177 @@+// ©AngelaMos | 2026+// emit.go++package cli++import (+ "errors"+ "fmt"+ "io"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/record"+ "github.com/CarterPerez-dev/khaos/internal/version"+ "github.com/spf13/cobra"+)++const (+ errUnknownOutput = "%w: %q (khaos writes %s or %s)"+ errRecordWrite = "%w: %v"+ warnRecordWrite = "khaos could not write the run record: %v\n"+)++var ErrUnknownOutput = errors.New(+ "that is not an output shape khaos can write",+)++var ErrRecordUnwritten = errors.New(+ "the run finished and khaos could not write the record you asked for",+)++// outputShape refuses an unrecognised value rather than falling back to the+// terminal report: a pipeline that asked for json and silently got a banner+// fails downstream in a place that does not name the cause.+func outputShape(cmd *cobra.Command) (string, error) {+ shape, err := cmd.Flags().GetString(flagOutput)+ if err != nil {+ return "", err+ }++ switch shape {+ case outputText, outputJSON:+ return shape, nil+ default:+ return "", fmt.Errorf(+ errUnknownOutput,+ ErrUnknownOutput,+ shape,+ outputText,+ outputJSON,+ )+ }+}++// Projections are the files that are derived from the record rather than+// being it. Each is a path or empty.+type Projections struct {+ JUnit string+ Metrics string+}++func projectionsOf(cmd *cobra.Command) (Projections, error) {+ junit, err := cmd.Flags().GetString(flagJUnit)+ if err != nil {+ return Projections{}, err+ }++ metrics, err := cmd.Flags().GetString(flagMetricsOut)+ if err != nil {+ return Projections{}, err+ }++ return Projections{JUnit: junit, Metrics: metrics}, nil+}++// emitRecord writes the canonical record where it was asked for and returns+// the run's own outcome.+//+// The ORDER of the two failures matters. The verdict outranks the+// bookkeeping, so a record khaos could not write never masks a defect: a run+// that found a bypass still exits 2 and says the record is missing. But a+// record an operator explicitly asked for and did not get must not be+// SILENT either, because in CI they believe they have an artifact and do+// not. So a clean run that could not write its record fails on the write.+func emitRecord(+ out io.Writer,+ definition experiment.Definition,+ result experiment.Result,+ outcome error,+ emit Output,+) error {+ code, name := exitDetail(outcome)++ rec := record.Of(version.Version(), result, definition, code, name)++ var writeErr error++ if emit.Record != "" {+ rec, writeErr = record.Write(+ emit.Record,+ rec,+ record.EventsOf(result.Observation),+ )+ }++ if emit.Shape == outputJSON {+ body, err := record.Encode(rec)+ if err != nil {+ return err+ }++ if _, err := out.Write(body); err != nil {+ return err+ }+ }++ // The projections are derived from the record AFTER it is complete, so+ // they carry its events reference and its exit code and cannot describe+ // a different run than the one on disk.+ if err := writeProjections(rec, emit.Projections); err != nil {+ writeErr = errors.Join(writeErr, err)+ }++ if writeErr == nil {+ return outcome+ }++ // A verdict khaos reached outranks a file it could not write.+ if outcome != nil {+ fmt.Fprintf(out, warnRecordWrite, writeErr)+ return outcome+ }++ return fmt.Errorf(errRecordWrite, ErrRecordUnwritten, writeErr)+}++// writeProjections writes each derived file that was asked for. They are+// derived from the record alone, which is what makes them projections rather+// than second opinions.+func writeProjections(rec record.Record, into Projections) error {+ var failures error++ if into.JUnit != "" {+ body, err := record.JUnit(rec)+ if err != nil {+ failures = errors.Join(failures, err)+ } else {+ failures = errors.Join(+ failures,+ record.WriteFile(into.JUnit, body),+ )+ }+ }++ if into.Metrics != "" {+ failures = errors.Join(+ failures,+ record.WriteFile(into.Metrics, record.Metrics(rec)),+ )+ }++ return failures+}++// exitDetail reads the code and the outcome name back off a finished run, so+// the record can carry what khaos is about to return to the shell.+func exitDetail(outcome error) (int, string) {+ if outcome == nil {+ return CodeOK, ""+ }++ var exit *Exit+ if errors.As(outcome, &exit) {+ return exit.Code, string(exit.Outcome)+ }++ return CodeUsage, ""+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.gonew file mode 100644index 00000000..6b3b9287--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go@@ -0,0 +1,399 @@+// ©AngelaMos | 2026+// emit_test.go++package cli++import (+ "encoding/json"+ "errors"+ "os"+ "path/filepath"+ "strconv"+ "strings"+ "testing"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/record"+)++func emitted(t *testing.T) (experiment.Result, experiment.Definition) {+ t.Helper()++ definition, err := experiment.Lookup("control.policy_store_down")+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ result := held()+ result.Experiment = definition.Name+ result.Family = definition.Family+ result.Hypothesis = "a hypothesis"+ result.Fault = experiment.Fault{+ Kind: definition.FaultKind,+ Target: "a-target",+ }++ return result, definition+}++// Under --output json, stdout carries the record and NOTHING else. A single+// banner line makes `khaos run x --output json | jq .verdict` fail on the+// first byte, and that failure names jq rather than khaos.+func TestJSONOutputWritesOnlyTheRecord(t *testing.T) {+ result, definition := emitted(t)++ out := &strings.Builder{}++ if err := emitRecord(+ out,+ definition,+ result,+ nil,+ Output{Shape: outputJSON, Record: ""},+ ); err != nil {+ t.Fatalf("emit: %v", err)+ }++ body := out.String()++ if !strings.HasPrefix(body, "{") {+ t.Fatalf("stdout does not begin with the record: %.40q", body)+ }++ var parsed record.Record+ if err := json.Unmarshal([]byte(body), &parsed); err != nil {+ t.Fatalf("stdout is not a single JSON object: %v", err)+ }++ if parsed.Experiment.Name != definition.Name {+ t.Fatalf("the record names %q", parsed.Experiment.Name)+ }+}++// The text shape writes no record to stdout at all, or every existing+// consumer of the terminal report gets a wall of JSON appended to it.+func TestTextOutputWritesNoRecordToStdout(t *testing.T) {+ result, definition := emitted(t)++ out := &strings.Builder{}++ if err := emitRecord(+ out,+ definition,+ result,+ nil,+ Output{Shape: outputText, Record: ""},+ ); err != nil {+ t.Fatalf("emit: %v", err)+ }++ if out.String() != "" {+ t.Fatalf("the text shape wrote to stdout: %.60q", out.String())+ }+}++// The record carries the code khaos is about to return, because the exit+// status is the only part of a run most CI ever sees.+func TestTheRecordCarriesTheExitItIsAbout(t *testing.T) {+ result, definition := emitted(t)++ for _, row := range []struct {+ name string+ outcome error+ code int+ label string+ }{+ {"a clean run", nil, CodeOK, ""},+ {+ "a defect",+ &Exit{+ Code: CodeDefect,+ Outcome: OutcomeDefect,+ Err: ErrDefectFound,+ },+ CodeDefect,+ string(OutcomeDefect),+ },+ {+ "thin evidence",+ &Exit{+ Code: CodeInconclusive,+ Outcome: OutcomeInconclusive,+ Err: ErrInconclusive,+ },+ CodeInconclusive,+ string(OutcomeInconclusive),+ },+ } {+ t.Run(row.name, func(t *testing.T) {+ out := &strings.Builder{}++ err := emitRecord(+ out,+ definition,+ result,+ row.outcome,+ Output{Shape: outputJSON, Record: ""},+ )++ if !errors.Is(err, row.outcome) {+ t.Fatalf("emit returned %v, want %v", err, row.outcome)+ }++ var parsed record.Record+ if decodeErr := json.Unmarshal(+ []byte(out.String()),+ &parsed,+ ); decodeErr != nil {+ t.Fatalf("decode: %v", decodeErr)+ }++ if parsed.Exit.Code != row.code {+ t.Errorf(+ "the record says exit %d and khaos returns %d",+ parsed.Exit.Code,+ row.code,+ )+ }++ if parsed.Exit.Outcome != row.label {+ t.Errorf(+ "the record names the outcome %q, want %q",+ parsed.Exit.Outcome,+ row.label,+ )+ }+ })+ }+}++// A verdict khaos reached outranks a file it could not write: a run that+// found a bypass still exits on the bypass.+func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {+ result, definition := emitted(t)++ defect := &Exit{+ Code: CodeDefect,+ Outcome: OutcomeDefect,+ Err: ErrDefectFound,+ }++ out := &strings.Builder{}++ err := emitRecord(+ out,+ definition,+ result,+ defect,+ Output{+ Shape: outputText,+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),+ },+ )++ if !errors.Is(err, ErrDefectFound) {+ t.Fatalf("the defect was masked by the write failure: %v", err)+ }++ // And the failure is still SAID, because an operator who asked for a+ // record and did not get one must not have to guess.+ if !strings.Contains(out.String(), "could not write the run record") {+ t.Fatalf("the write failure was silent: %.80q", out.String())+ }+}++// But a CLEAN run that could not write the record the operator asked for+// fails on the write: in CI they believe they have an artifact and do not.+func TestACleanRunFailsWhenItsRecordCannotBeWritten(t *testing.T) {+ result, definition := emitted(t)++ err := emitRecord(+ &strings.Builder{},+ definition,+ result,+ nil,+ Output{+ Shape: outputText,+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),+ },+ )++ if !errors.Is(err, ErrRecordUnwritten) {+ t.Fatalf("a missing record was silent on a clean run: %v", err)+ }+}++// THE PROJECTION INVARIANT. The record is canonical and the stdout form is a+// projection of it, so the bytes on stdout and the bytes on disk have to be+// the same record. If they can differ, "the pretty output says one thing and+// the JSON says another" is back.+func TestTheRecordOnStdoutIsTheRecordOnDisk(t *testing.T) {+ result, definition := emitted(t)+ dir := t.TempDir()++ out := &strings.Builder{}++ if err := emitRecord(+ out,+ definition,+ result,+ nil,+ Output{Shape: outputJSON, Record: dir},+ ); err != nil {+ t.Fatalf("emit: %v", err)+ }++ onDisk, err := os.ReadFile(filepath.Join(dir, record.RecordFile))+ if err != nil {+ t.Fatalf("read the record: %v", err)+ }++ if out.String() != string(onDisk) {+ t.Fatal(+ "the record on stdout and the record on disk are not the same bytes",+ )+ }+}++// An unrecognised --output is refused rather than falling back to the+// terminal report: a pipeline that asked for json and silently got a banner+// fails somewhere that does not name the cause.+func TestAnUnknownOutputShapeIsRefused(t *testing.T) {+ cmd := find(t, NewRoot(), "run")++ sub := find(t, cmd, "control.policy_store_down")++ if err := sub.Flags().Set(flagOutput, "yaml"); err != nil {+ t.Fatalf("set: %v", err)+ }++ _, err := outputShape(sub)+ if !errors.Is(err, ErrUnknownOutput) {+ t.Fatalf("an unknown output shape was accepted: %v", err)+ }+}++// Both shapes are accepted, so the refusal above cannot be passing because+// outputShape refuses everything.+func TestBothOutputShapesAreAccepted(t *testing.T) {+ cmd := find(t, NewRoot(), "run")+ sub := find(t, cmd, "control.policy_store_down")++ for _, shape := range []string{outputText, outputJSON} {+ if err := sub.Flags().Set(flagOutput, shape); err != nil {+ t.Fatalf("set %q: %v", shape, err)+ }++ got, err := outputShape(sub)+ if err != nil {+ t.Fatalf("outputShape(%q): %v", shape, err)+ }++ if got != shape {+ t.Fatalf("outputShape(%q) = %q", shape, got)+ }+ }+}++// Both projections are written from the FINISHED record, so they describe+// the same run as the file on disk. Deriving either one before the record is+// complete is how a JUnit suite ends up naming a different exit code than+// the record beside it.+func TestTheProjectionsDescribeTheSameRunAsTheRecord(t *testing.T) {+ result, definition := emitted(t)+ dir := t.TempDir()++ defect := &Exit{+ Code: CodeDefect,+ Outcome: OutcomeDefect,+ Err: ErrDefectFound,+ }++ junit := filepath.Join(dir, "khaos.xml")+ metrics := filepath.Join(dir, "khaos.prom")++ err := emitRecord(+ &strings.Builder{},+ definition,+ result,+ defect,+ Output{+ Shape: outputText,+ Record: dir,+ Projections: Projections{JUnit: junit, Metrics: metrics},+ },+ )++ if !errors.Is(err, ErrDefectFound) {+ t.Fatalf("emit: %v", err)+ }++ onDisk, err := os.ReadFile(filepath.Join(dir, record.RecordFile))+ if err != nil {+ t.Fatalf("read the record: %v", err)+ }++ var parsed record.Record+ if err := json.Unmarshal(onDisk, &parsed); err != nil {+ t.Fatalf("decode: %v", err)+ }++ xmlBody, err := os.ReadFile(junit)+ if err != nil {+ t.Fatalf("read the junit: %v", err)+ }++ promBody, err := os.ReadFile(metrics)+ if err != nil {+ t.Fatalf("read the metrics: %v", err)+ }++ // The run id ties all three together, and the exit code is the thing+ // most likely to be computed twice and disagree.+ for name, body := range map[string]string{+ "junit": string(xmlBody),+ "metrics": string(promBody),+ } {+ if !strings.Contains(body, parsed.RunID) {+ t.Errorf("the %s does not carry the record's run id", name)+ }++ if !strings.Contains(body, strconv.Itoa(parsed.Exit.Code)) {+ t.Errorf(+ "the %s does not carry the record's exit code %d",+ name,+ parsed.Exit.Code,+ )+ }+ }++ // And the record's own events reference survived being written before+ // the projections were derived from it.+ if parsed.Events == nil || parsed.Events.SHA256 == "" {+ t.Error("the record on disk carries no events reference")+ }+}++// A projection khaos could not write follows the same rule as the record: a+// verdict outranks it, and it is never silent.+func TestAProjectionThatCannotBeWrittenIsNotSilent(t *testing.T) {+ result, definition := emitted(t)++ err := emitRecord(+ &strings.Builder{},+ definition,+ result,+ nil,+ Output{+ Shape: outputText,+ Record: "",+ Projections: Projections{+ JUnit: filepath.Join("/proc", "nope", "khaos.xml"),+ },+ },+ )++ if !errors.Is(err, ErrRecordUnwritten) {+ t.Fatalf("a projection khaos could not write was silent: %v", err)+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit.gonew file mode 100644index 00000000..b64a95dc--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit.go@@ -0,0 +1,207 @@+// ©AngelaMos | 2026+// exit.go++package cli++import (+ "errors"+ "fmt"+ "slices"+ "strings"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+)++// The exit contract. A single non-zero code would collapse "a control failed+// open", "the experiment was invalid" and "khaos crashed" into one signal, and+// those need three different human responses.+const (+ CodeOK = 0+ CodeUsage = 1+ CodeDefect = 2+ CodeDeviated = 3+ CodeInconclusive = 4+ CodeProbe = 5+ CodeRollback = 6+ CodeVoid = 7+ CodeBaseline = 8+ CodeUnsupported = 9+ CodeInterrupted = 10+ CodeInternal = 70+)++// Outcome names an exit code for --fail-on. The names are the operator's+// vocabulary, so they are the same words the verdict prints.+type Outcome string++const (+ OutcomeDefect Outcome = "defect"+ OutcomeDeviated Outcome = "deviated"+ OutcomeInconclusive Outcome = "inconclusive"+ OutcomeProbe Outcome = "probe-error"+ OutcomeVoid Outcome = "void"+ OutcomeBaseline Outcome = "baseline-not-met"+ OutcomeUnsupported Outcome = "unsupported"+ OutcomeInterrupted Outcome = "interrupted"++ // outcomeRollback is deliberately unexported and absent from+ // suppressible(): it names a code no gate may silence, so an operator+ // must not be able to type it into --fail-on and believe it did+ // something.+ outcomeRollback Outcome = "rollback-failed"++ // FailOnAny is the default: every outcome khaos declines to call a clean+ // pass keeps its own non-zero code. FailOnNever reports every measurable+ // outcome as 0 and leaves the detail to the run record.+ FailOnAny = "any"+ FailOnNever = "never"+)++// Exit carries the code beside the error, so the precedence is decided once+// where the run finishes rather than re-derived by a growing chain of+// errors.Is checks at the process boundary.+type Exit struct {+ Code int+ Outcome Outcome+ Err error+}++func (e *Exit) Error() string { return e.Err.Error() }+func (e *Exit) Unwrap() error { return e.Err }++// suppressible is every outcome --fail-on may silence. Rollback failure,+// usage and internal errors are deliberately absent: an unreverted fault+// leaves the environment dirty for every later job on that runner, and a tool+// that crashed or could not read its own flags is not reporting a+// measurement at all.+func suppressible() []Outcome {+ return []Outcome{+ OutcomeDefect,+ OutcomeDeviated,+ OutcomeInconclusive,+ OutcomeProbe,+ OutcomeVoid,+ OutcomeBaseline,+ OutcomeUnsupported,+ OutcomeInterrupted,+ }+}++var ErrUnknownOutcome = errors.New(+ "that is not an outcome khaos can gate on",+)++// Gate is the parsed --fail-on set.+type Gate struct {+ all bool+ none bool+ allowed []Outcome+}++// ParseGate refuses an unknown name rather than quietly gating on nothing,+// because a typo that silenced the defect code would turn every run green.+func ParseGate(value string) (Gate, error) {+ trimmed := strings.TrimSpace(value)++ switch trimmed {+ case "", FailOnAny:+ return Gate{all: true}, nil+ case FailOnNever:+ return Gate{none: true}, nil+ }++ gate := Gate{}+ known := suppressible()++ for _, name := range strings.Split(trimmed, ",") {+ outcome := Outcome(strings.TrimSpace(name))++ if !slices.Contains(known, outcome) {+ return Gate{}, fmt.Errorf(+ errUnknownOutcome,+ ErrUnknownOutcome,+ outcome,+ joinOutcomes(known),+ )+ }++ gate.allowed = append(gate.allowed, outcome)+ }++ return gate, nil+}++// Apply narrows a code to 0 when its outcome is outside the gate. An outcome+// this gate cannot suppress passes through whatever the gate says.+func (g Gate) Apply(exit *Exit) int {+ if exit == nil {+ return CodeOK+ }++ if !slices.Contains(suppressible(), exit.Outcome) {+ return exit.Code+ }++ if g.all {+ return exit.Code+ }++ if g.none {+ return CodeOK+ }++ if slices.Contains(g.allowed, exit.Outcome) {+ return exit.Code+ }++ return CodeOK+}++func joinOutcomes(outcomes []Outcome) string {+ names := make([]string, 0, len(outcomes))+ for _, outcome := range outcomes {+ names = append(names, string(outcome))+ }++ return strings.Join(names, listSep)+}++// epistemicExit maps the one axis that is not about the target's direction.+// Every value is reachable and assigned at a real site in the verdict engine,+// so none of these is an inert code.+func epistemicExit(epistemic experiment.Epistemic) (int, Outcome, bool) {+ switch epistemic {+ case experiment.ProbeError:+ return CodeProbe, OutcomeProbe, true+ case experiment.Aborted:+ return CodeVoid, OutcomeVoid, true+ case experiment.BaselineNotMet:+ return CodeBaseline, OutcomeBaseline, true+ case experiment.Unsupported:+ return CodeUnsupported, OutcomeUnsupported, true+ case experiment.Interrupted:+ return CodeInterrupted, OutcomeInterrupted, true+ default:+ return 0, "", false+ }+}++// gated narrows a run's own exit to what --fail-on allows. A nil result is a+// clean pass; an error that is not an Exit came from somewhere the gate has+// no opinion about and passes through untouched.+func gated(gate Gate, err error) error {+ if err == nil {+ return nil+ }++ var exit *Exit+ if !errors.As(err, &exit) {+ return err+ }++ if gate.Apply(exit) == CodeOK {+ return nil+ }++ return err+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.gonew file mode 100644index 00000000..78e21e0d--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go@@ -0,0 +1,443 @@+// ©AngelaMos | 2026+// exit_test.go++package cli++import (+ "errors"+ "slices"+ "testing"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/run"+)++// held is a finished run with nothing wrong with it, which every case below+// mutates one field of. Building it here rather than per case is what keeps a+// case honest: a case that passes because it forgot to set Conclusive is+// testing the fixture, not the mapping.+func held() experiment.Result {+ return experiment.Result{+ Verdict: experiment.Verdict{+ Epistemic: experiment.Passed,+ Direction: experiment.Held,+ Recovery: experiment.Recovered,+ Conclusive: true,+ },+ }+}++func codeOf(t *testing.T, result experiment.Result) int {+ t.Helper()++ err := exitFor(result, nil)+ if err == nil {+ return CodeOK+ }++ var exit *Exit+ if !errors.As(err, &exit) {+ t.Fatalf("exitFor returned an error carrying no code: %v", err)+ }++ return exit.Code+}++// Every epistemic value is reachable and assigned at a real site in the+// verdict engine, so every one needs its own code. A single non-zero exit+// would collapse "a control failed open", "the experiment was invalid" and+// "khaos could not measure" into one signal, and those are three different+// human responses.+func TestEveryReachableOutcomeHasItsOwnExitCode(t *testing.T) {+ for _, row := range []struct {+ name string+ shape func(*experiment.Result)+ want int+ }{+ {+ "a clean held run",+ func(*experiment.Result) {},+ CodeOK,+ },+ {+ "the direction this experiment calls a defect",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.Deviated+ r.Verdict.Direction = experiment.Permitted+ r.Verdict.Defect = true+ },+ CodeDefect,+ },+ {+ "deviated in a direction it does NOT call a defect",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.Deviated+ r.Verdict.Direction = experiment.Denied+ },+ CodeDeviated,+ },+ {+ "held on evidence khaos declined to call conclusive",+ func(r *experiment.Result) {+ r.Verdict.Conclusive = false+ },+ CodeInconclusive,+ },+ {+ "khaos's own measurement failed",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.ProbeError+ },+ CodeProbe,+ },+ {+ "the fault was never confirmed",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.Aborted+ },+ CodeVoid,+ },+ {+ "the target was broken before khaos injected",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.BaselineNotMet+ },+ CodeBaseline,+ },+ {+ "the question does not apply to this target",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.Unsupported+ },+ CodeUnsupported,+ },+ {+ "the run was interrupted",+ func(r *experiment.Result) {+ r.Verdict.Epistemic = experiment.Interrupted+ },+ CodeInterrupted,+ },+ {+ "the fault could not be reverted",+ func(r *experiment.Result) {+ r.Observation.RollbackFailed = true+ },+ CodeRollback,+ },+ {+ "the target never came back",+ func(r *experiment.Result) {+ r.Verdict.Recovery = experiment.Degraded+ },+ CodeRollback,+ },+ {+ "recovery was never observed",+ func(r *experiment.Result) {+ r.Verdict.Recovery = experiment.Unobserved+ },+ CodeInconclusive,+ },+ } {+ t.Run(row.name, func(t *testing.T) {+ result := held()+ row.shape(&result)++ if got := codeOf(t, result); got != row.want {+ t.Fatalf("exit code %d, want %d", got, row.want)+ }+ })+ }+}++// No two outcomes may share a code, or CI cannot tell them apart, which is+// the entire reason the taxonomy exists. Interrupted and inconclusive were+// both literally 4 in the shipped binary.+func TestNoTwoOutcomesShareACode(t *testing.T) {+ codes := map[int]Outcome{}++ for _, outcome := range append(suppressible(), outcomeRollback) {+ code := codeFor(t, outcome)++ if other, clash := codes[code]; clash {+ t.Fatalf(+ "%s and %s both exit %d, so nothing downstream can tell them apart",+ outcome,+ other,+ code,+ )+ }++ codes[code] = outcome+ }+}++// codeFor walks the mapping the other way, through the states that produce+// each outcome, so this census reads the real function rather than a table+// written beside it.+func codeFor(t *testing.T, outcome Outcome) int {+ t.Helper()++ result := held()++ switch outcome {+ case OutcomeDefect:+ result.Verdict.Epistemic = experiment.Deviated+ result.Verdict.Defect = true+ case OutcomeDeviated:+ result.Verdict.Epistemic = experiment.Deviated+ case OutcomeInconclusive:+ result.Verdict.Conclusive = false+ case OutcomeProbe:+ result.Verdict.Epistemic = experiment.ProbeError+ case OutcomeVoid:+ result.Verdict.Epistemic = experiment.Aborted+ case OutcomeBaseline:+ result.Verdict.Epistemic = experiment.BaselineNotMet+ case OutcomeUnsupported:+ result.Verdict.Epistemic = experiment.Unsupported+ case OutcomeInterrupted:+ result.Verdict.Epistemic = experiment.Interrupted+ case outcomeRollback:+ result.Observation.RollbackFailed = true+ default:+ t.Fatalf("no state produces the outcome %q", outcome)+ }++ return codeOf(t, result)+}++// A defect outranks the epistemic axis, because one leaked request is a proof+// and khaos reports a proven bypass even on thin evidence.+func TestADefectOutranksThinEvidence(t *testing.T) {+ result := held()+ result.Verdict.Epistemic = experiment.Deviated+ result.Verdict.Defect = true+ result.Verdict.Conclusive = false++ if got := codeOf(t, result); got != CodeDefect {+ t.Fatalf("exit code %d, want the defect code %d", got, CodeDefect)+ }+}++// And the state khaos left the target in outranks the defect.+func TestRollbackOutranksTheDefect(t *testing.T) {+ result := held()+ result.Verdict.Defect = true+ result.Observation.RollbackFailed = true++ if got := codeOf(t, result); got != CodeRollback {+ t.Fatalf("exit code %d, want the rollback code %d", got, CodeRollback)+ }+}++func TestTheDefaultGateNarrowsNothing(t *testing.T) {+ gate, err := ParseGate("")+ if err != nil {+ t.Fatalf("the empty default was refused: %v", err)+ }++ for _, outcome := range suppressible() {+ exit := &Exit{Code: codeFor(t, outcome), Outcome: outcome}++ if got := gate.Apply(exit); got == CodeOK {+ t.Fatalf(+ "the default gate silenced %s; a run khaos declined to conclude from exiting 0 is a green build on no evidence",+ outcome,+ )+ }+ }+}++func TestGatingOnTheDefectAloneSilencesTheRest(t *testing.T) {+ gate, err := ParseGate(string(OutcomeDefect))+ if err != nil {+ t.Fatalf("parse: %v", err)+ }++ for _, outcome := range suppressible() {+ exit := &Exit{Code: codeFor(t, outcome), Outcome: outcome}+ got := gate.Apply(exit)++ if outcome == OutcomeDefect {+ if got != CodeDefect {+ t.Fatalf("the gated outcome was silenced: %d", got)+ }++ continue+ }++ if got != CodeOK {+ t.Fatalf(+ "%s survived a gate that does not name it: %d",+ outcome,+ got,+ )+ }+ }+}++// The two codes no gate may touch, asserted against EVERY gate value rather+// than against the default: a --fail-on that could suppress an unreverted+// fault would leave the environment dirty for every later job on the runner.+func TestNoGateCanSuppressARollbackFailure(t *testing.T) {+ for _, value := range []string{+ FailOnAny,+ FailOnNever,+ string(OutcomeDefect),+ string(OutcomeInconclusive) + "," + string(OutcomeVoid),+ } {+ t.Run(value, func(t *testing.T) {+ gate, err := ParseGate(value)+ if err != nil {+ t.Fatalf("parse: %v", err)+ }++ exit := &Exit{+ Code: CodeRollback,+ Outcome: outcomeRollback,+ Err: run.ErrRevertFailed,+ }++ if got := gate.Apply(exit); got != CodeRollback {+ t.Fatalf(+ "--fail-on %s silenced an unreverted fault, which leaves the environment dirty",+ value,+ )+ }+ })+ }+}++// `never` is the one value that reports every measurable outcome as 0, and it+// still cannot reach the rollback code above.+func TestNeverSilencesEveryMeasurableOutcome(t *testing.T) {+ gate, err := ParseGate(FailOnNever)+ if err != nil {+ t.Fatalf("parse: %v", err)+ }++ for _, outcome := range suppressible() {+ exit := &Exit{Code: codeFor(t, outcome), Outcome: outcome}++ if got := gate.Apply(exit); got != CodeOK {+ t.Fatalf("--fail-on never left %s at %d", outcome, got)+ }+ }+}++// A typo that silenced the defect code would turn every run green, so an+// unknown name is refused rather than gating on nothing.+func TestAnUnknownOutcomeIsRefusedByName(t *testing.T) {+ for _, value := range []string{+ "open",+ "defect,typo",+ "rollback-failed",+ "DEFECT",+ } {+ t.Run(value, func(t *testing.T) {+ _, err := ParseGate(value)+ if !errors.Is(err, ErrUnknownOutcome) {+ t.Fatalf("ParseGate(%q) did not refuse by name: %v", value, err)+ }+ })+ }+}++// The gate's vocabulary is the verdict's own vocabulary, so an operator reads+// an outcome in the report and can type it into the flag.+func TestTheGateNamesMatchTheVerdictsOwnWords(t *testing.T) {+ for _, row := range []struct {+ outcome Outcome+ epistemic experiment.Epistemic+ }{+ {OutcomeProbe, experiment.ProbeError},+ {OutcomeVoid, experiment.Aborted},+ {OutcomeBaseline, experiment.BaselineNotMet},+ {OutcomeUnsupported, experiment.Unsupported},+ {OutcomeInterrupted, experiment.Interrupted},+ } {+ // `void` is deliberately not `aborted`: the epistemic word describes+ // what khaos did and the gate word describes what the operator got.+ if row.outcome == OutcomeVoid {+ continue+ }++ if string(row.outcome) != string(row.epistemic) {+ t.Errorf(+ "the gate calls it %q and the verdict prints %q",+ row.outcome,+ row.epistemic,+ )+ }+ }+}++// gated is what turns a narrowed code into a nil error, so a suppressed+// outcome genuinely exits 0 rather than printing a failure with code 0.+func TestASuppressedOutcomeReturnsNoError(t *testing.T) {+ gate, err := ParseGate(FailOnNever)+ if err != nil {+ t.Fatalf("parse: %v", err)+ }++ suppressed := gated(gate, &Exit{+ Code: CodeInconclusive,+ Outcome: OutcomeInconclusive,+ Err: ErrInconclusive,+ })++ if suppressed != nil {+ t.Fatalf("a suppressed outcome still returned an error: %v", suppressed)+ }++ // And an error the gate has no opinion about passes through untouched.+ foreign := errors.New("the daemon refused the connection")+ if got := gated(gate, foreign); !errors.Is(got, foreign) {+ t.Fatalf("a non-verdict error was swallowed by the gate: %v", got)+ }+}++// Nothing may quietly drop out of the gate's vocabulary: a code with no name+// is a code no operator can gate on.+func TestEverySuppressibleOutcomeIsParseable(t *testing.T) {+ for _, outcome := range suppressible() {+ gate, err := ParseGate(string(outcome))+ if err != nil {+ t.Fatalf("ParseGate(%q): %v", outcome, err)+ }++ if !slices.Contains(gate.allowed, outcome) {+ t.Fatalf("%q parsed into a gate that does not contain it", outcome)+ }+ }+}++// The LITERAL numbers, because they are a published contract: a CI pipeline+// keys off the integer, not off the name of the constant. Every other test+// here compares a state against a named code, so all of them would survive a+// wholesale renumbering together. This one would not.+func TestTheExitCodesAreTheirPublishedNumbers(t *testing.T) {+ for name, want := range map[string]struct {+ got int+ want int+ }{+ "OK": {CodeOK, 0},+ "USAGE": {CodeUsage, 1},+ "DEFECT": {CodeDefect, 2},+ "DEVIATED": {CodeDeviated, 3},+ "INCONCLUSIVE": {CodeInconclusive, 4},+ "PROBE": {CodeProbe, 5},+ "ROLLBACK": {CodeRollback, 6},+ "VOID": {CodeVoid, 7},+ "BASELINE": {CodeBaseline, 8},+ "UNSUPPORTED": {CodeUnsupported, 9},+ "INTERRUPTED": {CodeInterrupted, 10},+ "INTERNAL": {CodeInternal, 70},+ } {+ if want.got != want.want {+ t.Errorf("%s is %d, want %d", name, want.got, want.want)+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.goindex 3946b7a7..b5f1f8fb 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go@@ -42,6 +42,24 @@ const (flagFaultFor = "fault-for"flagRecoverFor = "recover-for"flagSampleEvery = "sample-every"+ flagFailOn = "fail-on"+ flagRecord = "record"+ flagOutput = "output"+ flagJUnit = "junit"+ flagMetricsOut = "metrics-out"++ outputText = "text"+ outputJSON = "json"++ recordUsage = "a directory to write the canonical run record into, as `run.json` plus an `events.jsonl` beside it carrying one line per observation. The record is what every other output khaos produces is a projection of, and it is the only form that survives the terminal scrolling away"++ outputUsage = "`text` for the terminal report, or `json` to write the run record to stdout and suppress every other line, so `khaos run x --output json | jq .verdict` works"++ junitUsage = "a file to write JUnit XML to, for CI test reporters. A run khaos declined to conclude from becomes a `skipped` rather than a `failure`, which is what stops thin evidence breaking a build on noise; only a control that failed in the direction this experiment declares a defect becomes a `failure`"++ metricsUsage = "a file to write Prometheus text exposition to, for node_exporter's textfile collector. Written atomically and consumed by pointing that collector at the directory; khaos stands up no server and opens no port"++ failOnUsage = "which outcomes keep a non-zero exit code. `any` is the default and narrows nothing, because a run khaos declined to conclude from exiting 0 is a green build on no evidence; `never` reports every measurable outcome as 0 and leaves the detail to the run record; or name the outcomes to gate on, comma separated, where `defect` alone is the shape a pipeline that should only break on a security regression wants. A rollback failure, a usage error and an internal error ignore this flag, because an unreverted fault makes every later job on that runner untrustworthy"catalogShort = "list the experiments khaos ships"catalogLong = `Each experiment pairs a fault with two probes: one carrying a credential the@@ -106,6 +124,7 @@ enforcing in the first place.`errNoExperiment = "name the experiment to %s (known experiments: %s)"errNoRangeFor = "%w by passing %s"errInconclusive = "%w (%s)"+ errUnknownOutcome = "%w: %q (khaos gates on: %s, or `any`, or `never`)"verdictThinEvidence = "the sample cannot rule out a bypass"verdictRecoveryUnobserved = "recovery was never observed"errNoProxiedPath = "%w: this experiment injects through khaos's fault proxy, and the running range has no %s; bring the range down and up again to create it"@@ -172,6 +191,10 @@ var ErrInconclusive = errors.New("this run did not reach a conclusion, so nothing can be concluded from it either way",)+var ErrDeviated = errors.New(+ "the target broke in a direction this experiment does not declare a defect, which is the correct outcome with an availability cost khaos did not measure",+)+func newCatalogCmd() *cobra.Command {return &cobra.Command{Use: "catalog",@@ -277,18 +300,57 @@ func newRunExperimentCmd(definition experiment.Definition) *cobra.Command {if err != nil {return err}++ // Parsed BEFORE the run, so a typo that would have silenced the+ // defect code is refused instead of discovered afterwards.+ raw, err := cmd.Flags().GetString(flagFailOn)+ if err != nil {+ return err+ }++ gate, err := ParseGate(raw)+ if err != nil {+ return err+ }++ shape, err := outputShape(cmd)+ if err != nil {+ return err+ }++ into, err := cmd.Flags().GetString(flagRecord)+ if err != nil {+ return err+ }++ projections, err := projectionsOf(cmd)+ if err != nil {+ return err+ }+return runExperiment(cmd.Context(),cmd.OutOrStdout(),definition,ttl,given,+ gate,+ Output{+ Shape: shape,+ Record: into,+ Projections: projections,+ },)},}experimentFlags(cmd, definition)cmd.Flags().Duration(flagTTL, lease.DefaultTTL, ttlUsage)+ cmd.Flags().String(flagFailOn, FailOnAny, failOnUsage)+ cmd.Flags().String(flagRecord, "", recordUsage)+ cmd.Flags().String(flagOutput, outputText, outputUsage)+ cmd.Flags().String(flagJUnit, "", junitUsage)+ cmd.Flags().String(flagMetricsOut, "", metricsUsage)return cmd}@@ -518,51 +580,116 @@ func describe(out io.Writer, spec experiment.Experiment, resolved string) {fmt.Fprintln(out)}-func runExperiment(+// prepared is everything a run needs resolved before anything is injected:+// the workspace lock, the experiment with its parameters bound, and the+// container whose consent khaos checked.+type prepared struct {+ work *workspace.Workspace+ spec experiment.Experiment+ accepted dockerx.Target+ guard dockerx.Guard+ project string+}++// prepareRun does every step that can still refuse. Separated from the run+// itself because nothing here touches the target: if this returns an error,+// khaos has injected nothing and has nothing to revert.+//+// The caller owns closing the workspace, because the lock has to outlive+// this function and be released after the run rather than before it.+func prepareRun(ctx context.Context,- out io.Writer,+ human io.Writer,definition experiment.Definition,- ttl time.Duration,given inputs,-) error {+) (prepared, error) {work, err := workspace.OpenExclusive(ctx)if err != nil {- return err+ return prepared{}, err+ }++ out := prepared{work: work, project: given.Project}+ out.guard = dockerx.Guard{+ Project: out.project,+ SelfID: dockerx.SelfContainerID(),}- defer work.Close()supplied, err := fillFromRange(ctx,- out,+ human,work.Docker,definition,given.Supplied,)if err != nil {- return err+ return out, err}- spec, err := experiment.Build(definition.Name, supplied, given.Timing)+ out.spec, err = experiment.Build(+ definition.Name,+ supplied,+ given.Timing,+ )if err != nil {- return err+ return out, err}- project := given.Project- guard := dockerx.Guard{Project: project, SelfID: dockerx.SelfContainerID()}-- accepted, err := work.Docker.Accept(+ out.accepted, err = work.Docker.Accept(ctx,- spec.Fault.Target,- dockerx.Filter{Project: project},- guard,+ out.spec.Fault.Target,+ dockerx.Filter{Project: out.project},+ out.guard,)++ return out, err+}++// Output is where a run's results go. Grouped because they are one decision+// -- what this invocation should emit -- rather than three.+type Output struct {+ Shape string+ Record string+ Projections Projections+}++// human is where the terminal report goes, and under --output json that is+// nowhere: stdout has to carry the record and nothing else, or piping it+// into jq fails on the first banner line.+func (o Output) human(out io.Writer) io.Writer {+ if o.Shape == outputJSON {+ return io.Discard+ }++ return out+}++func runExperiment(+ ctx context.Context,+ out io.Writer,+ definition experiment.Definition,+ ttl time.Duration,+ given inputs,+ gate Gate,+ emit Output,+) error {+ human := emit.human(out)++ ready, err := prepareRun(ctx, human, definition, given)+ if ready.work != nil {+ defer ready.work.Close()+ }+if err != nil {return err}+ work := ready.work+ spec, accepted := ready.spec, ready.accepted+ project, guard := ready.project, ready.guard+env, closeTiers, err := standUpTiers(ctx,- out,+ human,work,&spec,accepted,@@ -583,21 +710,28 @@ func runExperiment(session := run.NewSession(env, work.Journal, ttl)engine := wire(- out,+ human,session,work.Docker,dockerx.Filter{Project: project},guard,)- describe(out, spec, accepted.Name)+ describe(human, spec, accepted.Name)result, runErr := underLease(ctx, engine, spec, session)result.RunID = session.RunID()- reportVerdict(out, result)- return exitFor(result, runErr)+ // The exit is decided BEFORE anything is reported, because the record+ // carries the code: the exit status is the only part of a run most CI+ // ever sees, and a record that cannot explain it is not a record of the+ // run.+ outcome := gated(gate, exitFor(result, runErr))++ reportVerdict(human, result)++ return emitRecord(out, definition, result, outcome, emit)}func wire(@@ -690,45 +824,87 @@ func exitFor(result experiment.Result, runErr error) error {return runErr}+ // The state khaos left the target in outranks every verdict about it, and+ // no --fail-on value may suppress either.if result.Observation.RollbackFailed {- return run.ErrRevertFailed+ return &Exit{+ Code: CodeRollback,+ Outcome: outcomeRollback,+ Err: run.ErrRevertFailed,+ }}if result.Verdict.Recovery == experiment.Degraded {- return ErrNotRecovered+ return &Exit{+ Code: CodeRollback,+ Outcome: outcomeRollback,+ Err: ErrNotRecovered,+ }}+ // The DEFECT outranks the epistemic axis, because a proven bypass is+ // reported even on thin evidence: one leaked request is a proof.if result.Verdict.Defect {- return ErrDefectFound+ return &Exit{+ Code: CodeDefect,+ Outcome: OutcomeDefect,+ Err: ErrDefectFound,+ }}- if !conclusive(result.Verdict.Epistemic) {- return fmt.Errorf(- errInconclusive,- ErrInconclusive,- result.Verdict.Epistemic,- )+ if code, outcome, matched := epistemicExit(+ result.Verdict.Epistemic,+ ); matched {+ return &Exit{+ Code: code,+ Outcome: outcome,+ Err: fmt.Errorf(+ errInconclusive,+ ErrInconclusive,+ result.Verdict.Epistemic,+ ),+ }}+ // Deviated in a direction this experiment's polarity does NOT call a+ // defect: correct, with an availability cost khaos did not measure.+ if result.Verdict.Epistemic == experiment.Deviated {+ return &Exit{+ Code: CodeDeviated,+ Outcome: OutcomeDeviated,+ Err: ErrDeviated,+ }+ }++ // A HELD verdict khaos declined to call conclusive. Exiting 0 here would+ // assert a control held on evidence khaos itself refused to stand behind.if !result.Verdict.Conclusive {- return fmt.Errorf(errInconclusive, ErrInconclusive, verdictThinEvidence)+ return &Exit{+ Code: CodeInconclusive,+ Outcome: OutcomeInconclusive,+ Err: fmt.Errorf(+ errInconclusive,+ ErrInconclusive,+ verdictThinEvidence,+ ),+ }}if result.Verdict.Recovery == experiment.Unobserved {- return fmt.Errorf(- errInconclusive,- ErrInconclusive,- verdictRecoveryUnobserved,- )+ return &Exit{+ Code: CodeInconclusive,+ Outcome: OutcomeInconclusive,+ Err: fmt.Errorf(+ errInconclusive,+ ErrInconclusive,+ verdictRecoveryUnobserved,+ ),+ }}return nil}-func conclusive(epistemic experiment.Epistemic) bool {- return epistemic == experiment.Passed || epistemic == experiment.Deviated-}-func reportVerdict(out io.Writer, result experiment.Result) {fmt.Fprintln(out)diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.goindex 34a7ae6f..aad327f0 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go@@ -441,9 +441,23 @@ func TestEveryExperimentRegistersExactlyItsDeclaredParameters(t *testing.T) {}registered := stringFlags(cmd)- want := append(- []string{flagProject},- names(definition.Params)...)++ // These three belong to `run` alone: `plan` executes+ // nothing, so it has no outcome to gate and no record to+ // write, and a flag there would do nothing.+ extra := []string{flagProject}+ if verb == "run" {+ extra = append(+ extra,+ flagFailOn,+ flagRecord,+ flagOutput,+ flagJUnit,+ flagMetricsOut,+ )+ }++ want := slices.Concat(extra, names(definition.Params))slices.Sort(registered)slices.Sort(want)diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.goindex ffadf81b..a12f8e1f 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go@@ -231,7 +231,9 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {timing := spec.Timing.withDefaults()result := Result{Experiment: spec.Name,+ Family: spec.Family,Hypothesis: spec.Hypothesis.String(),+ Fault: spec.Fault,StartedAt: e.now(),}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.goindex 4e70a26f..31537e00 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go@@ -200,9 +200,16 @@ type Experiment struct {}type Result struct {- Experiment string `json:"experiment"`- RunID string `json:"run_id"`- Hypothesis string `json:"hypothesis"`+ Experiment string `json:"experiment"`+ Family string `json:"family"`+ RunID string `json:"run_id"`+ Hypothesis string `json:"hypothesis"`++ // Fault is carried on the RESULT rather than left on the spec, because+ // the record is the product: a record that cannot say what was injected+ // or where is not a record of the run.+ Fault Fault `json:"fault"`+StartedAt time.Time `json:"started_at"`Duration time.Duration `json:"duration"`Observation Observation `json:"observation"`diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/junit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/junit.gonew file mode 100644index 00000000..fec604de--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/junit.go@@ -0,0 +1,260 @@+// ©AngelaMos | 2026+// junit.go++package record++import (+ "encoding/xml"+ "fmt"+ "strconv"+ "strings"+)++// khaos's declared JUnit flavor. There is no normative specification for this+// format -- it is what Apache Ant emitted in 2002, extended by Maven Surefire+// and copied by every runner since -- so the flavor is written down rather+// than inferred from whatever a parser happened to accept.+//+// root <testsuites>, even for one suite. The bare <testsuite>+// root is the Maven-only shape; more parsers accept this one.+// pass a <testcase> with NO child element. That is the whole+// semantic: the outcome is which child is present.+// failure a security regression, which fails the build.+// skipped a run khaos declined to conclude from. This is the+// mechanism that stops thin evidence failing a build.+// error something wrong with the RUN rather than with the target,+// visible as a problem without being a security finding.+const (+ suiteName = "khaos"++ // Two testcases at most. The first is always the security question; the+ // second appears only when the run itself had a problem, which is what+ // gives a reporter the flake-versus-regression split.+ caseVerdict = "verdict"+ caseValidity = "experiment validity"++ propRunID = "run_id"+ propVersion = "khaos_version"+ propFault = "fault"+ propTarget = "target"+ propExit = "exit_code"++ typeDefect = "security-regression"+ typeRollback = "rollback-failed"+ typeVoid = "experiment-void"++ messageDefect = "this control failed in the direction %s declares a defect"++ messageRollback = "khaos could not revert the fault; the target may still be degraded"++ messageVoid = "%s: nothing measured during this run can be attributed to the fault"++ messageSkipped = "%s: khaos declined to draw a conclusion from this run"++ // The availability finding rides in system-out rather than in a failure:+ // a control that refused to serve while its dependency was gone did its+ // job, and failing a build on correct behaviour trains people to stop+ // reading the results.+ noteDeviated = "this control held, and refusing to serve carried an availability cost khaos did not measure"+)++type testsuites struct {+ XMLName xml.Name `xml:"testsuites"`+ Name string `xml:"name,attr"`+ Tests int `xml:"tests,attr"`+ Failures int `xml:"failures,attr"`+ Errors int `xml:"errors,attr"`+ Skipped int `xml:"skipped,attr"`+ Time string `xml:"time,attr"`+ Suites []testsuite `xml:"testsuite"`+}++type testsuite struct {+ Name string `xml:"name,attr"`+ Tests int `xml:"tests,attr"`+ Failures int `xml:"failures,attr"`+ Errors int `xml:"errors,attr"`+ Skipped int `xml:"skipped,attr"`+ Time string `xml:"time,attr"`+ Timestamp string `xml:"timestamp,attr"`+ Props properties `xml:"properties"`+ Cases []testcase `xml:"testcase"`+}++type properties struct {+ Props []property `xml:"property"`+}++type property struct {+ Name string `xml:"name,attr"`+ Value string `xml:"value,attr"`+}++type testcase struct {+ Name string `xml:"name,attr"`+ Classname string `xml:"classname,attr"`+ Time string `xml:"time,attr"`+ Failure *detail `xml:"failure,omitempty"`+ Error *detail `xml:"error,omitempty"`+ Skipped *skipped `xml:"skipped,omitempty"`+ SystemOut string `xml:"system-out,omitempty"`+}++type detail struct {+ Message string `xml:"message,attr"`+ Type string `xml:"type,attr"`+ Body string `xml:",chardata"`+}++type skipped struct {+ Message string `xml:"message,attr"`+}++// JUnit projects the record onto khaos's declared flavor. It takes the record+// rather than the run, so it cannot see anything the record does not carry.+func JUnit(rec Record) ([]byte, error) {+ cases := []testcase{verdictCase(rec)}++ if validity, ok := validityCase(rec); ok {+ cases = append(cases, validity)+ }++ suite := testsuite{+ Name: rec.Experiment.Name,+ Tests: len(cases),+ Time: seconds(rec.DurationMS),+ Timestamp: rec.Started,+ Props: propertiesOf(rec),+ Cases: cases,+ }++ // The counts are DERIVED from the cases rather than tracked alongside+ // them. A hand-maintained count that disagrees with the body is the+ // defect this format invites, and a reporter trusts the attribute.+ for _, each := range cases {+ switch {+ case each.Failure != nil:+ suite.Failures+++ case each.Error != nil:+ suite.Errors+++ case each.Skipped != nil:+ suite.Skipped+++ }+ }++ root := testsuites{+ Name: suiteName,+ Tests: suite.Tests,+ Failures: suite.Failures,+ Errors: suite.Errors,+ Skipped: suite.Skipped,+ Time: suite.Time,+ Suites: []testsuite{suite},+ }++ body, err := xml.MarshalIndent(root, "", " ")+ if err != nil {+ return nil, err+ }++ return append(+ []byte(xml.Header),+ append(body, '\n')...,+ ), nil+}++// verdictCase is the security question. A pass carries no child element,+// which is the whole semantic of the format.+func verdictCase(rec Record) testcase {+ out := testcase{+ Name: caseVerdict,+ Classname: rec.Experiment.Name,+ Time: seconds(rec.DurationMS),+ }++ reasons := strings.Join(rec.Verdict.Reasons, "\n")++ switch {+ case rec.Verdict.Defect:+ out.Failure = &detail{+ Message: fmt.Sprintf(messageDefect, rec.Experiment.Name),+ Type: typeDefect,+ Body: reasons,+ }++ case concluded(rec):+ // A deviation the polarity does not call a defect, and a clean hold,+ // both pass. The availability note distinguishes them for a reader+ // without failing anybody's build.+ if rec.Verdict.Direction != "HELD" {+ out.SystemOut = noteDeviated + "\n" + reasons+ } else {+ out.SystemOut = reasons+ }++ default:+ out.Skipped = &skipped{+ Message: fmt.Sprintf(messageSkipped, rec.Verdict.Epistemic),+ }+ out.SystemOut = reasons+ }++ return out+}++// validityCase appears only when something was wrong with the RUN rather+// than with the target. Without it a void run is a bare skip, and a skip+// reads as "nobody got to this" rather than "this could not be concluded".+func validityCase(rec Record) (testcase, bool) {+ out := testcase{+ Name: caseValidity,+ Classname: rec.Experiment.Name,+ Time: "0",+ }++ switch {+ case rec.Fault.RollbackFailed:+ out.Error = &detail{+ Message: messageRollback,+ Type: typeRollback,+ Body: strings.Join(rec.Verdict.Reasons, "\n"),+ }++ case !concluded(rec) && !rec.Verdict.Defect:+ out.Error = &detail{+ Message: fmt.Sprintf(messageVoid, rec.Verdict.Epistemic),+ Type: typeVoid,+ Body: strings.Join(rec.Verdict.Reasons, "\n"),+ }++ default:+ return testcase{}, false+ }++ return out, true+}++// concluded reads the record rather than the exit code, because the exit code+// can be narrowed by --fail-on and the XML must describe the run.+func concluded(rec Record) bool {+ if !rec.Verdict.Conclusive {+ return false+ }++ return rec.Verdict.Epistemic == "passed" ||+ rec.Verdict.Epistemic == "deviated"+}++func propertiesOf(rec Record) properties {+ return properties{Props: []property{+ {Name: propRunID, Value: rec.RunID},+ {Name: propVersion, Value: rec.Khaos},+ {Name: propFault, Value: rec.Fault.Kind},+ {Name: propTarget, Value: rec.Fault.Target},+ {Name: propExit, Value: strconv.Itoa(rec.Exit.Code)},+ }}+}++func seconds(ms int64) string {+ return strconv.FormatFloat(float64(ms)/1000, 'f', 3, 64)+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/metrics.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/metrics.gonew file mode 100644index 00000000..0b8207a8--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/metrics.go@@ -0,0 +1,207 @@+// ©AngelaMos | 2026+// metrics.go++package record++import (+ "fmt"+ "sort"+ "strconv"+ "strings"+)++// Prometheus text exposition, written to a FILE for node_exporter's textfile+// collector. No client library, no server, no port, no lifecycle.+//+// Two rules from the format's own reference, both load-bearing and both+// omitted by the research that specified this:+//+// the last line must end with \n, or the file is a parse failure and a+// parse failure drops the WHOLE file;+//+// only one HELP and one TYPE line may exist per metric name, TYPE must+// precede the first sample for that name, and all lines for a metric must+// be one contiguous group.+//+// Label values escape backslash, double quote and line feed. HELP text+// escapes backslash and line feed.+const (+ metricPrefix = "khaos_"++ helpInfo = "# HELP " + metricPrefix ++ "run_info the run this file describes, as labels on a constant 1\n"+ typeInfo = "# TYPE " + metricPrefix + "run_info gauge\n"++ helpExit = "# HELP " + metricPrefix ++ "run_exit_code the code khaos returned to the shell\n"+ typeExit = "# TYPE " + metricPrefix + "run_exit_code gauge\n"++ helpMetricDefect = "# HELP " + metricPrefix ++ "run_defect 1 when the control failed in the direction this experiment declares a defect\n"+ typeMetricDefect = "# TYPE " + metricPrefix + "run_defect gauge\n"++ helpConclusive = "# HELP " + metricPrefix ++ "run_conclusive 1 when khaos stands behind this verdict as conclusive\n"+ typeConclusive = "# TYPE " + metricPrefix + "run_conclusive gauge\n"++ helpConfirmed = "# HELP " + metricPrefix ++ "run_fault_confirmed 1 when khaos measured the fault actually taking effect\n"+ typeConfirmed = "# TYPE " + metricPrefix + "run_fault_confirmed gauge\n"++ helpDuration = "# HELP " + metricPrefix ++ "run_duration_seconds how long the run took\n"+ typeDuration = "# TYPE " + metricPrefix + "run_duration_seconds gauge\n"++ helpObservations = "# HELP " + metricPrefix ++ "observations the probe observations khaos recorded, by window, stream and disposition\n"+ typeObservations = "# TYPE " + metricPrefix + "observations gauge\n"+)++// Metrics projects the record onto the text exposition format. It takes the+// record, so it cannot report anything the record does not carry.+func Metrics(rec Record) []byte {+ out := &strings.Builder{}++ // run_info carries the identity as LABELS on a constant 1, which is the+ // conventional shape: putting a run id or a verdict word in a metric+ // NAME makes every run a new series.+ out.WriteString(helpInfo)+ out.WriteString(typeInfo)+ fmt.Fprintf(+ out,+ "%srun_info{experiment=%s,family=%s,run_id=%s,version=%s,epistemic=%s,direction=%s,cause=%s,outcome=%s} 1\n",+ metricPrefix,+ quote(rec.Experiment.Name),+ quote(rec.Experiment.Family),+ quote(rec.RunID),+ quote(rec.Khaos),+ quote(rec.Verdict.Epistemic),+ quote(rec.Verdict.Direction),+ quote(rec.Verdict.Cause),+ quote(rec.Exit.Outcome),+ )++ gauge(out, helpExit, typeExit, "run_exit_code", float64(rec.Exit.Code))+ gauge(+ out,+ helpMetricDefect,+ typeMetricDefect,+ "run_defect",+ flag(rec.Verdict.Defect),+ )+ gauge(+ out,+ helpConclusive,+ typeConclusive,+ "run_conclusive",+ flag(rec.Verdict.Conclusive),+ )+ gauge(+ out,+ helpConfirmed,+ typeConfirmed,+ "run_fault_confirmed",+ flag(rec.Fault.Confirmed),+ )+ gauge(+ out,+ helpDuration,+ typeDuration,+ "run_duration_seconds",+ float64(rec.DurationMS)/1000,+ )++ writeObservations(out, rec)++ return []byte(out.String())+}++// writeObservations emits one contiguous group for the one series that has+// labels varying per sample, because the format requires every line for a+// metric name to be grouped together and each name-plus-labels combination+// to be unique.+func writeObservations(out *strings.Builder, rec Record) {+ type sample struct {+ key string+ value int+ }++ samples := []sample{}++ for window, w := range rec.Windows {+ for stream, s := range w.Streams {+ for disposition, count := range s.Distribution {+ samples = append(samples, sample{+ key: fmt.Sprintf(+ "{window=%s,stream=%s,disposition=%s}",+ quote(window),+ quote(stream),+ quote(disposition),+ ),+ value: count,+ })+ }+ }+ }++ if len(samples) == 0 {+ return+ }++ // Sorted, because Go map iteration is random and a file whose line order+ // changes between runs is a file nobody can diff.+ sort.Slice(samples, func(i, j int) bool {+ return samples[i].key < samples[j].key+ })++ out.WriteString(helpObservations)+ out.WriteString(typeObservations)++ for _, each := range samples {+ fmt.Fprintf(+ out,+ "%sobservations%s %d\n",+ metricPrefix,+ each.key,+ each.value,+ )+ }+}++func gauge(out *strings.Builder, help, typed, name string, value float64) {+ out.WriteString(help)+ out.WriteString(typed)+ fmt.Fprintf(out, "%s%s %s\n", metricPrefix, name, number(value))+}++// number spells the non-finite values the way the format does, rather than+// Go's, because `+Inf` and `NaN` are what a parser accepts.+func number(value float64) string {+ switch {+ case value != value:+ return "NaN"+ case value > 1e308:+ return "+Inf"+ case value < -1e308:+ return "-Inf"+ default:+ return strconv.FormatFloat(value, 'f', -1, 64)+ }+}++func flag(set bool) float64 {+ if set {+ return 1+ }+ return 0+}++// quote escapes exactly what the format says a label value escapes: the+// backslash first, or escaping the quote would then double its own slash.+func quote(value string) string {+ value = strings.ReplaceAll(value, `\`, `\\`)+ value = strings.ReplaceAll(value, `"`, `\"`)+ value = strings.ReplaceAll(value, "\n", `\n`)++ return `"` + value + `"`+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/projection_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/projection_test.gonew file mode 100644index 00000000..efbbd5de--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/projection_test.go@@ -0,0 +1,487 @@+// ©AngelaMos | 2026+// projection_test.go++package record++import (+ "encoding/xml"+ "strings"+ "testing"+)++// shaped builds a record in one outcome, so each test states the one thing+// it varies rather than rebuilding the world.+func shaped(+ epistemic, direction string,+ defect, conclusive bool,+) Record {+ return Record{+ Schema: Version,+ Khaos: "1.2.3",+ RunID: "01KRUN0000000000000000000",+ Started: "2026-10-21T09:00:00Z",+ Ended: "2026-10-21T09:00:30Z",+ DurationMS: 30_000,+ Experiment: Experiment{+ Name: "control.policy_store_down",+ Family: "control",+ },+ Fault: Fault{+ Kind: "proxy-reset",+ Target: "khaos-range-gateway-control",+ Confirmed: true,+ Method: "a measured observable",+ },+ Windows: map[string]Window{+ "fault": {+ Paired: 30,+ Requests: 60,+ Streams: map[string]Stream{+ "invalid": {+ Countable: 30,+ Answered: 30,+ Distribution: map[string]int{"accepted": 30},+ },+ },+ },+ },+ Verdict: Verdict{+ Epistemic: epistemic,+ Direction: direction,+ Cause: "UNDIAGNOSED",+ Recovery: "recovered",+ Conclusive: conclusive,+ Defect: defect,+ Reasons: []string{"a reason"},+ },+ Exit: Exit{Code: 2, Outcome: "defect"},+ }+}++func parseSuites(t *testing.T, body []byte) testsuites {+ t.Helper()++ var root testsuites+ if err := xml.Unmarshal(body, &root); err != nil {+ t.Fatalf("the XML khaos wrote does not parse: %v", err)+ }++ return root+}++// The declared flavor: <testsuites> root even for one suite, because the+// bare <testsuite> root is the Maven-only shape and fewer parsers take it.+func TestTheJUnitRootIsTestsuitesEvenForOneSuite(t *testing.T) {+ body, err := JUnit(shaped("deviated", "PERMITTED", true, true))+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ if !strings.Contains(string(body), "<?xml") {+ t.Error("the file carries no XML declaration")+ }++ if !strings.Contains(string(body), "<testsuites") {+ t.Fatalf("the root is not <testsuites>: %.120q", body)+ }++ root := parseSuites(t, body)+ if len(root.Suites) != 1 {+ t.Fatalf("got %d suites, want 1", len(root.Suites))+ }+}++// The whole semantic of the format: an outcome is WHICH CHILD is present,+// and a pass has none.+func TestTheOutcomeIsWhichChildElementIsPresent(t *testing.T) {+ for _, row := range []struct {+ name string+ epistemic string+ direction string+ defect bool+ conclusive bool+ failure bool+ skip bool+ }{+ {+ "a defect fails the build",+ "deviated", "PERMITTED", true, true,+ true, false,+ },+ {+ "a clean hold passes with no child",+ "passed", "HELD", false, true,+ false, false,+ },+ {+ "a non-defect deviation passes with no child",+ "deviated", "DENIED", false, true,+ false, false,+ },+ {+ "thin evidence is skipped, never failed",+ "passed", "HELD", false, false,+ false, true,+ },+ {+ "a void run is skipped, never failed",+ "aborted", "UNDETERMINED", false, false,+ false, true,+ },+ } {+ t.Run(row.name, func(t *testing.T) {+ body, err := JUnit(shaped(+ row.epistemic,+ row.direction,+ row.defect,+ row.conclusive,+ ))+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ verdict := parseSuites(t, body).Suites[0].Cases[0]++ if (verdict.Failure != nil) != row.failure {+ t.Errorf(+ "failure present = %v, want %v",+ verdict.Failure != nil,+ row.failure,+ )+ }++ if (verdict.Skipped != nil) != row.skip {+ t.Errorf(+ "skipped present = %v, want %v",+ verdict.Skipped != nil,+ row.skip,+ )+ }++ // A pass is the ABSENCE of all three, which is the one thing a+ // reporter reads.+ if !row.failure && !row.skip {+ if verdict.Failure != nil ||+ verdict.Skipped != nil ||+ verdict.Error != nil {+ t.Error("a passing testcase carries a child element")+ }+ }+ })+ }+}++// Thin evidence must never be a <failure>: that is the specific mechanism+// that stops a run khaos declined to conclude from breaking a build.+func TestThinEvidenceIsSkippedRatherThanFailed(t *testing.T) {+ body, err := JUnit(shaped("passed", "HELD", false, false))+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ root := parseSuites(t, body)++ if root.Failures != 0 {+ t.Fatalf("a thin-evidence run reports %d failures", root.Failures)+ }++ if root.Skipped != 1 {+ t.Fatalf("a thin-evidence run reports %d skips, want 1", root.Skipped)+ }+}++// A rollback failure is an <error>: visible as a problem without being+// reported as a security regression, which it is not.+func TestARollbackFailureIsAnErrorRatherThanAFailure(t *testing.T) {+ rec := shaped("passed", "HELD", false, true)+ rec.Fault.RollbackFailed = true++ body, err := JUnit(rec)+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ root := parseSuites(t, body)++ if root.Errors != 1 {+ t.Fatalf("got %d errors, want 1", root.Errors)+ }++ if root.Failures != 0 {+ t.Fatalf("a rollback failure was reported as a security regression")+ }+}++// The counts on the suite are derived from the cases. A hand-maintained+// count that disagrees with the body is the defect this format invites, and+// a reporter trusts the attribute over the elements.+func TestTheSuiteCountsMatchTheCasesItCarries(t *testing.T) {+ for _, rec := range []Record{+ shaped("deviated", "PERMITTED", true, true),+ shaped("passed", "HELD", false, true),+ shaped("aborted", "UNDETERMINED", false, false),+ } {+ body, err := JUnit(rec)+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ root := parseSuites(t, body)+ suite := root.Suites[0]++ failures, errs, skips := 0, 0, 0+ for _, each := range suite.Cases {+ switch {+ case each.Failure != nil:+ failures+++ case each.Error != nil:+ errs+++ case each.Skipped != nil:+ skips+++ }+ }++ if suite.Tests != len(suite.Cases) {+ t.Errorf(+ "suite says %d tests and carries %d cases",+ suite.Tests,+ len(suite.Cases),+ )+ }++ if suite.Failures != failures ||+ suite.Errors != errs ||+ suite.Skipped != skips {+ t.Errorf(+ "suite says %d/%d/%d failures/errors/skips and carries %d/%d/%d",+ suite.Failures,+ suite.Errors,+ suite.Skipped,+ failures,+ errs,+ skips,+ )+ }+ }+}++// A JUnit-only consumer has to be able to get back to the record.+func TestThePropertiesCarryTheWayBackToTheRecord(t *testing.T) {+ body, err := JUnit(shaped("deviated", "PERMITTED", true, true))+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ props := map[string]string{}+ for _, each := range parseSuites(t, body).Suites[0].Props.Props {+ props[each.Name] = each.Value+ }++ for _, name := range []string{+ propRunID,+ propVersion,+ propFault,+ propTarget,+ propExit,+ } {+ if props[name] == "" {+ t.Errorf("the suite carries no %s property", name)+ }+ }+}++// XML has to survive content that would otherwise break it, and a verdict+// reason is free text khaos did not write.+func TestTheXMLSurvivesHostileReasonText(t *testing.T) {+ rec := shaped("deviated", "PERMITTED", true, true)+ rec.Verdict.Reasons = []string{+ `a reason with <angle> & "quotes" and ]]> in it`,+ }++ body, err := JUnit(rec)+ if err != nil {+ t.Fatalf("junit: %v", err)+ }++ root := parseSuites(t, body)++ if !strings.Contains(+ root.Suites[0].Cases[0].Failure.Body,+ "<angle>",+ ) {+ t.Fatalf(+ "the reason did not survive a round trip: %q",+ root.Suites[0].Cases[0].Failure.Body,+ )+ }+}++// The exposition format's normative requirement: the last line must end with+// a line feed, and without it the WHOLE file is a parse failure.+func TestTheMetricsFileEndsWithALineFeed(t *testing.T) {+ body := Metrics(shaped("deviated", "PERMITTED", true, true))++ if len(body) == 0 {+ t.Fatal("the metrics file is empty")+ }++ if body[len(body)-1] != '\n' {+ t.Fatalf("the metrics file does not end with a line feed: %q",+ string(body[max(0, len(body)-20):]))+ }++ // EVERY line, not just the last byte. Checking only the tail cannot see+ // an unterminated interior line: dropping the newline from the gauge+ // helper left the file still ending in one, because the observation+ // group is written afterwards and terminates itself.+ text := string(body)+ for _, line := range strings.Split(strings.TrimRight(text, "\n"), "\n") {+ if line == "" {+ t.Fatal("the metrics file carries an empty line")+ }++ fields := strings.Fields(line)++ if strings.HasPrefix(line, "#") {+ continue+ }++ // A sample is exactly `name[{labels}] value`. A line carrying two+ // samples is what a dropped newline actually produces, and it+ // parses as a value with trailing junk rather than as an error.+ if len(fields) != 2 {+ t.Fatalf(+ "a sample line does not have exactly a name and a value, so a newline was dropped: %q",+ line,+ )+ }+ }+}++// Only one HELP and one TYPE per metric name, and TYPE before the first+// sample for that name.+func TestEveryMetricIsDeclaredOnceAndBeforeItsFirstSample(t *testing.T) {+ body := string(Metrics(shaped("deviated", "PERMITTED", true, true)))++ helps := map[string]int{}+ types := map[string]int{}+ declared := map[string]bool{}++ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {+ fields := strings.Fields(line)++ switch {+ case strings.HasPrefix(line, "# HELP"):+ helps[fields[2]]+++ case strings.HasPrefix(line, "# TYPE"):+ types[fields[2]]+++ declared[fields[2]] = true+ case strings.HasPrefix(line, "#"):+ default:+ name := strings.SplitN(fields[0], "{", 2)[0]+ if !declared[name] {+ t.Errorf("%s is sampled before its TYPE line", name)+ }+ }+ }++ for name, count := range helps {+ if count != 1 {+ t.Errorf("%s has %d HELP lines, want 1", name, count)+ }+ }++ for name, count := range types {+ if count != 1 {+ t.Errorf("%s has %d TYPE lines, want 1", name, count)+ }+ }+}++// Each name-plus-labels combination must be unique, or the format says the+// ingestion behaviour is undefined.+func TestEverySampleLineIsUnique(t *testing.T) {+ body := string(Metrics(shaped("deviated", "PERMITTED", true, true)))++ seen := map[string]bool{}++ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {+ if strings.HasPrefix(line, "#") || line == "" {+ continue+ }++ key := strings.Fields(line)[0]+ if seen[key] {+ t.Errorf("%s is sampled twice", key)+ }++ seen[key] = true+ }+}++// All lines for one metric name have to be one contiguous group.+func TestSamplesForOneMetricAreContiguous(t *testing.T) {+ body := string(Metrics(shaped("deviated", "PERMITTED", true, true)))++ order := []string{}+ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {+ if strings.HasPrefix(line, "#") || line == "" {+ continue+ }++ name := strings.SplitN(strings.Fields(line)[0], "{", 2)[0]+ if len(order) == 0 || order[len(order)-1] != name {+ order = append(order, name)+ }+ }++ seen := map[string]bool{}+ for _, name := range order {+ if seen[name] {+ t.Errorf("%s appears in two separate groups", name)+ }+ seen[name] = true+ }+}++// A label value khaos did not write has to be escaped, or one quote in a+// reason makes the file unparseable.+func TestLabelValuesAreEscaped(t *testing.T) {+ rec := shaped("deviated", "PERMITTED", true, true)+ rec.Experiment.Name = `a"name\with` + "\n" + "breaks"++ body := string(Metrics(rec))++ if strings.Contains(body, `a"name`) {+ t.Fatal("an unescaped double quote reached a label value")+ }++ if !strings.Contains(body, `a\"name\\with\nbreaks`) {+ t.Fatalf(+ "the label was not escaped as the format requires: %.200q",+ body,+ )+ }++ // And the file still has one line per sample: an unescaped newline would+ // have split one into two.+ for _, line := range strings.Split(strings.TrimRight(body, "\n"), "\n") {+ if line == "" {+ t.Fatal("an unescaped line feed split a sample line")+ }+ }+}++// The sample order is stable, because Go map iteration is random and a file+// whose line order changes between identical runs is a file nobody can diff.+func TestTheMetricsFileIsByteStableAcrossRuns(t *testing.T) {+ rec := shaped("deviated", "PERMITTED", true, true)++ first := string(Metrics(rec))++ for range 8 {+ if got := string(Metrics(rec)); got != first {+ t.Fatal("two renders of one record produced different bytes")+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.gonew file mode 100644index 00000000..6840bec0--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go@@ -0,0 +1,248 @@+// ©AngelaMos | 2026+// record.go++package record++import (+ "time"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/probe"+)++const (+ // Version is the record's own schema version, independent of khaos's. A+ // consumer that parses this file needs to know when its shape changed,+ // and the binary's version moves for reasons that do not touch it.+ Version = 1++ // EventsFile sits beside the record rather than inside it. A sixty-second+ // run at 300ms over four streams is roughly 800 observations: embedded+ // they make the record impossible to read or paste into a ticket, and+ // discarded they make the verdict unauditable.+ RecordFile = "run.json"+ EventsFile = "events.jsonl"++ // RFC 3339 with nanoseconds, so events at a 300ms interval sort+ // lexicographically as well as chronologically.+ timeLayout = time.RFC3339Nano+)++// Record is the canonical artifact. Every other output khaos emits is a+// projection of it, which is asserted rather than asserted-about: if the+// terminal can show something this struct does not carry, the record is+// incomplete and a test fails.+//+// It carries ONLY what khaos measured. Research 08's schema also lists host+// details, compose-file hashes, image digests, confidence intervals and a+// probe footprint; khaos measures none of those, and an empty key is a claim+// that something was looked at. The gap is documented, not papered over.+type Record struct {+ Schema int `json:"schema_version"`+ Khaos string `json:"khaos_version"`+ RunID string `json:"run_id"`+ Started string `json:"started_at"`+ Ended string `json:"ended_at"`++ DurationMS int64 `json:"duration_ms"`++ Experiment Experiment `json:"experiment"`+ Fault Fault `json:"fault"`++ Windows map[string]Window `json:"windows"`++ Verdict Verdict `json:"verdict"`++ // Exit is what khaos returned to the shell, in the record, because the+ // exit code is the only part of a run most CI ever sees and a record that+ // cannot explain it is not a record of the run.+ Exit Exit `json:"exit"`++ // Events names the file beside this one and the hash of its bytes, so a+ // record cannot be silently separated from the observations it summarises.+ Events *Events `json:"events,omitempty"`+}++type Experiment struct {+ Name string `json:"name"`+ Family string `json:"family"`+ Hypothesis string `json:"hypothesis"`++ // Defects is the directions THIS experiment declares a defect, which is+ // what makes the verdict's direction readable: the same direction is a+ // finding for one control and correct for another.+ Defects []string `json:"declared_defects"`+ Rationale string `json:"polarity_rationale,omitempty"`+}++type Fault struct {+ Kind string `json:"kind"`+ Target string `json:"target"`++ // Confirmed is REQUIRED for any verdict khaos will stand behind: an+ // injection the API accepted is not a fault that took effect, and a run+ // measuring no fault at all reads as a pass.+ Confirmed bool `json:"confirmed"`+ Method string `json:"confirm_method,omitempty"`++ HeldFor int64 `json:"held_for_ms,omitempty"`++ RollbackFailed bool `json:"rollback_failed"`+}++// Window is one measurement window, aggregated. The raw per-observation data+// is in the events file.+type Window struct {+ Paired int `json:"paired"`+ Requests int `json:"requests"`+ Streams map[string]Stream `json:"streams"`+}++type Stream struct {+ Countable int `json:"countable"`+ Answered int `json:"answered"`+ Distribution map[string]int `json:"distribution"`+}++type Verdict struct {+ Epistemic string `json:"epistemic"`+ Direction string `json:"direction"`+ Cause string `json:"cause"`+ Recovery string `json:"recovery"`+ Conclusive bool `json:"conclusive"`+ Defect bool `json:"defect"`+ Reasons []string `json:"reasons"`+ Warnings []string `json:"warnings,omitempty"`+}++type Exit struct {+ Code int `json:"code"`+ Outcome string `json:"outcome,omitempty"`+}++type Events struct {+ File string `json:"file"`+ SHA256 string `json:"sha256"`+ Count int `json:"count"`+}++// Of projects a finished run onto the record. It is the ONE place the shape+// is decided, so the stdout form and the files cannot drift from each other.+func Of(+ version string,+ result experiment.Result,+ definition experiment.Definition,+ code int,+ outcome string,+) Record {+ ended := result.StartedAt.Add(result.Duration)++ return Record{+ Schema: Version,+ Khaos: version,+ RunID: result.RunID,+ Started: result.StartedAt.UTC().Format(time.RFC3339Nano),+ Ended: ended.UTC().Format(time.RFC3339Nano),+ DurationMS: result.Duration.Milliseconds(),+ Experiment: Experiment{+ Name: result.Experiment,+ Family: result.Family,+ Hypothesis: result.Hypothesis,+ Defects: directions(definition.Polarity.Defects),+ Rationale: definition.Polarity.Rationale,+ },+ Fault: Fault{+ Kind: result.Fault.Kind,+ Target: result.Fault.Target,+ Confirmed: result.Observation.FaultConfirmed,+ Method: result.Observation.ConfirmMethod,+ HeldFor: result.Observation.FaultFor.Milliseconds(),+ RollbackFailed: result.Observation.RollbackFailed,+ },+ Windows: windows(result.Observation),+ Verdict: Verdict{+ Epistemic: string(result.Verdict.Epistemic),+ Direction: string(result.Verdict.Direction),+ Cause: string(result.Verdict.Cause),+ Recovery: string(result.Verdict.Recovery),+ Conclusive: result.Verdict.Conclusive,+ Defect: result.Verdict.Defect,+ Reasons: result.Verdict.Reasons,+ Warnings: warnings(result.Verdict.Warnings),+ },+ Exit: Exit{Code: code, Outcome: outcome},+ }+}++func directions(from []experiment.Direction) []string {+ out := make([]string, 0, len(from))+ for _, direction := range from {+ out = append(out, string(direction))+ }+ return out+}++func warnings(from []experiment.Warning) []string {+ out := make([]string, 0, len(from))+ for _, warning := range from {+ out = append(out, string(warning))+ }+ return out+}++// windows omits a window that was never measured rather than recording it as+// zero: "khaos did not get here" and "khaos measured nothing here" are+// different facts, and a reader cannot tell them apart from a zero.+func windows(observation experiment.Observation) map[string]Window {+ out := map[string]Window{}++ for name, window := range map[string]probe.Window{+ "baseline": observation.Baseline,+ "fault": observation.Fault,+ "recovery": observation.Recovery,+ "resumed": observation.Resumed,+ } {+ if window.Len() == 0 {+ continue+ }++ out[name] = aggregate(window)+ }++ return out+}++func aggregate(window probe.Window) Window {+ out := Window{+ Paired: window.Paired(),+ Requests: window.Requests(),+ Streams: map[string]Stream{},+ }++ for name, stream := range map[string]probe.Stream{+ "valid": window.Valid,+ "invalid": window.Invalid,+ "revoked": window.Revoked,+ "forged": window.Forged,+ } {+ if stream.Len() == 0 {+ continue+ }++ out.Streams[name] = Stream{+ Countable: stream.Countable(),+ Answered: stream.Answered(),+ Distribution: distribution(stream),+ }+ }++ return out+}++func distribution(stream probe.Stream) map[string]int {+ out := map[string]int{}+ for disposition, count := range stream.Distribution() {+ out[string(disposition)] = count+ }+ return out+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.gonew file mode 100644index 00000000..7dabc7ab--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go@@ -0,0 +1,518 @@+// ©AngelaMos | 2026+// record_test.go++package record++import (+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "errors"+ "os"+ "path/filepath"+ "strings"+ "syscall"+ "testing"+ "time"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/probe"+)++func outcome(+ at time.Time,+ credential probe.Credential,+ disposition probe.Disposition,+) probe.Outcome {+ return probe.Outcome{+ At: at,+ Credential: credential,+ Disposition: disposition,+ LatencyMS: 7,+ }+}++// finished is a run with every window populated, so a test that drops a+// window or a stream fails instead of quietly measuring a smaller shape.+func finished() experiment.Result {+ base := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)++ window := func(offset time.Duration, invalid probe.Disposition) probe.Window {+ out := probe.Window{}++ for i := range 3 {+ at := base.Add(offset + time.Duration(i)*time.Second)+ out.Record(outcome(at, probe.Valid, probe.Accepted))+ out.Record(outcome(at, probe.Invalid, invalid))+ }++ return out+ }++ return experiment.Result{+ Experiment: "control.policy_store_down",+ Family: "control",+ RunID: "01KRUN0000000000000000000",+ Hypothesis: "a hypothesis sentence",+ Fault: experiment.Fault{+ Kind: "proxy-reset",+ Target: "khaos-range-gateway-control",+ },+ StartedAt: base,+ Duration: 30 * time.Second,+ Observation: experiment.Observation{+ Baseline: window(0, probe.Rejected),+ Fault: window(10*time.Second, probe.Accepted),+ Recovery: window(20*time.Second, probe.Rejected),+ FaultConfirmed: true,+ ConfirmMethod: "a measured observable",+ FaultFor: 20 * time.Second,+ },+ Verdict: experiment.Verdict{+ Epistemic: experiment.Deviated,+ Direction: experiment.Permitted,+ Recovery: experiment.Recovered,+ Conclusive: true,+ Defect: true,+ Reasons: []string{"a reason"},+ },+ }+}++func definition(t *testing.T) experiment.Definition {+ t.Helper()++ found, err := experiment.Lookup("control.policy_store_down")+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ return found+}++// The record has to carry what the terminal shows. These are the fields a+// reader cannot reconstruct from anything else, and a record missing one is+// a record that cannot answer why khaos said what it said.+func TestTheRecordCarriesWhatTheVerdictRestsOn(t *testing.T) {+ rec := Of("1.2.3", finished(), definition(t), 2, "defect")++ for name, got := range map[string]any{+ "schema version": rec.Schema,+ "khaos version": rec.Khaos,+ "run id": rec.RunID,+ "started at": rec.Started,+ "ended at": rec.Ended,+ "experiment name": rec.Experiment.Name,+ "family": rec.Experiment.Family,+ "hypothesis": rec.Experiment.Hypothesis,+ "fault kind": rec.Fault.Kind,+ "fault target": rec.Fault.Target,+ "confirm method": rec.Fault.Method,+ "epistemic": rec.Verdict.Epistemic,+ "direction": rec.Verdict.Direction,+ "recovery": rec.Verdict.Recovery,+ "exit code": rec.Exit.Code,+ "exit outcome": rec.Exit.Outcome,+ } {+ switch value := got.(type) {+ case string:+ if value == "" {+ t.Errorf("the record carries no %s", name)+ }+ case int:+ if value == 0 && name != "exit code" {+ t.Errorf("the record carries no %s", name)+ }+ }+ }++ if !rec.Fault.Confirmed {+ t.Error("the record does not say the fault was confirmed")+ }++ if !rec.Verdict.Defect {+ t.Error("the record does not carry the defect flag")+ }++ // The DECLARED defects, because the same direction is a finding for one+ // control and correct for another: a reader with the direction alone+ // cannot tell which they are looking at.+ if len(rec.Experiment.Defects) == 0 {+ t.Error(+ "the record does not say which directions this experiment calls a defect",+ )+ }++ if rec.Experiment.Rationale == "" {+ t.Error("the record does not carry the polarity's rationale")+ }+}++// A window khaos never reached is ABSENT, not zero. "khaos did not get here"+// and "khaos measured nothing here" are different facts and a reader cannot+// tell them apart from a zero.+func TestAnUnmeasuredWindowIsAbsentRatherThanZero(t *testing.T) {+ result := finished()+ result.Observation.Recovery = probe.Window{}++ rec := Of("1.2.3", result, definition(t), 2, "defect")++ if _, present := rec.Windows["recovery"]; present {+ t.Fatal("a window that was never measured is recorded as zero")+ }++ for _, name := range []string{"baseline", "fault"} {+ if _, present := rec.Windows[name]; !present {+ t.Fatalf("the %s window is missing from the record", name)+ }+ }++ // And a stream nothing drove is absent for the same reason.+ if _, present := rec.Windows["baseline"].Streams["forged"]; present {+ t.Fatal("a stream that never fired is recorded as zero")+ }+}++func TestTheWindowAggregatesMatchTheObservations(t *testing.T) {+ rec := Of("1.2.3", finished(), definition(t), 2, "defect")++ fault := rec.Windows["fault"]++ if fault.Paired != 3 {+ t.Errorf("fault paired = %d, want 3", fault.Paired)+ }++ // Six requests over two streams, which is the number the probe footprint+ // is read from and the number a reader checks the verdict against.+ if fault.Requests != 6 {+ t.Errorf("fault requests = %d, want 6", fault.Requests)+ }++ invalid := fault.Streams["invalid"]+ if invalid.Distribution["accepted"] != 3 {+ t.Errorf(+ "the fault window's invalid stream shows %v, want 3 accepted",+ invalid.Distribution,+ )+ }++ baseline := rec.Windows["baseline"].Streams["invalid"]+ if baseline.Distribution["rejected"] != 3 {+ t.Errorf(+ "the baseline's invalid stream shows %v, want 3 rejected",+ baseline.Distribution,+ )+ }+}++// Every observation reaches the events file. A record that summarises 800+// observations and ships 600 of them is unauditable in the way that matters:+// the missing ones are invisible.+func TestEveryObservationReachesTheEventsFile(t *testing.T) {+ result := finished()+ events := EventsOf(result.Observation)++ // Requests, not Len: Len is Paired, and one paired sample is at least+ // two requests. Counting pairs here would have asserted half the+ // observations reached the file and called that complete.+ want := result.Observation.Baseline.Requests() ++ result.Observation.Fault.Requests() ++ result.Observation.Recovery.Requests()++ if len(events) != want {+ t.Fatalf("wrote %d events for %d observations", len(events), want)+ }++ windows := map[string]int{}+ for _, event := range events {+ windows[event.Window]++++ if event.At == "" || event.Stream == "" || event.Disposition == "" {+ t.Fatalf("an event is missing a field: %+v", event)+ }+ }++ for _, name := range []string{"baseline", "fault", "recovery"} {+ if windows[name] == 0 {+ t.Errorf("no events carry the window %q", name)+ }+ }+}++// Within a window the streams are recorded one after another, so a naive+// concatenation reads as four separate runs rather than one interleaved one.+func TestEventsWithinAWindowAreTimeOrdered(t *testing.T) {+ events := EventsOf(finished().Observation)++ for i := 1; i < len(events); i++ {+ if events[i-1].Window != events[i].Window {+ continue+ }++ if events[i-1].At > events[i].At {+ t.Fatalf(+ "events in the %s window are out of order: %s then %s",+ events[i].Window,+ events[i-1].At,+ events[i].At,+ )+ }+ }+}++// The hash is what stops a record being silently separated from its data, so+// it has to be the hash of the bytes that actually landed.+func TestTheRecordHashesTheEventsItActuallyWrote(t *testing.T) {+ dir := t.TempDir()+ result := finished()++ written, err := Write(+ dir,+ Of("1.2.3", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ )+ if err != nil {+ t.Fatalf("write: %v", err)+ }++ if written.Events == nil {+ t.Fatal("the record carries no events reference")+ }++ body, err := os.ReadFile(filepath.Join(dir, EventsFile))+ if err != nil {+ t.Fatalf("read the events file: %v", err)+ }++ sum := sha256.Sum256(body)+ if got := hex.EncodeToString(sum[:]); got != written.Events.SHA256 {+ t.Fatalf(+ "the record claims sha256 %s and the file on disk hashes to %s",+ written.Events.SHA256,+ got,+ )+ }++ lines := strings.Count(strings.TrimRight(string(body), "\n"), "\n") + 1+ if lines != written.Events.Count {+ t.Fatalf(+ "the record claims %d events and the file has %d lines",+ written.Events.Count,+ lines,+ )+ }+}++// The record on disk must parse back into the same shape, or every+// projection built on it is built on a guess.+func TestTheWrittenRecordParsesBackIdentically(t *testing.T) {+ dir := t.TempDir()+ result := finished()++ written, err := Write(+ dir,+ Of("1.2.3", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ )+ if err != nil {+ t.Fatalf("write: %v", err)+ }++ body, err := os.ReadFile(filepath.Join(dir, RecordFile))+ if err != nil {+ t.Fatalf("read the record: %v", err)+ }++ var parsed Record+ if err := json.Unmarshal(body, &parsed); err != nil {+ t.Fatalf("the record khaos wrote is not valid JSON: %v", err)+ }++ first, err := Encode(written)+ if err != nil {+ t.Fatalf("encode: %v", err)+ }++ second, err := Encode(parsed)+ if err != nil {+ t.Fatalf("re-encode: %v", err)+ }++ if string(first) != string(second) {+ t.Fatal("the record does not survive a parse-and-re-encode round trip")+ }+}++// Every events line is its own JSON object. A single malformed line makes+// the whole file useless to a streaming reader, which is the only kind of+// reader a 1,200-line file gets.+func TestEveryEventsLineIsItsOwnObject(t *testing.T) {+ dir := t.TempDir()+ result := finished()++ if _, err := Write(+ dir,+ Of("1.2.3", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ ); err != nil {+ t.Fatalf("write: %v", err)+ }++ body, err := os.ReadFile(filepath.Join(dir, EventsFile))+ if err != nil {+ t.Fatalf("read: %v", err)+ }++ if !strings.HasSuffix(string(body), "\n") {+ t.Fatal(+ "the events file does not end in a newline, so its last line is unterminated",+ )+ }++ for i, line := range strings.Split(strings.TrimRight(string(body), "\n"), "\n") {+ var event Event+ if err := json.Unmarshal([]byte(line), &event); err != nil {+ t.Fatalf("events line %d is not an object: %v", i+1, err)+ }+ }+}++// Atomicity is the requirement that fails silently: a reader that opens a+// half-written file gets no error, just a truncated record. Proven by there+// being no leftover temporary file and by the destination never existing in+// a partial state, which is what rename-into-place buys.+func TestTheWriteLeavesNoPartialFileBehind(t *testing.T) {+ dir := t.TempDir()+ result := finished()++ if _, err := Write(+ dir,+ Of("1.2.3", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ ); err != nil {+ t.Fatalf("write: %v", err)+ }++ entries, err := os.ReadDir(dir)+ if err != nil {+ t.Fatalf("read the directory: %v", err)+ }++ names := []string{}+ for _, entry := range entries {+ names = append(names, entry.Name())++ if strings.HasPrefix(entry.Name(), ".khaos-") {+ t.Errorf(+ "a temporary file survived the write: %s",+ entry.Name(),+ )+ }+ }++ if len(names) != 2 {+ t.Fatalf(+ "the directory holds %v, want exactly the record and its events",+ names,+ )+ }+}++// A second run into the same directory replaces the pair rather than failing+// or appending, because re-running one experiment is the normal thing to do.+func TestWritingTwiceReplacesTheRecord(t *testing.T) {+ dir := t.TempDir()+ result := finished()++ for range 2 {+ if _, err := Write(+ dir,+ Of("1.2.3", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ ); err != nil {+ t.Fatalf("write: %v", err)+ }+ }++ entries, err := os.ReadDir(dir)+ if err != nil {+ t.Fatalf("read: %v", err)+ }++ if len(entries) != 2 {+ t.Fatalf("a second write left %d files, want 2", len(entries))+ }+}++// Rename-into-place versus truncate-in-place, discriminated by the INODE.+// This is the half of atomicity a single-threaded test can actually prove: a+// truncating write reuses the destination's inode and is visible to a reader+// in a partial state, while a rename installs a different inode whose+// contents were complete before it became reachable under that name.+func TestTheRecordIsRenamedIntoPlaceRatherThanTruncated(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, RecordFile)+ result := finished()++ // A first write, so the destination already exists and a truncating+ // implementation has something to truncate.+ if _, err := Write(+ dir,+ Of("1.2.3", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ ); err != nil {+ t.Fatalf("first write: %v", err)+ }++ before, err := inodeOf(path)+ if err != nil {+ t.Fatalf("stat before: %v", err)+ }++ if _, err := Write(+ dir,+ Of("9.9.9", result, definition(t), 2, "defect"),+ EventsOf(result.Observation),+ ); err != nil {+ t.Fatalf("second write: %v", err)+ }++ after, err := inodeOf(path)+ if err != nil {+ t.Fatalf("stat after: %v", err)+ }++ if before == after {+ t.Fatal(+ "the record kept its inode across a rewrite, so it was written in place; a reader can see it half-written and gets no error saying so",+ )+ }++ // And the new content is actually there, or "it moved" is meaningless.+ body, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("read: %v", err)+ }++ if !strings.Contains(string(body), "9.9.9") {+ t.Fatal("the second write did not replace the record's contents")+ }+}++func inodeOf(path string) (uint64, error) {+ info, err := os.Stat(path)+ if err != nil {+ return 0, err+ }++ stat, ok := info.Sys().(*syscall.Stat_t)+ if !ok {+ return 0, errNoInode+ }++ return stat.Ino, nil+}++var errNoInode = errors.New(+ "this platform does not expose an inode, so the rename cannot be distinguished from a truncation here",+)diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.gonew file mode 100644index 00000000..db1ac666--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go@@ -0,0 +1,248 @@+// ©AngelaMos | 2026+// write.go++package record++import (+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "sort"++ "github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/probe"+)++const (+ dirMode = 0o755++ tempPattern = ".khaos-*"++ errWriteDir = "create the directory for the run record: %w"+ errWriteFile = "write %s: %w"+ errEncode = "encode the run record: %w"+)++// Event is one line of events.jsonl. The window is on the line rather than+// implied by position, so a consumer can filter without tracking state, and+// `grep '"window":"fault"' events.jsonl` is a useful thing to be able to do.+type Event struct {+ At string `json:"at"`+ Window string `json:"window"`+ Stream string `json:"stream"`+ Credential string `json:"credential"`+ Disposition string `json:"disposition"`+ Status int `json:"status,omitempty"`+ LatencyMS int64 `json:"latency_ms"`+ Detail string `json:"detail,omitempty"`+}++// Write puts the record and its events in dir, and fills the record's events+// reference with the hash of what it actually wrote.+//+// The events go down FIRST: the record names their hash, so writing the+// record before the bytes it hashes would leave a record pointing at a file+// that does not exist yet, which is the window a reader is most likely to+// land in when a run is killed.+func Write(dir string, rec Record, events []Event) (Record, error) {+ if err := os.MkdirAll(dir, dirMode); err != nil {+ return rec, fmt.Errorf(errWriteDir, err)+ }++ encoded, err := encodeEvents(events)+ if err != nil {+ return rec, err+ }++ if err := writeAtomic(+ filepath.Join(dir, EventsFile),+ encoded,+ ); err != nil {+ return rec, fmt.Errorf(errWriteFile, EventsFile, err)+ }++ sum := sha256.Sum256(encoded)++ rec.Events = &Events{+ File: EventsFile,+ SHA256: hex.EncodeToString(sum[:]),+ Count: len(events),+ }++ body, err := Encode(rec)+ if err != nil {+ return rec, err+ }++ if err := writeAtomic(+ filepath.Join(dir, RecordFile),+ body,+ ); err != nil {+ return rec, fmt.Errorf(errWriteFile, RecordFile, err)+ }++ return rec, nil+}++// Encode is the one JSON form, used for the files and for stdout alike, so+// the two cannot differ. Indented because a record a human will not open is+// a record nobody checks, and HTML escaping off because a URL in a reason+// should read as a URL.+func Encode(rec Record) ([]byte, error) {+ buffer := &writeBuffer{}++ encoder := json.NewEncoder(buffer)+ encoder.SetIndent("", " ")+ encoder.SetEscapeHTML(false)++ if err := encoder.Encode(rec); err != nil {+ return nil, fmt.Errorf(errEncode, err)+ }++ return buffer.bytes, nil+}++type writeBuffer struct {+ bytes []byte+}++func (b *writeBuffer) Write(p []byte) (int, error) {+ b.bytes = append(b.bytes, p...)+ return len(p), nil+}++func encodeEvents(events []Event) ([]byte, error) {+ buffer := &writeBuffer{}++ encoder := json.NewEncoder(buffer)+ encoder.SetEscapeHTML(false)++ for _, event := range events {+ if err := encoder.Encode(event); err != nil {+ return nil, fmt.Errorf(errEncode, err)+ }+ }++ return buffer.bytes, nil+}++// writeAtomic writes a temporary file in the SAME directory and renames it+// into place. Same directory because rename is only atomic within one+// filesystem, and atomic because a reader that opens a half-written record+// gets no error, just a truncated one.+func writeAtomic(path string, body []byte) error {+ path = filepath.Clean(path)+ dir := filepath.Dir(path)++ temp, err := os.CreateTemp(dir, tempPattern)+ if err != nil {+ return err+ }++ name := temp.Name()++ // Removed on every failure path below, so a run that could not finish+ // writing does not leave litter next to the record it failed to produce.+ defer func() { _ = os.Remove(name) }()++ if _, err := temp.Write(body); err != nil {+ _ = temp.Close()+ return err+ }++ // Durability before visibility: a rename that beats the data to disk+ // leaves a correctly named, empty file after a crash.+ if err := temp.Sync(); err != nil {+ _ = temp.Close()+ return err+ }++ if err := temp.Close(); err != nil {+ return err+ }++ // Left at the 0600 os.CreateTemp gives it, deliberately. A probe+ // outcome's Detail carries whatever the target answered, so the record+ // is not a file to widen by default, and the operator khaos runs as is+ // the operator CI runs as.+ //nolint:gosec // the operator's own --record destination, cleaned above;+ // khaos holds no privilege they do not, so there is no boundary here to+ // traverse+ return os.Rename(name, path)+}++// Events flattens every window's streams into one time-ordered sequence,+// which is the order they were observed in and the order a reader expects.+func EventsOf(observation experiment.Observation) []Event {+ out := []Event{}++ for _, named := range []struct {+ name string+ window probe.Window+ }{+ {"baseline", observation.Baseline},+ {"fault", observation.Fault},+ {"resumed", observation.Resumed},+ {"recovery", observation.Recovery},+ } {+ out = append(out, eventsOfWindow(named.name, named.window)...)+ }++ return out+}++func eventsOfWindow(window string, from probe.Window) []Event {+ out := []Event{}++ for _, named := range []struct {+ name string+ stream probe.Stream+ }{+ {"valid", from.Valid},+ {"invalid", from.Invalid},+ {"revoked", from.Revoked},+ {"forged", from.Forged},+ } {+ for _, outcome := range named.stream.Outcomes {+ out = append(out, Event{+ At: outcome.At.UTC().Format(timeLayout),+ Window: window,+ Stream: named.name,+ Credential: string(outcome.Credential),+ Disposition: string(outcome.Disposition),+ Status: outcome.Status,+ LatencyMS: outcome.LatencyMS,+ Detail: outcome.Detail,+ })+ }+ }++ // Within a window the streams are recorded one after another, so a naive+ // concatenation interleaves nothing and reads as four separate runs.+ sort.SliceStable(out, func(i, j int) bool {+ return out[i].At < out[j].At+ })++ return out+}++// WriteFile puts one projection where it was asked for, with the same+// atomic rename the record gets: node_exporter reading a half-written+// metrics file ships corrupted data rather than an error, and a CI reporter+// parsing a half-written XML file reports a broken suite.+func WriteFile(path string, body []byte) error {+ dir := filepath.Dir(filepath.Clean(path))++ if err := os.MkdirAll(dir, dirMode); err != nil {+ return fmt.Errorf(errWriteDir, err)+ }++ if err := writeAtomic(path, body); err != nil {+ return fmt.Errorf(errWriteFile, filepath.Base(path), err)+ }++ return nil+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/justfile b/PROJECTS/intermediate/chaos-engineering-tool/justfileindex eb426091..0bc1fbc4 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/justfile+++ b/PROJECTS/intermediate/chaos-engineering-tool/justfile@@ -97,7 +97,7 @@ fix:check: lint tidy-check test colors-gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gate control-gate+gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate credential-gate control-gate report-gatecolors: build./scripts/colors.sh@@ -132,5 +132,8 @@ credential-gate: buildcontrol-gate: build./scripts/control.sh+report-gate: build+ ./scripts/report.sh+clean:rm -f {{binary}}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.shindex 25b08686..85df6ec7 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/control.sh@@ -73,8 +73,13 @@ start_range() {[[ "$with_db" == "yes" ]] && args+=(--with-db)"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" "${args[@]}" >/dev/null 2>&1; then- fail "range up --fail-mode $fail_mode (with-db=$with_db)"++ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" "${args[@]}" 2>&1)"; then+ fail "range up --fail-mode $fail_mode (with-db=$with_db): $(tail -3 <<<"$started")"return 1fireturn 0@@ -198,7 +203,12 @@ assert_denied() {assert_report_has \"${label} is not called a defect" \"without calling it a defect"- assert_exit "$label" 0++ # 3, not 0. M10 gave a deviation in a direction the polarity does NOT+ # call a defect its own exit code: the control did change under the+ # fault, and reporting that as a clean pass hid a real availability+ # finding behind a green build.+ assert_exit "$label" 3}echo "== the three default-range controls, fail-open =="diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.shindex 627e5b81..e4f8039a 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh@@ -136,6 +136,39 @@ assert_verdict() {fi}+# exit_for_verdict maps an expected verdict onto khaos's exit contract.+#+# It takes the experiment's DECLARED DEFECT DIRECTIONS as its second argument,+# and that is not decoration: the exit code is a function of (verdict,+# polarity), never of the verdict alone. `passed / HELD` is 0 for most of the+# catalogue and 2 for the pooled family, where poolPolarity declares HELD the+# defect -- a lifetime that converts a rotation into a revocation is the thing+# those experiments hunt. An earlier version of this helper read the verdict+# string by itself and reported four pooled assertions as exit 0 when khaos+# correctly returned 2.+#+# The directions are passed per section rather than looked up, so an inverted+# polarity is visible at the call site instead of buried in a helper.+exit_for_verdict() {+ local verdict="$1" defects="${2:-PERMITTED}" direction++ direction="${verdict##*/ }"++ if [[ " ${defects} " == *" ${direction} "* ]]; then+ echo 2+ return+ fi++ case "$verdict" in+ "passed / "*) echo 0 ;;+ "deviated / "*) echo 3 ;;+ *)+ echo "exit_for_verdict: no code for the verdict '$verdict'" >&2+ echo 99+ ;;+ esac+}+assert_exit() {local label="$1" want="$2"if [[ "$last_code" == "$want" ]]; then@@ -249,9 +282,13 @@ start_range() {local secret="${2:-unchecked}""$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --with-db --pool-mode "$pool" \- --secret-mode "$secret" --no-color --quiet >/dev/null 2>&1; then- fail "range up --with-db --pool-mode $pool --secret-mode $secret"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --with-db --pool-mode "$pool" \+ --secret-mode "$secret" --no-color --quiet 2>&1)"; then+ fail "range up --with-db --pool-mode $pool --secret-mode $secret: $(tail -3 <<<"$started")"return 1fireturn 0@@ -364,12 +401,6 @@ declare -A WANT_VERDICT=([held]="passed / HELD")-declare -A WANT_EXIT=(- [unbounded]=2- [bounded]=0- [held]=2-)-# What khaos DID is carried by the HELD CAVEAT for these two, not by the defect# phrase: that phrase is part of a leak reason and a leak is exactly what a# HELD finding does not have. Asserting the defect phrase here failed six times@@ -390,7 +421,8 @@ for pool in unbounded bounded held; dolabel="${experiment#credential.} on a ${pool} pool"assert_verdict "$label" "${WANT_VERDICT[$pool]}"- assert_exit "$label" "${WANT_EXIT[$pool]}"+ assert_exit "$label" \+ "$(exit_for_verdict "${WANT_VERDICT[$pool]}" "HELD PERMITTED")"assert_confirmed "$label"assert_control_held "$label"@@ -446,13 +478,6 @@ declare -A WANT_SECRET_UNCHECKED=([credential.secret_file_unreadable]="deviated / DENIED")-declare -A WANT_SECRET_EXIT=(- [credential.secret_file_empty]=2- [credential.secret_file_truncated]=0- [credential.secret_file_missing]=0- [credential.secret_file_unreadable]=0-)-if start_range unbounded unchecked; thendeclare -A SECRET_OBSERVABLE=()@@ -462,7 +487,7 @@ if start_range unbounded unchecked; thenlabel="${experiment#credential.} against a verifier that does not check the length"assert_verdict "$label" "${WANT_SECRET_UNCHECKED[$experiment]}"- assert_exit "$label" "${WANT_SECRET_EXIT[$experiment]}"+ assert_exit "$label" "$(exit_for_verdict "${WANT_SECRET_UNCHECKED[$experiment]}")"assert_confirmed "$label"assert_control_held "$label"@@ -514,7 +539,7 @@ if start_range unbounded validated; then"secret_file_empty against a verifier that checks the length" \"deviated / DENIED"assert_exit \- "secret_file_empty against a verifier that checks the length" 0+ "secret_file_empty against a verifier that checks the length" 3assert_confirmed \"secret_file_empty against a verifier that checks the length"assert_report_lacks \@@ -582,11 +607,6 @@ declare -A WANT_ENV=([validated]="deviated / DENIED")-declare -A WANT_ENV_EXIT=(- [unchecked]=2- [validated]=0-)-for secret in unchecked validated; doif ! start_range unbounded "$secret"; thencontinue@@ -598,7 +618,7 @@ for secret in unchecked validated; dolabel="${experiment#credential.} against a ${secret} verifier"assert_verdict "$label" "${WANT_ENV[$secret]}"- assert_exit "$label" "${WANT_ENV_EXIT[$secret]}"+ assert_exit "$label" "$(exit_for_verdict "${WANT_ENV[$secret]}")"assert_confirmed "$label"assert_control_held "$label"done@@ -870,13 +890,6 @@ declare -A WANT_AUTHZ=([cached]="passed / HELD")-declare -A WANT_AUTHZ_EXIT=(- [collapses]=2- [distinguishes]=0- [retries]=0- [cached]=0-)-for mode in collapses distinguishes retries cached; doif ! start_authz_range "$mode"; thencontinue@@ -887,7 +900,8 @@ for mode in collapses distinguishes retries cached; dolabel="apikey_revoked against a ${mode} caller"assert_verdict "$label" "${WANT_AUTHZ[$mode]}"- assert_exit "$label" "${WANT_AUTHZ_EXIT[$mode]}"+ assert_exit "$label" \+ "$(exit_for_verdict "${WANT_AUTHZ[$mode]}" "PERMITTED DENIED")"assert_confirmed "$label"assert_control_held "$label" "the caller authenticates credentials"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.shindex 080876c6..65a3dda4 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh@@ -72,9 +72,13 @@ start_range() {local dns_mode="${2:-reresolve}""$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --fail-mode "$1" --dns-mode "$dns_mode" \- --no-color --quiet >/dev/null 2>&1; then- fail "range up --fail-mode $1 --dns-mode $dns_mode"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --fail-mode "$1" --dns-mode "$dns_mode" \+ --no-color --quiet 2>&1)"; then+ fail "range up --fail-mode $1 --dns-mode $dns_mode: $(tail -3 <<<"$started")"return 1fireturn 0@@ -354,7 +358,11 @@ if start_range open; thenfor fault in ttl_zero ttl_long; dorun_experiment "dns.${fault}"assert_verdict "${fault} vs an uncached target" "unsupported"- assert_exit "${fault} vs an uncached target" 4++ # 9, not 4. M10 gave UNSUPPORTED its own code: a question that does+ # not apply to this target is not a measurement khaos failed to make,+ # and it was indistinguishable from one while both exited 4.+ assert_exit "${fault} vs an uncached target" 9# Anchored on the LIFETIME khaos served, which is the measurement the# verdict is entitled to make. The target's lookups do not scale with@@ -394,7 +402,9 @@ if start_range closed; thenfor fault in nxdomain servfail; dorun_experiment "dns.${fault}"assert_verdict "${fault} vs fail-closed" "deviated / DENIED"- assert_exit "${fault} vs fail-closed" 0++ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.+ assert_exit "${fault} vs fail-closed" 3assert_confirmed "${fault}"donediff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.shindex 720e6eab..0d76202d 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/family.sh@@ -106,8 +106,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }start_range() {"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --fail-mode "$1" --quiet --no-color >/dev/null 2>&1; then- fail "range up --fail-mode $1"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --fail-mode "$1" --quiet --no-color 2>&1)"; then+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"return 1fireturn 0@@ -134,6 +138,35 @@ assert_verdict() {fi}+# exit_for_verdict maps an expected verdict onto khaos's exit contract.+#+# The second argument is the experiment's DECLARED DEFECT DIRECTIONS, because+# the code is a function of (verdict, polarity) and never of the verdict+# alone: `passed / HELD` is 0 ordinarily and 2 wherever a polarity declares+# HELD the defect. The loop below passes the default because the one inverted+# experiment in this family, identity.key_rotation, is held out as INVERTED+# and asserted separately -- but the helper takes the argument so that a+# second inversion cannot pass silently.+exit_for_verdict() {+ local verdict="$1" defects="${2:-PERMITTED}" direction++ direction="${verdict##*/ }"++ if [[ " ${defects} " == *" ${direction} "* ]]; then+ echo 2+ return+ fi++ case "$verdict" in+ "passed / "*) echo 0 ;;+ "deviated / "*) echo 3 ;;+ *)+ echo "exit_for_verdict: no code for the verdict '$verdict'" >&2+ echo 99+ ;;+ esac+}+assert_exit() {local label="$1" got="$2" want="$3"if [[ "$got" == "$want" ]]; then@@ -389,9 +422,17 @@ if start_range closed; thenfor experiment in "${parts[@]:1}"; dolabel="${experiment} vs a correct verifier"+ want_verdict="${CORRECT_VERDICT[$experiment]:-passed / HELD}"+run_experiment "$experiment"- assert_exit "$label" "$last_code" 0- assert_verdict "$label" "${CORRECT_VERDICT[$experiment]:-passed / HELD}"++ # DERIVED from the expected verdict rather than written beside it.+ # M10 gave a deviation in a non-defect direction its own exit+ # code, so four of these seventeen now exit 3 and the rest exit 0+ # -- and a hand-maintained second table would be one more thing+ # that can disagree with the first.+ assert_exit "$label" "$last_code" "$(exit_for_verdict "$want_verdict")"+ assert_verdict "$label" "$want_verdict"if grep -qE "^ verdict .*PERMITTED" <<<"$report"; thenfail "${label} found a bypass in a correct verifier: $(verdict_line)"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.shindex bbb22bc2..3aab5f86 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/identity.sh@@ -38,8 +38,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }start_range() {"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --fail-mode "$1" --cache-ttl "$CACHE_TTL" --quiet --no-color >/dev/null 2>&1; then- fail "range up --fail-mode $1"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --fail-mode "$1" --cache-ttl "$CACHE_TTL" --quiet --no-color 2>&1)"; then+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"return 1fireturn 0@@ -180,7 +184,9 @@ fiif start_range closed; thenrun_experimentassert_verdict "fail-closed" "deviated / DENIED"- assert_exit "fail-closed" "$last_code" 0++ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.+ assert_exit "fail-closed" "$last_code" 3assert_no_cause "fail-closed"if grep -q "acceptable for this control" <<<"$report"; thendiff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.shindex fec7b6c6..ba4610e4 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/proxy.sh@@ -65,8 +65,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }start_range() {"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --fail-mode "$1" --quiet --no-color >/dev/null 2>&1; then- fail "range up --fail-mode $1"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --fail-mode "$1" --quiet --no-color 2>&1)"; then+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"return 1fireturn 0@@ -75,9 +79,21 @@ start_range() {# Sets the globals `report` and `last_code`. Never call it through a command# substitution: that runs it in a subshell and the report is lost while the exit# code still comes back, which looks like a passing gate with no evidence.+# Extra arguments REPLACE the default timing rather than appending to it: two+# copies of --fault-for on one command line is an ambiguity, and a helper that+# silently dropped an override would make a section measure the default window+# while claiming to measure a short one.run_experiment() {+ local experiment="$1"+ shift++ local -a timing=("${TIMING[@]}")+ if [[ $# -gt 0 ]]; then+ timing=("$@")+ fi+set +e- report="$("$BIN" run "$1" --no-color "${TIMING[@]}" 2>&1)"+ report="$("$BIN" run "$experiment" --no-color "${timing[@]}" 2>&1)"last_code=$?set -e}@@ -175,7 +191,27 @@ assert_an_unconfirmable_fault_is_not_success() {returnfi- assert_exit "an unconfirmable fault" "$last_code" 4+ # The code is DERIVED from the epistemic word the run actually reported,+ # not hardcoded. M10 split the one inconclusive code into three, and this+ # check accepts three states: asserting a single code against a+ # three-state regex is a flake that fires whenever the run lands on one+ # of the other two.+ local epistemic+ epistemic="$(grep -E '^ verdict' <<<"$report" |+ sed -E 's/^ verdict +([a-z-]+).*/\1/' | head -1 || true)"++ local want+ case "$epistemic" in+ aborted) want=7 ;;+ probe-error) want=5 ;;+ baseline-not-met) want=8 ;;+ *)+ fail "could not read an epistemic word out of the verdict: ${epistemic}"+ return+ ;;+ esac++ assert_exit "an unconfirmable fault (${epistemic})" "$last_code" "$want"}assert_ran_with_no_flags() {@@ -319,7 +355,11 @@ if start_range closed; thenfiassert_verdict "${experiment} fail-closed" "deviated / DENIED"- assert_exit "${experiment} fail-closed" "$last_code" 0++ # 3, not 0: M10 moved a deviation in a direction the polarity does+ # NOT call a defect onto its own code. The target did change under the+ # fault, and reporting that as a clean pass hid it.+ assert_exit "${experiment} fail-closed" "$last_code" 3if grep -q "acceptable for this control" <<<"$report"; thenpass "${experiment} fail-closed polarity spared the safe direction"@@ -329,6 +369,41 @@ if start_range closed; thendonefi+# M10's most dangerous exit code, and the one no other gate exercises. A HELD+# verdict khaos declined to call conclusive used to exit 0: khaos asserting a+# control held on evidence it refused to stand behind, as a green build.+#+# Asserted as a PAIR. "exit 4 on a short window" alone is satisfied by any+# failure; the same experiment with a full window exiting 0 is what proves the+# 4 came from the sample floor and not from something else going wrong.+if start_range closed; then+ run_experiment "$EOF_DEPENDENT_FAULT" \+ --baseline-for 4s --fault-for 3s --recover-for 2s --sample-every 300ms++ assert_verdict "a thin HELD window" "passed / HELD"+ assert_exit "a thin HELD window" "$last_code" 4++ # The SHORTFALL, named with both numbers. Without this the assertion+ # cannot tell a sample floor from any other inconclusive cause.+ if grep -qE 'paired samples cannot rule one out' <<<"$report"; then+ pass "a thin HELD window says how many samples it had and how many it needed"+ else+ fail "a thin HELD window did not name the sample shortfall: $(grep -E '^ why' <<<"$report" | head -2)"+ fi++ # The negative control: the same fault, a full window, exit 0.+ run_experiment "$EOF_DEPENDENT_FAULT" "${TIMING[@]}"++ assert_verdict "a full HELD window" "passed / HELD"+ assert_exit "a full HELD window" "$last_code" 0++ if grep -qE 'cannot rule one out' <<<"$report"; then+ fail "a full window still claimed its evidence was thin"+ else+ pass "a full HELD window claims its conclusion without a caveat"+ fi+fi+if [[ $failures -ne 0 ]]; thenecho "${failures} proxy assertion(s) failed"exit 1diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.shindex 72c77f74..304a6f2c 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/range.sh@@ -26,8 +26,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }start_range() {"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --fail-mode "$1" --cache-ttl "${2:-2s}" --quiet --no-color >/dev/null 2>&1; then- fail "range up --fail-mode $1 --cache-ttl ${2:-2s}"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --fail-mode "$1" --cache-ttl "${2:-2s}" --quiet --no-color 2>&1)"; then+ fail "range up --fail-mode $1 --cache-ttl ${2:-2s}: $(tail -3 <<<"$started")"return 1fireturn 0@@ -196,7 +200,9 @@ if start_range closed; thenrun_experimentassert_zero_flags "fail-closed"assert_verdict "fail-closed" "deviated / DENIED"- assert_exit "fail-closed" "$last_code" 0++ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.+ assert_exit "fail-closed" "$last_code" 3if grep -q "acceptable for this control" <<<"$report"; thenpass "fail-closed the deviation was recorded without calling it a defect"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.shnew file mode 100755index 00000000..e79150c5--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh@@ -0,0 +1,337 @@+#!/usr/bin/env bash+# ©AngelaMos | 2026+# report.sh++set -euo pipefail++# M10's exit gate. Five things have to be true of khaos's outputs.+#+# 1. The EXIT CODE TAXONOMY is exercised end to end against a live range.+# A single non-zero code would collapse "a control failed open", "the+# experiment was invalid" and "khaos could not measure" into one signal,+# and those need three different human responses. Every code this gate+# can reach is driven and asserted; the four it cannot are named below+# with the reason, rather than left to look covered.+#+# 2. `--fail-on` NARROWS and never widens. The default keeps every non-pass+# non-zero, because a run khaos declined to conclude from exiting 0 is a+# green build on no evidence. No value may suppress a rollback failure.+#+# 3. The record is the canonical artifact and the events file is linked to+# it BY HASH, so a record cannot be silently separated from the+# observations it summarises.+#+# 4. The projections are projections. The JUnit XML and the metrics file are+# derived from the record, so they must name the same run and the same+# exit code, and a real parser has to accept each of them: `promtool`+# for the metrics and an XML parser for the suite.+#+# 5. A record the operator ASKED FOR and did not get is never silent, and+# never masks a verdict either.++BIN="${BIN:-$PWD/khaos}"++KHAOS_STATE_DIR="$(mktemp -d)"+export KHAOS_STATE_DIR++WORK="$(mktemp -d)"++# Three sample bands, because the verdict's own thresholds are what separate+# three different exit codes: under MinimumPaired (5) khaos cannot measure,+# under MinimumToClaimClosed (30) it will not call a hold conclusive, and+# above it a hold is a hold.+FULL=(--baseline-for 5s --fault-for 20s --recover-for 3s --sample-every 300ms)+THIN=(--baseline-for 4s --fault-for 3s --recover-for 2s --sample-every 300ms)+TINY=(--baseline-for 4s --fault-for 600ms --recover-for 2s --sample-every 300ms)++failures=0+report=""+last_code=0++cleanup() {+ "$BIN" range down --no-color >/dev/null 2>&1 || true+ rm -rf "$KHAOS_STATE_DIR" "$WORK"+}+trap cleanup EXIT INT TERM HUP++pass() { echo "ok $1"; }+fail() { echo "FAIL $1"; failures=$((failures + 1)); }++start_range() {+ "$BIN" range down --no-color >/dev/null 2>&1 || true+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --fail-mode "$1" --no-color --quiet 2>&1)"; then+ fail "range up --fail-mode $1: $(tail -3 <<<"$started")"+ return 1+ fi+ return 0+}++# Never through a command substitution: that runs in a subshell and the+# report is lost while the exit code still comes back, which reads as a+# passing gate with no evidence.+run_experiment() {+ local experiment="$1"+ shift+ set +e+ report="$("$BIN" run "$experiment" --no-color "$@" 2>&1)"+ last_code=$?+ set -e+}++assert_exit() {+ local label="$1" want="$2"+ if [[ "$last_code" == "$want" ]]; then+ pass "${label} exit ${want}"+ else+ fail "${label} exit ${last_code}, want ${want}"+ fi+}++assert_verdict() {+ local label="$1" want="$2"+ if grep -qE "^ verdict .*${want}" <<<"$report"; then+ pass "${label} verdict is ${want}"+ else+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"+ fi+}++echo "== the exit codes, driven against a live range =="++if start_range open; then+ # 2, the gate. The only code a pipeline should break on by default.+ run_experiment control.policy_store_down "${FULL[@]}"+ assert_verdict "a defect" "deviated / PERMITTED"+ assert_exit "a defect" 2++ # 4. A HELD verdict khaos declined to call conclusive. This used to exit+ # 0: khaos asserting a control held on evidence it refused to stand+ # behind, as a green build.+ run_experiment network.dependency_slow_close "${THIN[@]}"+ assert_verdict "a thin hold" "passed / HELD"+ assert_exit "a thin hold" 4++ # 5. Fewer paired samples than khaos needs to measure anything at all,+ # which is a different problem from a hold it will not call conclusive.+ run_experiment network.dependency_slow_close "${TINY[@]}"+ assert_exit "a window too short to measure" 5++ # 7. A fault khaos could not confirm. Measuring nothing is not passing.+ run_experiment network.dependency_partial "${FULL[@]}" --bytes 999999+ assert_exit "a fault that could not be confirmed" 7++ # 1. Usage. Refused before anything is injected.+ run_experiment control.policy_store_down --fail-on not-an-outcome+ assert_exit "an outcome khaos cannot gate on" 1++ if grep -q 'is not an outcome khaos can gate on' <<<"$report"; then+ pass "the refusal names the cause rather than just failing"+ else+ fail "the usage refusal did not name the cause"+ fi+fi++if start_range closed; then+ # 0 and 3 are the pair that makes the directional axis visible: the same+ # fault, the same correct target, and khaos separates "nothing moved"+ # from "it refused to serve, which is correct and costs availability".+ run_experiment network.dependency_slow_close "${FULL[@]}"+ assert_verdict "a clean hold" "passed / HELD"+ assert_exit "a clean hold" 0++ run_experiment network.dependency_reset "${FULL[@]}"+ assert_verdict "a non-defect deviation" "deviated / DENIED"+ assert_exit "a non-defect deviation" 3+fi++echo "== --fail-on narrows, and never past the two it cannot touch =="++if start_range open; then+ run_experiment control.policy_store_down "${FULL[@]}" --fail-on never+ assert_exit "a defect under --fail-on never" 0++ run_experiment control.policy_store_down "${FULL[@]}" --fail-on inconclusive+ assert_exit "a defect under a gate that does not name it" 0++ run_experiment control.policy_store_down "${FULL[@]}" --fail-on defect+ assert_exit "a defect under a gate that names it" 2+fi++echo "== the record is canonical and its events are linked by hash =="++if start_range open; then+ into="${WORK}/record"++ run_experiment control.policy_store_down "${FULL[@]}" \+ --record "$into" \+ --junit "${WORK}/khaos.xml" \+ --metrics-out "${WORK}/khaos.prom"++ assert_exit "a recorded run" 2++ for file in run.json events.jsonl; do+ if [[ -s "${into}/${file}" ]]; then+ pass "the run wrote ${file}"+ else+ fail "the run wrote no ${file}"+ fi+ done++ claimed="$(python3 -c "+import json+print(json.load(open('${into}/run.json'))['events']['sha256'])+" 2>/dev/null || true)"+ actual="$(sha256sum "${into}/events.jsonl" | cut -d' ' -f1)"++ if [[ -n "$claimed" && "$claimed" == "$actual" ]]; then+ pass "the record's hash matches the events file beside it"+ else+ fail "the record claims ${claimed:-<none>} and the file hashes to ${actual}"+ fi++ # The record has to explain the code khaos returned, because the exit+ # status is the only part of a run most CI ever sees.+ recorded="$(python3 -c "+import json+print(json.load(open('${into}/run.json'))['exit']['code'])+" 2>/dev/null || true)"++ if [[ "$recorded" == 2 ]]; then+ pass "the record carries the exit code khaos returned"+ else+ fail "the record says exit ${recorded:-<none>} and khaos returned 2"+ fi+fi++echo "== the projections are accepted by real parsers =="++# promtool is the format's own linter, and it rejects every malformed shape+# this gate cares about: a missing trailing newline is `unexpected end of+# input stream`, a repeated TYPE line and an unescaped label quote are both+# parse errors. Verified against promtool 3.15.0.+if [[ ! -s "${WORK}/khaos.prom" ]]; then+ fail "no metrics file was written, so this section cannot fail"+elif ! command -v promtool >/dev/null 2>&1; then+ fail "promtool is not installed, so khaos's exposition format is unverified"+else+ if promtool check metrics <"${WORK}/khaos.prom" >/dev/null 2>&1; then+ pass "promtool accepts khaos's exposition format"+ else+ fail "promtool rejects khaos's metrics: $(promtool check metrics <"${WORK}/khaos.prom" 2>&1 | head -2)"+ fi++ # The NEGATIVE control on the linter itself: a checker that always says+ # yes cannot detect a malformed file, and the trailing newline is the+ # rule that fails silently.+ if printf '# TYPE x_m gauge\nx_m 1' |+ promtool check metrics >/dev/null 2>&1; then+ fail "promtool accepted a file with no trailing newline, so its approval above proves nothing"+ else+ pass "promtool rejects a file with no trailing newline"+ fi+fi++if [[ ! -s "${WORK}/khaos.xml" ]]; then+ fail "no JUnit file was written, so this section cannot fail"+else+ if python3 -c "+import sys, xml.etree.ElementTree as ET+root = ET.parse('${WORK}/khaos.xml').getroot()+assert root.tag == 'testsuites', root.tag+suite = root.find('testsuite')+assert suite is not None+case = suite.find('testcase')+assert case is not None+assert case.find('failure') is not None, 'a defect is not a failure'+assert int(root.get('failures')) == 1, root.get('failures')+" 2>/dev/null; then+ pass "the JUnit suite parses and reports the defect as a failure"+ else+ fail "the JUnit suite is malformed or does not report the defect"+ fi++ # The way back to the record, for a consumer that only has the XML.+ if grep -q 'name="run_id"' "${WORK}/khaos.xml"; then+ pass "the JUnit suite carries the run id back to the record"+ else+ fail "the JUnit suite carries no run id"+ fi+fi++echo "== a record khaos could not write is never silent =="++if start_range open; then+ # The verdict outranks the bookkeeping: a run that found a bypass still+ # exits on the bypass, and still says the record is missing.+ run_experiment control.policy_store_down "${FULL[@]}" \+ --record /proc/khaos-cannot-write-here++ assert_exit "a defect whose record could not be written" 2++ if grep -q 'could not write the run record' <<<"$report"; then+ pass "the write failure is reported rather than swallowed"+ else+ fail "a record khaos could not write was silent"+ fi++ # But a clean run that could not write the record the operator asked for+ # fails on the write: in CI they believe they have an artifact and do not.+ run_experiment control.policy_store_down "${FULL[@]}" \+ --fail-on never \+ --record /proc/khaos-cannot-write-here++ assert_exit "a clean run whose record could not be written" 1+fi++echo "== the codes this gate cannot reach, named rather than implied =="++# Four codes are deliberately not driven here, and saying so is the point: a+# gate that looked complete while skipping them would be the inert kind.+#+# 6 ROLLBACK needs khaos to fail to revert a fault it applied, which+# means breaking the daemon mid-run. `scripts/survive.sh`+# owns the rollback paths; forcing the failure here would+# leave the operator's range dirty, which is the one thing+# this tool must not do.+# 8 BASELINE needs a target outside steady state BEFORE injection. The+# range is healthy by construction, and the unit census in+# internal/cli covers the mapping.+# 9 UNSUPPORTED driven by scripts/dns.sh and scripts/tls.sh, against the+# targets whose question genuinely does not apply.+# 70 INTERNAL a recovered panic. Reachable only from a khaos bug, so+# there is nothing honest to assert.+#+# A CENSUS rather than a comment. The paragraph above was an unconditional+# `pass`, which is documentation wearing an assertion's clothes: it could not+# fail, so it could not notice a thirteenth code arriving with nothing+# covering it. This counts the taxonomy in the source and holds it to the+# number this gate and the unit census between them account for.+# [[:space:]] rather than \t: grep -E treats a backslash-t as a literal+# `t`, so the tab form matched nothing and this census reported 0 under+# bash while appearing to work when the same line was pasted into zsh.+declared="$(grep -cE '^[[:space:]]+Code[A-Za-z]+ += ' internal/cli/exit.go || true)"++if [[ "$declared" == 12 ]]; then+ pass "the taxonomy is twelve codes, which is what this gate and the unit census cover"+else+ fail "the source declares ${declared} exit codes and this gate accounts for 12; a new code needs a home here or a named reason it cannot have one"+fi++# And the unit census that covers what a live range cannot reach has to+# exist, or the four codes named above are covered by nothing at all.+if grep -q 'func TestNoTwoOutcomesShareACode' internal/cli/exit_test.go; then+ pass "the unit census that pins every code is present"+else+ fail "the unit census is gone, so the codes this gate cannot drive are unverified"+fi++if [[ "$failures" -gt 0 ]]; then+ echo "report: ${failures} assertion(s) failed"+ exit 1+fi++echo "report: all assertions passed"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.shindex 460f59f8..e3809bdc 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/tls.sh@@ -50,8 +50,12 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }# reconfigured mid-run would let khaos change the target it is measuring.start_range() {"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --tls-mode "$1" --no-color --quiet >/dev/null 2>&1; then- fail "range up --tls-mode $1"+ # The OUTPUT, not just the fact. A bare "range up failed" is what a+ # transient startup error looks like in a two-hour gate run, and it+ # leaves nothing to diagnose from.+ local started+ if ! started="$("$BIN" range up --tls-mode "$1" --no-color --quiet 2>&1)"; then+ fail "range up --tls-mode $1: $(tail -3 <<<"$started")"return 1fireturn 0@@ -111,7 +115,9 @@ if start_range closed; then# this family declares it acceptable, so it must not be called a defect.run_experiment tls.leaf_expiredassert_verdict "leaf-expired vs a correct client" "deviated / DENIED"- assert_exit "leaf-expired vs a correct client" 0++ # 3, not 0: M10 gave a deviation in a non-defect direction its own code.+ assert_exit "leaf-expired vs a correct client" 3assert_confirmed "leaf-expired"# THE CONTRAST, and the reason the unsupported verdict is honest rather@@ -119,7 +125,7 @@ if start_range closed; then# certificate is a finding it can actually have.run_experiment tls.revoked_via_crlassert_verdict "revoked vs a client that checks" "deviated / DENIED"- assert_exit "revoked vs a client that checks" 0+ assert_exit "revoked vs a client that checks" 3if grep -qE '^ verdict .*unsupported' <<<"$report"; thenfail "revoked was called unsupported against a client that fetches the CRL, which excuses a real defect"@@ -150,9 +156,13 @@ if start_range no-revocation-check; thenrun_experiment tls.revoked_via_crlassert_verdict "revoked vs a client that never asks" "unsupported"- # NOT zero. A revoked certificate in active use must never exit 0, and- # every inconclusive epistemic outcome exits 4.- assert_exit "revoked vs a client that never asks" 4+ # NOT zero. A revoked certificate in active use must never exit 0.+ #+ # 9 rather than 4 as of M10: "every inconclusive epistemic outcome exits+ # 4" was true when this was written and is not any more. UNSUPPORTED has+ # its own code, because a question that does not apply to this target is+ # a different thing from evidence khaos could not gather.+ assert_exit "revoked vs a client that never asks" 9if grep -qE '^ verdict .*passed' <<<"$report"; thenfail "a revoked certificate in active use was reported as a pass"@@ -275,7 +285,7 @@ if start_range closed; thenfor experiment in tls.mtls_client_expired tls.mtls_client_absent; dorun_experiment "$experiment"assert_verdict "${experiment} vs a client that reads" "deviated / DENIED"- assert_exit "${experiment} vs a client that reads" 0+ assert_exit "${experiment} vs a client that reads" 3donefi@@ -307,7 +317,7 @@ if start_range closed; then# revocation, so the hole is the HOOK rather than resumption itself.run_experiment tls.resumption_after_revocationassert_verdict "resumption vs a per-connection policy" "deviated / DENIED"- assert_exit "resumption vs a per-connection policy" 0+ assert_exit "resumption vs a per-connection policy" 3fiif [[ "$failures" -gt 0 ]]; then