Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
fix(khaos): the seven findings an outside audit could see and this project could not
CarterPerez-dev9c4d850c4bc1
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.goindex 69ea2fd0..fe04e0ce 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main.go@@ -9,6 +9,7 @@ import ("fmt""os""os/signal"+ "runtime/debug""strconv""strings""syscall"@@ -36,6 +37,11 @@ const (noColorFlagTrue = noColorFlag + "="errorFormat = "%s: %v\n"fixFormat = " fix: %s\n"+ panicFormat = " cause: %v\n"+)++var internalFailure = errors.New(+ "khaos hit a bug and stopped; every fault it had applied was reverted on the way out",)var errInterrupted = errors.New(@@ -43,7 +49,34 @@ var errInterrupted = errors.New()func main() {- os.Exit(execute())+ os.Exit(guarded(execute))+}++// guarded is the boundary that makes CodeInternal reachable. The taxonomy+// published code 70 as "a recovered panic" and nothing in khaos recovered+// one, so a bug crashed with a Go stack trace and whatever exit status the+// runtime chose -- indistinguishable to CI from a tool that could not parse+// its flags.+//+// Recovering HERE and nowhere deeper is deliberate: the panic still unwinds+// every frame below, so every deferred revert, lease release and lock+// handoff runs exactly as it does on any other error path. khaos only+// decides what to say about it.+func guarded(run func() int) (code int) {+ defer func() {+ recovered := recover()+ if recovered == nil {+ return+ }++ fmt.Fprintf(os.Stderr, errorFormat, version.Name, internalFailure)+ fmt.Fprintf(os.Stderr, panicFormat, recovered)+ fmt.Fprint(os.Stderr, string(debug.Stack()))++ code = cli.CodeInternal+ }()++ return run()}func execute() int {@@ -110,36 +143,30 @@ func noColorRequested(args []string) bool {// verdict outcomes is decided once where the run ends rather than re-derived// by a chain here that grows with every epistemic value. The checks below it// are for errors that never reached a verdict at all.+// fail decides what the operator READS. What the process RETURNS is decided by+// cli.ExitCode, which is the same classification the run record carries, so+// the status and the artifact cannot disagree. This function used to carry its+// own table, and a rollback failure exited 6 while run.json said 1.func fail(err error) int {- if errors.Is(err, run.ErrRevertFailed) {- fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)- printFix(err)- return exitRevertFailed- }-- // Checked before the carried code as well as inside it: this is the one- // outcome nothing may suppress, so it is matched on the sentinel too and- // not only on a code some other layer computed.- if errors.Is(err, cli.ErrNotRecovered) {- fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)- return exitRevertFailed- }-- var exit *cli.Exit- if errors.As(err, &exit) {- fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)- return exit.Code- }+ code := cli.ExitCode(err)+ // Cancellation gets its own wording, because "context canceled" is Go's+ // phrasing for a decision the operator made.if errors.Is(err, context.Canceled) {fmt.Fprintf(os.Stderr, errorFormat, version.Name, errInterrupted)- return exitInterrupted++ return code}fmt.Fprintf(os.Stderr, errorFormat, version.Name, err)- printFix(err)- return exitError+ // The fix line is for a refusal that names one, which is every refusal+ // predicate and every unrevertable fault.+ if errors.Is(err, run.ErrRevertFailed) || code == exitError {+ printFix(err)+ }++ return code}func printFix(err error) {diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.goindex 48e0ce9c..d19c13fc 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos/main_test.go@@ -205,3 +205,51 @@ func TestRollbackFailureOutranksEveryOtherExitCode(t *testing.T) {})}}++// Exit code 70 was published in the taxonomy as "a recovered panic" and+// nothing in khaos recovered one: `CodeInternal` appeared in its own+// declaration and in a numeric assertion, and in no production path at all.+// A bug therefore crashed with a Go stack trace and whatever status the+// runtime chose, which CI cannot tell from a tool that could not parse its+// flags.+//+// The deferred cleanup below is the half that matters more than the code: the+// panic still unwinds every frame, so a fault khaos had applied is still+// reverted on the way out.+func TestAPanicBecomesTheInternalCodeWithCleanupIntact(t *testing.T) {+ reverted := false++ code := guarded(func() int {+ defer func() { reverted = true }()++ panic("a bug in khaos")+ })++ if code != cli.CodeInternal {+ t.Errorf(+ "a recovered panic exited %d, want %d: the taxonomy publishes 70 for exactly this",+ code,+ cli.CodeInternal,+ )+ }++ if !reverted {+ t.Error(+ "a deferred revert did not run while the panic unwound, so khaos would leave the fault applied",+ )+ }+}++// And the boundary is not a blanket: an ordinary run's code passes through+// untouched, or every exit in the taxonomy would collapse to 70.+func TestTheGuardLeavesAnOrdinaryExitAlone(t *testing.T) {+ for _, want := range []int{+ cli.CodeOK,+ cli.CodeDefect,+ cli.CodeRollback,+ } {+ if got := guarded(func() int { return want }); got != want {+ t.Errorf("guarded changed exit %d into %d", want, got)+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.goindex e95fab45..7517540d 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit.go@@ -4,12 +4,14 @@package cliimport (+ "context""errors""fmt""io""github.com/CarterPerez-dev/khaos/internal/experiment""github.com/CarterPerez-dev/khaos/internal/record"+ "github.com/CarterPerez-dev/khaos/internal/run""github.com/CarterPerez-dev/khaos/internal/version""github.com/spf13/cobra")@@ -124,9 +126,13 @@ func emitRecord(return outcome}- // A verdict khaos reached outranks a file it could not write.+ // A verdict khaos reached outranks a file it could not write. The warning+ // goes to the DIAGNOSTIC stream: writing it to out appended prose to the+ // JSON document under --output json, which breaks that contract on the one+ // path where CI needs machine-readable evidence.if outcome != nil {- fmt.Fprintf(out, warnRecordWrite, writeErr)+ fmt.Fprintf(emit.diagnostic(), warnRecordWrite, writeErr)+return outcome}@@ -163,15 +169,41 @@ func writeProjections(rec record.Record, into Projections) error {// exitDetail reads the code and the outcome name back off a finished run, so// the record can carry what khaos is about to return to the shell.+//+// It is the SAME classification the process boundary uses, because these were+// two lists: this one mapped anything without a typed Exit to usage code 1+// while the boundary recognised the rollback and cancellation sentinels on its+// own, so a real rollback failure wrote exit.code 1 into run.json and exited 6.func exitDetail(outcome error) (int, string) {if outcome == nil {return CodeOK, ""}+ // Matched on the sentinels BEFORE any carried code, because these are the+ // outcomes nothing may suppress and they must not depend on some other+ // layer having wrapped them correctly.+ if errors.Is(outcome, run.ErrRevertFailed) ||+ errors.Is(outcome, ErrNotRecovered) {+ return CodeRollback, string(outcomeRollback)+ }+var exit *Exitif errors.As(outcome, &exit) {return exit.Code, string(exit.Outcome)}+ if errors.Is(outcome, context.Canceled) {+ return CodeInterrupted, string(OutcomeInterrupted)+ }+return CodeUsage, ""}++// ExitCode is the one authority on what a finished run returns to the shell.+// The process boundary reads it rather than carrying its own table, so the+// status and the record cannot disagree.+func ExitCode(outcome error) int {+ code, _ := exitDetail(outcome)++ return code+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.goindex 30d03e2b..85dfdf8e 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/emit_test.go@@ -180,6 +180,7 @@ func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {}out := &strings.Builder{}+ diagnostic := &strings.Builder{}err := emitRecord(out,@@ -187,8 +188,9 @@ func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {result,defect,Output{- Shape: outputText,- Record: filepath.Join("/proc", "khaos-cannot-write-here"),+ Shape: outputText,+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),+ Diagnostic: diagnostic,},)@@ -197,9 +199,21 @@ func TestAVerdictOutranksARecordThatCouldNotBeWritten(t *testing.T) {}// And the failure is still SAID, because an operator who asked for a- // record and did not get one must not have to guess.- if !strings.Contains(out.String(), "could not write the run record") {- t.Fatalf("the write failure was silent: %.80q", out.String())+ // record and did not get one must not have to guess. On the DIAGNOSTIC+ // stream, not the report stream: the same line on stdout appended prose to+ // the JSON document under --output json.+ if !strings.Contains(+ diagnostic.String(),+ "could not write the run record",+ ) {+ t.Fatalf("the write failure was silent: %.80q", diagnostic.String())+ }++ if strings.Contains(out.String(), "could not write") {+ t.Fatalf(+ "the write failure reached the report stream, which is what breaks --output json: %.80q",+ out.String(),+ )}}@@ -432,3 +446,60 @@ func TestTheReportingFlagsAdvertiseWhatTheyTake(t *testing.T) {}}}++// --output json promises stdout carries the record and nothing else, so+// `khaos run x --output json | jq .verdict` works. A write failure broke that+// promise on exactly the path CI needs most: the JSON went to `out`, and so+// did the prose warning about the artifact khaos could not write, so a defect+// plus an invalid --record destination produced a JSON document followed by a+// sentence. Diagnostics belong on stderr; the machine format gets the stream+// it was asked for.+func TestAnArtifactFailureNeverContaminatesTheJSONStream(t *testing.T) {+ result, definition := emitted(t)++ out := &strings.Builder{}+ diagnostic := &strings.Builder{}++ outcome := &Exit{+ Code: CodeDefect,+ Outcome: OutcomeDefect,+ Err: ErrDefectFound,+ }++ err := emitRecord(+ out,+ definition,+ result,+ outcome,+ Output{+ Shape: outputJSON,+ Record: filepath.Join("/proc", "khaos-cannot-write-here"),+ Diagnostic: diagnostic,+ },+ )++ // The verdict outranks the bookkeeping: the defect still comes back.+ if !errors.Is(err, ErrDefectFound) {+ t.Fatalf("the defect was replaced by the write failure: %v", err)+ }++ body := out.String()++ var parsed record.Record+ if jsonErr := json.Unmarshal([]byte(body), &parsed); jsonErr != nil {+ t.Fatalf(+ "stdout is not a single decodable JSON document, so the --output json contract is broken on the path CI depends on: %v\nstdout was: %q",+ jsonErr,+ body,+ )+ }++ // And the failure is not swallowed: in CI the operator believes they have+ // an artifact, so the warning has to land somewhere they will see it.+ if !strings.Contains(diagnostic.String(), "could not write") {+ t.Fatalf(+ "the artifact failure was not reported on the diagnostic stream: %q",+ diagnostic.String(),+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.goindex 78e21e0d..309c42b0 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/exit_test.go@@ -4,9 +4,13 @@package cliimport (+ "context""errors"+ "fmt"+ "io""slices""testing"+ "time""github.com/CarterPerez-dev/khaos/internal/experiment""github.com/CarterPerez-dev/khaos/internal/run"@@ -441,3 +445,100 @@ func TestTheExitCodesAreTheirPublishedNumbers(t *testing.T) {}}}++// The code in the record and the code the process returns are the SAME+// contract, and they were computed by two different lists.+//+// exitFor returned an execution error unclassified, exitDetail mapped anything+// without a typed Exit to usage code 1, and the process boundary separately+// recognised the rollback and cancellation sentinels. So a real rollback+// failure wrote `exit.code: 1` into run.json while the process exited 6 -- the+// artifact disagreeing with the status about the one outcome no --fail-on value+// may suppress. The unit census never caught it because it called exitFor with+// a nil execution error.+func TestTheRecordedCodeIsTheCodeTheProcessReturns(t *testing.T) {+ for name, tc := range map[string]struct {+ err error+ want int+ }{+ "a fault that could not be reverted": {+ err: run.ErrRevertFailed,+ want: CodeRollback,+ },+ "a target that never recovered": {+ err: ErrNotRecovered,+ want: CodeRollback,+ },+ "an interrupted run": {+ err: context.Canceled,+ want: CodeInterrupted,+ },+ "an ordinary execution error": {+ err: errors.New("the daemon refused"),+ want: CodeUsage,+ },+ "a wrapped rollback failure": {+ err: fmt.Errorf("reverting: %w", run.ErrRevertFailed),+ want: CodeRollback,+ },++ // errors.Join is where the precedence actually bites: the state khaos+ // left the target in outranks the verdict about it, and the RECORD has+ // to agree with that, not just the status.+ "a rollback failure joined with a defect": {+ err: errors.Join(ErrDefectFound, run.ErrRevertFailed),+ want: CodeRollback,+ },+ "a rollback failure joined with cancellation": {+ err: errors.Join(context.Canceled, run.ErrRevertFailed),+ want: CodeRollback,+ },+ "a carried defect code joined with a rollback failure": {+ err: errors.Join(+ &Exit{Code: CodeDefect, Err: ErrDefectFound},+ run.ErrRevertFailed,+ ),+ want: CodeRollback,+ },+ } {+ t.Run(name, func(t *testing.T) {+ outcome := exitFor(held(), tc.err)++ if got := ExitCode(outcome); got != tc.want {+ t.Errorf(+ "the process would exit %d",+ got,+ )+ }++ if got, _ := exitDetail(outcome); got != tc.want {+ t.Errorf(+ "the record would say exit.code %d, want %d: an artifact that disagrees with the status is worse than no artifact",+ got,+ tc.want,+ )+ }+ })+ }+}++// `reap --interval 0` reached time.NewTicker(0), which panics, and it got+// there AFTER the workspace was opened and its lock taken -- so the operator+// paid for a refusal that arrived as a Go stack trace. Validated before+// anything is opened.+func TestANonPositiveReapIntervalIsRefusedBeforeAnythingOpens(t *testing.T) {+ for _, interval := range []time.Duration{+ 0,+ -time.Second,+ } {+ err := runReap(context.Background(), io.Discard, interval)++ if !errors.Is(err, ErrIntervalNotPositive) {+ t.Fatalf(+ "reap --interval %s returned %v, want a refusal naming the interval",+ interval,+ err,+ )+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.goindex 0dc05dfd..fe133e63 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go@@ -339,6 +339,7 @@ func newRunExperimentCmd(definition experiment.Definition) *cobra.Command {Shape: shape,Record: into,Projections: projections,+ Diagnostic: cmd.ErrOrStderr(),},)},@@ -650,6 +651,13 @@ type Output struct {Shape stringRecord stringProjections Projections++ // Diagnostic is where a failure that is NOT the verdict goes. It is+ // separate from the report stream because a warning on stdout turns the+ // --output json contract into a JSON document followed by a sentence, and+ // it is separate from the report because under json the report is+ // discarded while this must still be seen.+ Diagnostic io.Writer}// human is where the terminal report goes, and under --output json that is@@ -663,6 +671,16 @@ func (o Output) human(out io.Writer) io.Writer {return out}+// diagnostic defaults to stderr rather than to the report stream, so a caller+// that forgets to set it cannot silently reintroduce the contamination.+func (o Output) diagnostic() io.Writer {+ if o.Diagnostic == nil {+ return os.Stderr+ }++ return o.Diagnostic+}+func runExperiment(ctx context.Context,out io.Writer,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.goindex 99b6575a..5a91887e 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/reclaim.go@@ -5,6 +5,7 @@ package cliimport ("context"+ "errors""fmt""io""time"@@ -53,6 +54,15 @@ every other fault on the machine.`msgWatching = "watching %s every %s, until you stop it"msgStopped = "stopped watching, %d fault(s) reclaimed"msgDamaged = "the journal has a damaged line; khaos acted on every record it could read, but cannot promise that is all of them"++ errIntervalUsage = "%w: %s"+)++// A non-positive interval reached time.NewTicker, which panics, AFTER the+// workspace had been opened and its lock taken. Refused by name instead, and+// before anything is opened.+var ErrIntervalNotPositive = errors.New(+ "the reap interval has to be longer than zero, or there is no tick to check the journal on",)func newReclaimCmd() *cobra.Command {@@ -125,6 +135,12 @@ func runReclaim(ctx context.Context, out io.Writer, expiredOnly bool) error {}func runReap(ctx context.Context, out io.Writer, interval time.Duration) error {+ // Before the workspace, because a refusal that has already taken the lock+ // is a refusal that cost the operator something.+ if interval <= 0 {+ return fmt.Errorf(errIntervalUsage, ErrIntervalNotPositive, interval)+ }+work, err := workspace.OpenShared(ctx)if err != nil {return errdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.goindex 14588af1..782dd8bb 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/forgery_verdict_test.go@@ -616,3 +616,86 @@ func TestARequiredForgedStreamWithNoDecisionsCannotReportHeld(t *testing.T) {verdict.Epistemic, Passed, strings.Join(verdict.Reasons, "; "))}}++// The guard that requires this experiment's own stream to have been MEASURED+// counted transport observations as decisions.+//+// `Countable()` excludes only khaos's own failures, so a timeout, an+// unreachable target and a throttle all satisfied a check whose own name and+// comment say "the target actually decided about it". Thirty forged probes that+// timed out therefore cleared the floor, and a healthy valid/invalid pair then+// produced a CONCLUSIVE hold about a credential the target was never observed+// to judge. `Answered()` is the discriminator -- the same one that split+// `Unavailable` from `Unreachable` -- and this was the fourth site reading the+// wrong one.+func TestATimedOutForgedStreamIsNotEvidenceTheTargetDecided(t *testing.T) {+ for _, disposition := range []probe.Disposition{+ probe.Unreachable,+ probe.Unavailable,+ } {+ t.Run(string(disposition), func(t *testing.T) {+ observation := forgeryObservation(+ window(+ probe.Accepted,+ probe.Rejected,+ probe.Accepted,+ "",+ MinimumPaired,+ ),+ window(+ probe.Accepted,+ probe.Rejected,+ disposition,+ "",+ MinimumPaired,+ ),+ probe.ExpectAccepted,+ )++ verdict := Judge(observation, idpPolarity)++ if verdict.Epistemic != ProbeError {+ t.Fatalf(+ "verdict = %s / %s conclusive=%v, want %s: every forged probe was a %s, so the target was never seen to decide about the credential this experiment is about (%s)",+ verdict.Epistemic,+ verdict.Direction,+ verdict.Conclusive,+ ProbeError,+ disposition,+ strings.Join(verdict.Reasons, "; "),+ )+ }+ })+ }+}++// The other half, because a guard that rejects everything is not a guard: a+// forged stream the target actually DECIDED about still clears the floor.+func TestADecidedForgedStreamStillClearsTheFloor(t *testing.T) {+ observation := forgeryObservation(+ window(+ probe.Accepted,+ probe.Rejected,+ probe.Accepted,+ "",+ MinimumPaired,+ ),+ window(+ probe.Accepted,+ probe.Rejected,+ probe.Rejected,+ "",+ MinimumPaired,+ ),+ probe.ExpectAccepted,+ )++ verdict := Judge(observation, idpPolarity)++ if verdict.Epistemic == ProbeError {+ t.Fatalf(+ "a forged stream the target refused every time was reported unmeasurable: %s",+ strings.Join(verdict.Reasons, "; "),+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.goindex 7e12d50b..258a2cfc 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go@@ -776,7 +776,11 @@ func requiredStreamThin(observation Observation) (string, bool) {return "", false}- if decided := observation.Fault.Forged.Countable(); decided < MinimumPaired {+ // Answered(), not Countable(): a timeout or an unreachable target is a+ // transport observation and not a decision about the credential, and+ // Countable() excludes only khaos's own failures. This is the same+ // discriminator that splits Unavailable from Unreachable.+ if decided := observation.Fault.Forged.Answered(); decided < MinimumPaired {return sprintf(reasonThinForged, decided, MinimumPaired), true}@@ -1164,25 +1168,49 @@ func unanswered(// recoveryOf requires a run of consecutive healthy pairs, because a state// transition declared on one lucky sample is how a flaky recovery reads as a// clean one. An empty window is UNOBSERVED, never recovered.+//+// The whole window is read, because this used to return the moment it found a+// qualifying run: two healthy pairs followed by twenty broken ones was+// RECOVERED, and a run whose last observed state was broken could exit 0.+//+// Recovery therefore needs two things -- it has to have been ESTABLISHED, and+// it must not have collapsed for a sustained stretch afterwards. The same run+// length governs both, because the reason a single healthy pair cannot declare+// recovery is the reason a single failed one cannot withdraw it. A window of+// fifteen pairs whose last sample happens to be a blip is still a recovery;+// this verdict maps to the one exit code no --fail-on value can suppress, so+// the cost of declaring it on noise lands on every later job.func recoveryOf(recovery probe.Window, closed []probe.Disposition) Recovery {if recovery.Paired() == 0 {return Unobserved}- run := 0+ healthy := 0+ established := false+ collapsed := 0+for i := 0; i < recovery.Valid.Len() && i < recovery.Invalid.Len(); i++ {if recovery.Valid.Outcomes[i].Disposition == probe.Accepted &&shutBy(recovery.Invalid.Outcomes[i].Disposition,decided(closed),) {- run++- if run >= RecoveryRunLength {- return Recovered+ healthy+++ collapsed = 0++ if healthy >= RecoveryRunLength {+ established = true}+continue}- run = 0++ healthy = 0+ collapsed+++ }++ if established && collapsed < RecoveryRunLength {+ return Recovered}return Degradeddiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.goindex f621a863..a6f88c3c 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict_test.go@@ -546,3 +546,111 @@ func TestADeclaredClosureNeverHidesARefusal(t *testing.T) {)}}++// recoveryOf returned the moment it found its run of consecutive healthy pairs+// and never looked at the rest of the window, so a recovery that collapsed+// after two good samples was reported RECOVERED. A run whose last observed+// state was broken could then exit 0.+//+// The criterion is now the END of the window rather than the first qualifying+// run: recovery has to still be established when khaos stops looking. A late+// recovery still counts, because a window that ends with a full healthy run+// ends healthy however it started.+func TestRecoveryIsJudgedAtTheEndOfTheWindowNotTheFirstGoodRun(t *testing.T) {+ healthy := func(n int) []probe.Disposition {+ out := make([]probe.Disposition, n)+ for i := range out {+ out[i] = probe.Accepted+ }+ return out+ }++ for name, tc := range map[string]struct {+ valid []probe.Disposition+ want Recovery+ }{+ "healthy throughout": {+ valid: healthy(6),+ want: Recovered,+ },+ "two good pairs then collapse": {+ valid: []probe.Disposition{+ probe.Accepted,+ probe.Accepted,+ probe.Unreachable,+ probe.Unreachable,+ probe.Unreachable,+ probe.Unreachable,+ },+ want: Degraded,+ },+ "broken first then a full healthy run": {+ valid: []probe.Disposition{+ probe.Unreachable,+ probe.Unreachable,+ probe.Accepted,+ probe.Accepted,+ probe.Accepted,+ },+ want: Recovered,+ },+ "alternating never settles": {+ valid: []probe.Disposition{+ probe.Accepted,+ probe.Unreachable,+ probe.Accepted,+ probe.Unreachable,+ probe.Accepted,+ },+ want: Degraded,+ },+ "recovers only on the very last pair": {+ valid: []probe.Disposition{+ probe.Unreachable,+ probe.Unreachable,+ probe.Unreachable,+ probe.Accepted,+ },+ want: Degraded,+ },+ // The brittleness guard. A single trailing blip in a long healthy+ // window must NOT withdraw a recovery: this verdict maps to the one+ // exit code no --fail-on value can suppress, and declaring it on one+ // unlucky last sample makes every later job on that runner suspect.+ "one trailing blip after a long healthy run": {+ valid: append(healthy(12), probe.Unreachable),+ want: Recovered,+ },+ "two trailing blips are a sustained collapse": {+ valid: append(+ healthy(12),+ probe.Unreachable,+ probe.Unreachable,+ ),+ want: Degraded,+ },+ } {+ t.Run(name, func(t *testing.T) {+ recovery := probe.Window{}++ for _, disposition := range tc.valid {+ at := time.Now().UTC()+ recovery.Record(probe.Outcome{+ At: at,+ Credential: probe.Valid,+ Disposition: disposition,+ })+ recovery.Record(probe.Outcome{+ At: at,+ Credential: probe.Invalid,+ Disposition: probe.Rejected,+ })+ }++ got := recoveryOf(recovery, nil)+ if got != tc.want {+ t.Fatalf("recovery = %s, want %s", got, tc.want)+ }+ })+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.goindex 6840bec0..fceb86d1 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record.go@@ -190,23 +190,44 @@ func warnings(from []experiment.Warning) []string {return out}+// Phase is one measured window under the name the record calls it by.+type Phase struct {+ Name string+ Window probe.Window+}++// Phases is the ONE enumeration of the measured windows, in the order the+// engine measures them. The aggregate serializer and the raw-event serializer+// each used to carry their own list and each listed four of the five: `shift`,+// the second fault window the authorization tier needs and the verdict reads+// at three sites, was in neither. A defect established from it was written to+// a record that did not contain the observations establishing it, with an+// events hash and count that were internally consistent about the wrong set.+//+// Both serializers read this, so a phase added to the engine cannot go missing+// from one of them.+func Phases(observation experiment.Observation) []Phase {+ return []Phase{+ {"baseline", observation.Baseline},+ {"fault", observation.Fault},+ {"shift", observation.Shift},+ {"resumed", observation.Resumed},+ {"recovery", observation.Recovery},+ }+}+// windows omits a window that was never measured rather than recording it as// zero: "khaos did not get here" and "khaos measured nothing here" are// different facts, and a reader cannot tell them apart from a zero.func windows(observation experiment.Observation) map[string]Window {out := map[string]Window{}- for name, window := range map[string]probe.Window{- "baseline": observation.Baseline,- "fault": observation.Fault,- "recovery": observation.Recovery,- "resumed": observation.Resumed,- } {- if window.Len() == 0 {+ for _, phase := range Phases(observation) {+ if phase.Window.Len() == 0 {continue}- out[name] = aggregate(window)+ out[phase.Name] = aggregate(phase.Window)}return outdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.goindex 7dabc7ab..97c4c2c1 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/record_test.go@@ -61,8 +61,17 @@ func finished() experiment.Result {StartedAt: base,Duration: 30 * time.Second,Observation: experiment.Observation{- Baseline: window(0, probe.Rejected),- Fault: window(10*time.Second, probe.Accepted),+ Baseline: window(0, probe.Rejected),+ Fault: window(10*time.Second, probe.Accepted),++ // Shift and Resumed are the two windows this fixture used to+ // claim it carried and did not. Both are real measured phases --+ // shift is the second fault window the authorization tier needs,+ // and resumed is what the idleness discriminator reads -- and+ // leaving them empty is why a serializer that dropped one passed.+ Shift: window(15*time.Second, probe.Accepted),+ Shifted: true,+ Resumed: window(18*time.Second, probe.Rejected),Recovery: window(20*time.Second, probe.Rejected),FaultConfirmed: true,ConfirmMethod: "a measured observable",@@ -215,9 +224,14 @@ func TestEveryObservationReachesTheEventsFile(t *testing.T) {// Requests, not Len: Len is Paired, and one paired sample is at least// two requests. Counting pairs here would have asserted half the// observations reached the file and called that complete.- want := result.Observation.Baseline.Requests() +- result.Observation.Fault.Requests() +- result.Observation.Recovery.Requests()+ //+ // Summed over Phases rather than over a list written here, because a+ // hardcoded list of three windows is what let the shift window go+ // unnoticed in both serializers.+ want := 0+ for _, phase := range Phases(result.Observation) {+ want += phase.Window.Requests()+ }if len(events) != want {t.Fatalf("wrote %d events for %d observations", len(events), want)@@ -516,3 +530,63 @@ func inodeOf(path string) (uint64, error) {var errNoInode = errors.New("this platform does not expose an inode, so the rename cannot be distinguished from a truncation here",)++// The record is canonical and every other output is a projection of it, so a+// window the VERDICT reads must survive serialization. The shift window did+// not: both the aggregate and the raw-event serializers enumerated their+// phases separately and both listed four of the five, so a defect established+// from the second fault window was saved without the observations that+// established it -- with an events hash and count that were internally+// consistent about the wrong set.+func TestEveryMeasuredPhaseSurvivesTheRecord(t *testing.T) {+ result := finished()++ for name, window := range map[string]probe.Window{+ "baseline": result.Observation.Baseline,+ "fault": result.Observation.Fault,+ "shift": result.Observation.Shift,+ "resumed": result.Observation.Resumed,+ "recovery": result.Observation.Recovery,+ } {+ if window.Len() == 0 {+ t.Fatalf(+ "the fixture does not populate %q, so this test cannot see it dropped",+ name,+ )+ }+ }++ rec := Of("1.2.3", result, definition(t), 2, "defect")++ for _, name := range []string{+ "baseline",+ "fault",+ "shift",+ "resumed",+ "recovery",+ } {+ if _, ok := rec.Windows[name]; !ok {+ t.Errorf(+ "the record has no %q window, so a verdict resting on it cannot be audited from the record",+ name,+ )+ }+ }++ seen := map[string]int{}+ for _, event := range EventsOf(result.Observation) {+ seen[event.Window]+++ }++ for _, name := range []string{+ "baseline",+ "fault",+ "shift",+ "resumed",+ "recovery",+ } {+ if seen[name] == 0 {+ t.Errorf("events.jsonl carries no %q observation", name)+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.goindex db1ac666..c66ce763 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/record/write.go@@ -179,16 +179,8 @@ func writeAtomic(path string, body []byte) error {func EventsOf(observation experiment.Observation) []Event {out := []Event{}- for _, named := range []struct {- name string- window probe.Window- }{- {"baseline", observation.Baseline},- {"fault", observation.Fault},- {"resumed", observation.Resumed},- {"recovery", observation.Recovery},- } {- out = append(out, eventsOfWindow(named.name, named.window)...)+ for _, phase := range Phases(observation) {+ out = append(out, eventsOfWindow(phase.Name, phase.Window)...)}return outdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.goindex dd51e4cb..026ddf72 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile.go@@ -51,6 +51,8 @@ const (defaultTruncate = 4+ tempPrefix = ".khaos-holding-"+describeEmpty = "truncate %s to zero bytes in place, which is the one secret fault that is not an error at the syscall level: os.ReadFile returns no error and an empty slice, so every error-handling path in the target is bypassed"describeTruncated = "cut %s down to a few bytes in place, which leaves a silently WRONG key rather than a missing one"@@ -182,7 +184,7 @@ func (m Mutation) Apply() error {return os.Truncate(m.Path, m.keep(info.Size()))case Unreadable:- if err := copyFile(m.Path, m.Aside, info.Mode()); err != nil {+ if err := publishCopy(m.Path, m.Aside, info.Mode()); err != nil {return err}@@ -200,8 +202,13 @@ func (m Mutation) Apply() error {// hold copies the original aside and then makes the secret writable, in that// order: the copy has to happen while the file can still be read.+//+// The backup is PUBLISHED atomically, so its existence means it is complete.+// Restore cannot tell a truncated backup from a genuinely short secret by+// looking at it, and a 5-byte secret is a legitimate secret, so the guarantee+// has to be structural rather than validated.func (m Mutation) hold(info os.FileInfo) error {- if err := copyFile(m.Path, m.Aside, info.Mode()); err != nil {+ if err := publishCopy(m.Path, m.Aside, info.Mode()); err != nil {return err}@@ -251,7 +258,7 @@ func (m Mutation) Restore() error {return err}- if err := copyFile(m.Aside, m.Path, held.Mode()); err != nil {+ if err := copyInto(m.Aside, m.Path, held.Mode()); err != nil {return err}@@ -262,7 +269,75 @@ func (m Mutation) Restore() error {return os.Remove(m.Aside)}-func copyFile(from, to string, mode os.FileMode) error {+// publishCopy writes the copy to a temporary file in the destination's own+// directory, flushes it, and renames it into place. A failed copy leaves+// NOTHING at the destination, which is what lets a later Restore treat the+// backup's existence as proof that the whole original is in it.+//+// Same directory because rename is only atomic within a filesystem, and the+// flush before it because a rename that beats the data to disk publishes a+// complete-looking backup with no bytes in it.+func publishCopy(from, to string, mode os.FileMode) error {+ //nolint:gosec // the path is the fault's subject, guarded by the nonce+ source, err := os.Open(from)+ if err != nil {+ return err+ }+ defer func() { _ = source.Close() }()++ if mode.Perm() == 0 {+ mode = restoredMode+ }++ temp, err := os.CreateTemp(filepath.Dir(to), tempPrefix)+ if err != nil {+ return err+ }++ published := false++ defer func() {+ if !published {+ _ = temp.Close()+ //nolint:gosec // os.CreateTemp's own name, in the directory+ // khaos derived from the destination; no caller supplies it+ _ = os.Remove(temp.Name())+ }+ }()++ if _, err := io.Copy(temp, source); err != nil {+ return err+ }++ if err := temp.Sync(); err != nil {+ return err+ }++ if err := temp.Chmod(mode.Perm()); err != nil {+ return err+ }++ if err := temp.Close(); err != nil {+ return err+ }++ //nolint:gosec // same: both paths are khaos's own, and the rename is what+ // makes the backup's existence mean it is complete+ if err := os.Rename(temp.Name(), to); err != nil {+ return err+ }++ published = true++ return nil+}++// copyInto overwrites the destination IN PLACE, keeping its inode. The secret+// is restored this way rather than by rename because the whole reason this+// tier runs in a helper container is that unlink-and-recreate is invisible to+// a target holding the file open, and a restore that swapped the inode would+// leave that target reading the fault forever.+func copyInto(from, to string, mode os.FileMode) error {//nolint:gosec // the path is the fault's subject, guarded by the noncesource, err := os.Open(from)if err != nil {diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.goindex 12629d9e..d3618268 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/secretfile/secretfile_test.go@@ -9,6 +9,7 @@ import ("io/fs""os""path/filepath"+ "strings""testing")@@ -342,3 +343,133 @@ func TestVerifyRefusesAMalformedCredential(t *testing.T) {}}}++// The one outcome this package must never produce: khaos destroying the secret+// it was asked to preserve.+//+// The backup used to be written at its FINAL path, so a copy that failed+// partway left a short file where a complete original was expected, and a read+// error left a zero-byte one. Restore read existence as proof of a complete+// original, opened the untouched secret with O_TRUNC, and copied the stub over+// it. A short write or a full filesystem was enough, and truncating the+// original could itself free the space that let the destructive restore finish+// and report success.+//+// Restore cannot tell a truncated backup from a genuinely short secret by+// looking at it -- a 5-byte secret is a legitimate secret -- so the fix is+// structural: the backup is published atomically, and a failed hold leaves+// nothing behind for Restore to trust.+func TestAFailedHoldLeavesTheOriginalIntactAndUnrestorable(t *testing.T) {+ path, aside := seeded(t)++ // /proc/self/mem opens cleanly and fails on the first read, so the copy+ // fails with the destination already created under the old code.+ failing := Mutation{Kind: Empty, Path: "/proc/self/mem", Aside: aside}++ if err := failing.hold(modeOf(t, path)); err == nil {+ t.Fatal("copying from an unreadable source reported success")+ }++ if _, err := os.Stat(aside); err == nil {+ body, _ := os.ReadFile(aside)+ t.Fatalf(+ "a failed backup left %d bytes at the path Restore trusts as the whole original",+ len(body),+ )+ }++ // The chain that destroyed the secret: the revert that follows a failed+ // apply must refuse, not restore from whatever is lying there.+ mutation := Mutation{Kind: Empty, Path: path, Aside: aside}++ if err := mutation.Restore(); err != nil {+ t.Fatalf(+ "an untouched secret with no backup should be already-restored, got: %v",+ err,+ )+ }++ body, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("the original is unreadable after the revert: %v", err)+ }++ if string(body) != original {+ t.Fatalf(+ "khaos destroyed the secret it was preserving: now %q (%d bytes), was %q (%d bytes)",+ body,+ len(body),+ original,+ len(original),+ )+ }+}++// Publishing atomically must not leave its temporary file behind either: the+// secret's own directory is inside the target's mount, and a stray copy of a+// credential there outlives the run.+func TestAFailedHoldLeavesNoTemporaryCopyOfTheSecret(t *testing.T) {+ path, aside := seeded(t)++ failing := Mutation{Kind: Empty, Path: "/proc/self/mem", Aside: aside}+ _ = failing.hold(modeOf(t, path))++ entries, err := os.ReadDir(filepath.Dir(path))+ if err != nil {+ t.Fatalf("reading the secret's directory: %v", err)+ }++ for _, entry := range entries {+ if strings.HasPrefix(entry.Name(), tempPrefix) {+ t.Fatalf(+ "a failed hold left %q beside the secret, which is a stray copy of a credential",+ entry.Name(),+ )+ }+ }+}++// The backup a SUCCESSFUL hold publishes has to be byte-identical and carry the+// original's mode, or the restore puts back something the target cannot read.+func TestThePublishedBackupIsTheWholeOriginal(t *testing.T) {+ path, aside := seeded(t)++ mutation := Mutation{Kind: Empty, Path: path, Aside: aside}++ if err := mutation.hold(modeOf(t, path)); err != nil {+ t.Fatalf("hold: %v", err)+ }++ body, err := os.ReadFile(aside)+ if err != nil {+ t.Fatalf("the published backup is unreadable: %v", err)+ }++ if string(body) != original {+ t.Fatalf("the backup holds %q, want %q", body, original)+ }++ info, err := os.Stat(aside)+ if err != nil {+ t.Fatalf("stat the backup: %v", err)+ }++ if info.Mode().Perm() != secretPerm {+ t.Fatalf(+ "the backup is mode %#o, want the original's %#o",+ info.Mode().Perm(),+ secretPerm,+ )+ }+}++func modeOf(t *testing.T, path string) os.FileInfo {+ t.Helper()++ info, err := os.Stat(path)+ if err != nil {+ t.Fatalf("stat: %v", err)+ }++ return info+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.shindex e79150c5..87362b58 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/report.sh@@ -302,8 +302,14 @@ echo "== the codes this gate cannot reach, named rather than implied =="# internal/cli covers the mapping.# 9 UNSUPPORTED driven by scripts/dns.sh and scripts/tls.sh, against the# targets whose question genuinely does not apply.-# 70 INTERNAL a recovered panic. Reachable only from a khaos bug, so-# there is nothing honest to assert.+# 70 INTERNAL a recovered panic, recovered at the process boundary in+# cmd/khaos. This comment used to say there was "nothing+# honest to assert" about it, which was itself the dishonest+# part: no production path returned the code at all, and no+# recover() existed anywhere, so the gate described a+# mechanism khaos did not have. It is driven by a unit test+# that panics through the real boundary and asserts both the+# code and that deferred reverts still ran.## A CENSUS rather than a comment. The paragraph above was an unconditional# `pass`, which is documentation wearing an assertion's clothes: it could not@@ -329,6 +335,35 @@ elsefail "the unit census is gone, so the codes this gate cannot drive are unverified"fi+# A code with no producer is a published contract khaos does not honour, which+# is what 70 was: declared, counted by the census above, asserted by number,+# and returned from nowhere. The census cannot see that, because counting+# constants proves declaration and never reachability.+if grep -q 'code = cli.CodeInternal' cmd/khaos/main.go &&+ grep -q 'func TestAPanicBecomesTheInternalCodeWithCleanupIntact' cmd/khaos/main_test.go; then+ pass "the internal code has a producer and a test that panics through it"+else+ fail "exit 70 is declared with no production path, so the taxonomy publishes a code khaos never returns"+fi++# The refusal that used to be a panic: a non-positive interval reached+# time.NewTicker AFTER the workspace lock was taken.+if ! out="$("$BIN" reap --interval 0 --no-color 2>&1)"; then+ if grep -q "longer than zero" <<<"$out"; then+ pass "a non-positive reap interval is refused by name instead of panicking"+ else+ fail "reap --interval 0 failed without naming the cause: ${out}"+ fi+else+ fail "reap --interval 0 was accepted"+fi++if grep -q "panic" <<<"$out"; then+ fail "reap --interval 0 still panics: ${out}"+else+ pass "reap --interval 0 produces no stack trace"+fi+if [[ "$failures" -gt 0 ]]; thenecho "report: ${failures} assertion(s) failed"exit 1