Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
fix(khaos): two ways the journal stopped describing what khaos had done
CarterPerez-dev04e4aca4bfaa
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.goindex 9dbe5a94..a337ccc9 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go@@ -130,6 +130,28 @@ func (c *Client) Endpoints(return result.Container.NetworkSettings.Networks, nil}+// Running reports whether the container is up, which is how a restart+// obligation knows there is still something to put back.+func (c *Client) Running(+ ctx context.Context,+ containerID string,+) (bool, error) {+ result, err := c.api.ContainerInspect(+ ctx,+ containerID,+ client.ContainerInspectOptions{},+ )+ if err != nil {+ return false, fmt.Errorf(+ errState,+ ShortID(containerID),+ c.classify(err),+ )+ }++ return result.Container.State != nil && result.Container.State.Running, nil+}+func (c *Client) Paused(ctx context.Context, containerID string) (bool, error) {result, err := c.api.ContainerInspect(ctx,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.goindex ea6f7ebc..5142187c 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go@@ -11,6 +11,7 @@ import ("net""net/http""net/http/httptest"+ "slices""strings""testing""time"@@ -295,8 +296,12 @@ func TestInProcessKindsAreNotOfferedForDirectInjection(t *testing.T) {)}+ // Obligations are the third category and are DERIVED too: khaos's own+ // recovery bookkeeping, resolvable by Lookup and never offered.+ obligations := ObligationKinds()+for _, kind := range Kinds() {- wantDirect := true+ wantDirect := !slices.Contains(obligations, kind)for _, prefix := range prefixes {if strings.HasPrefix(kind, prefix) {wantDirect = falsediff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.goindex 355ed0a4..7ed8f285 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go@@ -24,6 +24,7 @@ import (const (KindPause = "container-pause"+ KindRestart = "container-restart"KindDisconnect = "network-disconnect"errUnknownKind = "%w: %q (known kinds: %s)"@@ -34,6 +35,8 @@ const (confirmDisconnect = "reading the container back as detached from every network"describePause = "freeze every process in %s so it stops answering without closing its connections"+ describeRestart = "stop %s and start it again, so it initialises without a dependency it already had"+ confirmRestart = "%s is stopped, read back off the daemon"describeDisconnect = "detach %s from every network it is on, then reattach it with the same aliases, addresses and driver options")@@ -100,6 +103,7 @@ type Mutator interface {var registry = map[string]Mutator{KindPause: pauseMutator{},+ KindRestart: restartMutator{},KindDisconnect: disconnectMutator{},}@@ -133,7 +137,7 @@ func Kinds() []string {func DirectKinds() []string {kinds := make([]string, 0, len(registry))for kind, mutator := range registry {- if inProcess(mutator) {+ if inProcess(mutator) || internal(mutator) {continue}kinds = append(kinds, kind)@@ -155,6 +159,18 @@ func inProcess(mutator Mutator) bool {return ok}+// obligation is a mutation khaos records so its OWN recovery can undo it,+// never one an operator asks for. Declared by the same self-identifying+// shape as perRun rather than by a list somebody has to remember.+type obligation interface {+ Obligation()+}++func internal(mutator Mutator) bool {+ _, ok := mutator.(obligation)+ return ok+}+// PerRunPrefixes is every prefix belonging to a tier that lives inside a run,// read out of the registry rather than restated.func PerRunPrefixes() []string {@@ -177,6 +193,23 @@ func PerRunPrefixes() []string {return prefixes}+// ObligationKinds are khaos's own recovery bookkeeping, DERIVED from the+// registry so the exclusion cannot drift from the mutators that declare it.+// A hand-written list is how thirty-six pki- kinds once shipped as offerable.+func ObligationKinds() []string {+ kinds := make([]string, 0, len(registry))++ for kind, mutator := range registry {+ if internal(mutator) {+ kinds = append(kinds, kind)+ }+ }++ slices.Sort(kinds)++ return kinds+}+func IsDirect(kind string) bool {return slices.Contains(DirectKinds(), kind)}@@ -444,3 +477,89 @@ func (disconnectMutator) Confirm(}return confirmDisconnect, true, nil}++// restartMutator carries the obligation a cold start creates between its stop+// and its start. The pair is not a fault -- it leaves the container where it+// was -- but only once BOTH halves complete, and a stop whose start never+// follows leaves the operator's target down with nothing owed to anyone.+type restartMutator struct{}++func (restartMutator) Describe(_ Env, t dockerx.Target) string {+ return fmt.Sprintf(describeRestart, t.Name)+}++// RestartPayload records the state the target was in BEFORE khaos stopped it,+// so recovery starts only what khaos took down.+type RestartPayload struct {+ WasRunning bool `json:"was_running"`+}++func (restartMutator) Plan(+ ctx context.Context,+ env Env,+ t dockerx.Target,+) (json.RawMessage, error) {+ running, err := env.Docker.Running(ctx, t.ID)+ if err != nil {+ return nil, err+ }++ return journal.EncodePayload(RestartPayload{WasRunning: running})+}++func (restartMutator) Apply(+ ctx context.Context,+ env Env,+ t dockerx.Target,+ _ json.RawMessage,+) error {+ return env.Docker.Stop(ctx, t.ID)+}++// Revert puts back only a container khaos stopped and that is still down.+func (restartMutator) Revert(+ ctx context.Context,+ env Env,+ record journal.Record,+) error {+ var payload RestartPayload+ if len(record.Payload) > 0 {+ if err := record.DecodePayload(&payload); err != nil {+ return err+ }+ }++ if !payload.WasRunning {+ return nil+ }++ running, err := env.Docker.Running(ctx, record.TargetID)+ if err != nil {+ return err+ }++ if running {+ return nil+ }++ return env.Docker.Start(ctx, record.TargetID)+}++// Confirm reads back that the target is down, which is the only observable+// half of a restart that a reclaim can act on.+func (restartMutator) Confirm(+ ctx context.Context,+ env Env,+ t dockerx.Target,+) (string, bool, error) {+ running, err := env.Docker.Running(ctx, t.ID)+ if err != nil || running {+ return "", false, err+ }++ return fmt.Sprintf(confirmRestart, t.Name), true, nil+}++// Obligation marks this as khaos's own bookkeeping: it is resolvable by+// Lookup so a reclaim can revert it, and absent from `khaos inject`.+func (restartMutator) Obligation() {}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry_test.gonew file mode 100644index 00000000..d0b87016--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry_test.go@@ -0,0 +1,30 @@+// ©AngelaMos | 2026+// registry_test.go++package inject++import (+ "slices"+ "testing"+)++// An obligation kind is khaos's own bookkeeping: a reclaim has to be able to+// resolve it, and `khaos inject` must never offer it. Both halves matter --+// unresolvable means a target stays down after a failed restart, and offered+// means the CLI advertises a command that is not a fault anyone asked for.+func TestAnObligationKindIsResolvableButNeverOffered(t *testing.T) {+ if _, err := Lookup(KindRestart); err != nil {+ t.Fatalf(+ "%s does not resolve, so a reclaim cannot undo a restart khaos left half-done: %v",+ KindRestart,+ err,+ )+ }++ if slices.Contains(DirectKinds(), KindRestart) {+ t.Errorf(+ "`khaos inject` offers %s, which is khaos's own recovery bookkeeping rather than a fault",+ KindRestart,+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.goindex 1f5db72d..6ca08f30 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go@@ -21,6 +21,11 @@ const (PhaseApplied Phase = "applied"PhaseReverted Phase = "reverted"PhaseFailed Phase = "failed"++ // PhaseRenewed is an expiry extension and nothing else. It is distinct+ // from PhaseApplied because an application re-opens a mutation a reaper+ // already reverted and a renewal must never be able to.+ PhaseRenewed Phase = "renewed")const (@@ -100,7 +105,7 @@ func (r Record) Expired(now time.Time) bool {func (r Record) Outstanding() bool {return r.Phase == PhasePending || r.Phase == PhaseApplied ||- r.Phase == PhaseFailed+ r.Phase == PhaseFailed || r.Phase == PhaseRenewed}type Journal struct {@@ -206,10 +211,17 @@ func Outstanding(path string) ([]Record, error) {order = append(order, record.MutationID)}- // A completed reversion is TERMINAL: nothing re-applies a mutation ID,- // and last-write-wins would let a stale renewal resurrect the record.- if record.Phase == PhaseReverted {+ // A reversion is terminal against a renewal, because last-write-wins+ // would let a stale one resurrect the record. It is NOT terminal+ // against an application: a reaper can revert a mutation whose Apply+ // is still in flight, and that Apply leaves a live fault that has to+ // stay recoverable.+ switch record.Phase {+ case PhaseReverted:settled[record.MutationID] = true+ case PhaseApplied:+ delete(settled, record.MutationID)+ case PhasePending, PhaseFailed, PhaseRenewed:}latest[record.MutationID] = recorddiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.goindex 138c8890..d6a5f475 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go@@ -360,43 +360,66 @@ func TestIDsAreDistinct(t *testing.T) {}}-// A completed reversion is terminal. Plain last-write-wins let a stale renewal-// resurrect a fault the reaper had already undone, and khaos then renewed a-// lease for a target nothing was holding.-func TestAReversionCannotBeUndoneByALaterApplied(t *testing.T) {- path := filepath.Join(t.TempDir(), "journal.ndjson")-- log, err := Open(path)- if err != nil {- t.Fatalf("open: %v", err)- }- defer func() { _ = log.Close() }()-- const id = "mutation-1"-- for _, record := range []Record{- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhasePending},- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},- // The reaper wins the race and reverts.- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseReverted},- // A renewal that read the journal BEFORE that reversion lands late.- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},+// A completed reversion is terminal against a RENEWAL and re-openable by an+// APPLICATION, and the journal has to tell those apart: a stale renewal must+// not resurrect a fault the reaper undid, while an Apply that completed after+// the reaper reverted it leaves a LIVE fault that has to stay recoverable.+func TestAReversionIsTerminalForARenewalAndNotForAnApplication(t *testing.T) {+ for name, tc := range map[string]struct {+ last Phase+ want int+ }{+ "a stale renewal landing after the reversion": {+ last: PhaseRenewed,+ want: 0,+ },+ "an apply that completed after the reversion": {+ last: PhaseApplied,+ want: 1,+ },} {- if err := log.Append(record); err != nil {- t.Fatalf("append: %v", err)- }- }-- outstanding, err := Outstanding(path)- if err != nil {- t.Fatalf("outstanding: %v", err)- }-- if len(outstanding) != 0 {- t.Fatalf(- "a reverted mutation came back as outstanding: %+v",- outstanding,- )+ t.Run(name, func(t *testing.T) {+ path := filepath.Join(t.TempDir(), "journal.ndjson")++ log, err := Open(path)+ if err != nil {+ t.Fatalf("open: %v", err)+ }+ defer func() { _ = log.Close() }()++ const id = "mutation-1"++ for _, phase := range []Phase{+ PhasePending,+ PhaseApplied,+ // The reaper wins the race and reverts.+ PhaseReverted,+ tc.last,+ } {+ if err := log.Append(Record{+ MutationID: id,+ Kind: "container-pause",+ TargetID: "abc",+ Phase: phase,+ }); err != nil {+ t.Fatalf("append: %v", err)+ }+ }++ outstanding, err := Outstanding(path)+ if err != nil {+ t.Fatalf("outstanding: %v", err)+ }++ if len(outstanding) != tc.want {+ t.Fatalf(+ "%d outstanding, want %d: a %s record following a reversion",+ len(outstanding),+ tc.want,+ tc.last,+ )+ }+ })}}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.goindex a1f48f00..81ce1aaa 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go@@ -17,6 +17,38 @@ import (// settleSuffix mirrors statedir.Paths.Settle.const settleSuffix = ".settle"+const (+ // Contention on the settle lock is two short critical sections+ // colliding, so it is worth waiting out rather than walking past.+ settleAttempts = 25+ settleBackoff = 20 * time.Millisecond+)++// holdSettle serialises read-then-append between a run and the reaper. The+// returned release is always safe to call.+//+// A filesystem that cannot flock is a different answer from a lock somebody+// holds: retrying it would stall every recovery there forever, so it+// proceeds unlocked.+func holdSettle(path string) func() {+ for attempt := range settleAttempts {+ settle, err := lock.Acquire(path + settleSuffix)+ if err == nil {+ return func() { _ = settle.Release() }+ }++ if !errors.Is(err, lock.ErrHeld) {+ return func() {}+ }++ if attempt < settleAttempts-1 {+ time.Sleep(settleBackoff)+ }+ }++ return func() {}+}+type Reclaimed struct {Record journal.RecordErr error@@ -53,9 +85,7 @@ func Reclaim(now time.Time,expiredOnly bool,) Sweep {- if settle, err := lock.Acquire(log.Path() + settleSuffix); err == nil {- defer func() { _ = settle.Release() }()- }+ defer holdSettle(log.Path())()return reclaimWith(ctx, env, log, now, expiredOnly, inject.Lookup)}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.goindex fcff8914..dbf62919 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go@@ -5,10 +5,13 @@ package runimport ("errors"+ "path/filepath""strings""testing"+ "time""github.com/CarterPerez-dev/khaos/internal/journal"+ "github.com/CarterPerez-dev/khaos/internal/lock")// A fault on another daemon is left alone AND left outstanding.@@ -74,3 +77,42 @@ func TestASweepWillNotSettleAFaultFromAnotherDaemon(t *testing.T) {)}}++// The settle lock serialises read-then-append between a run and the reaper,+// so ordinary contention has to be waited out rather than walked past.+func TestTheSettleLockIsWaitedForRatherThanWalkedPast(t *testing.T) {+ path := filepath.Join(t.TempDir(), "journal.jsonl")++ held, err := lock.Acquire(path + settleSuffix)+ if err != nil {+ t.Fatalf("seed the contention: %v", err)+ }++ entered := make(chan time.Time, 1)++ go func() {+ release := holdSettle(path)+ entered <- time.Now()+ release()+ }()++ // It must NOT have sailed straight through while the lock was held.+ select {+ case at := <-entered:+ t.Fatalf(+ "holdSettle returned at %s with the lock still held by someone else, so the critical section is not serialised",+ at.Format(time.RFC3339Nano),+ )+ case <-time.After(60 * time.Millisecond):+ }++ if err := held.Release(); err != nil {+ t.Fatalf("release: %v", err)+ }++ select {+ case <-entered:+ case <-time.After(2 * time.Second):+ t.Fatal("holdSettle never acquired the lock after it was released")+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/restart_obligation_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/restart_obligation_test.gonew file mode 100644index 00000000..61089430--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/restart_obligation_test.go@@ -0,0 +1,302 @@+// ©AngelaMos | 2026+// restart_obligation_test.go++package run++import (+ "context"+ "fmt"+ "net"+ "os/exec"+ "strings"+ "testing"+ "time"++ "github.com/CarterPerez-dev/khaos/internal/dockerx"+ "github.com/CarterPerez-dev/khaos/internal/inject"+ "github.com/CarterPerez-dev/khaos/internal/journal"+)++const restartImage = "alpine:3.22"++func dockerOrSkip(t *testing.T) *dockerx.Client {+ t.Helper()++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ client, err := dockerx.Open(ctx)+ if err != nil {+ t.Skipf("no docker daemon: %v", err)+ }++ t.Cleanup(func() { _ = client.Close() })++ return client+}++// A stop whose start cannot follow leaves the operator's target DOWN, and+// nothing records that it owes a restart: Restart journalled nothing at all,+// on the reasoning that a stop followed by a start leaves the container where+// it was -- which holds only when both halves complete.+//+// The start is made to fail for real rather than simulated: the container+// publishes a host port, the test holds that port, and the daemon refuses to+// allocate it. Stop on an already-stopped container succeeds, so the window+// is exact.+func TestAStopWhoseStartFailsLeavesARestartObligation(t *testing.T) {+ client := dockerOrSkip(t)++ port := freeHighPort(t)+ name := fmt.Sprintf("khaos-restart-obligation-%d", port)++ id := seedPublished(t, name, port)++ // Stop it, then take the port so the daemon cannot give it back.+ if err := exec.Command("docker", "stop", name).Run(); err != nil {+ t.Skipf("could not stop the fixture: %v", err)+ }++ blocker, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))+ if err != nil {+ t.Skipf("could not hold the port: %v", err)+ }+ defer func() { _ = blocker.Close() }()++ log := tempJournal(t)+ session := newSessionWith(+ inject.Env{Docker: client},+ log,+ time.Hour,+ inject.Lookup,+ )++ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ target := dockerx.Target{ID: id, Name: name}++ if err := session.Restart(ctx, target); err == nil {+ t.Fatal(+ "the start succeeded while its port was held; the fixture did not reproduce the window",+ )+ }++ if running(name) {+ t.Fatal("the fixture is running, so the start did not actually fail")+ }++ outstanding, err := journal.Outstanding(log.Path())+ if err != nil {+ t.Fatalf("outstanding: %v", err)+ }++ if len(outstanding) == 0 {+ t.Fatal(+ "khaos stopped the operator's target, failed to start it again, and recorded no obligation to bring it back: a cold-started reclaim has nothing to act on",+ )+ }+}++// The control: a Restart that completes leaves NOTHING outstanding, or the+// obligation above would simply be a fault khaos never clears.+func TestACompletedRestartLeavesNoObligation(t *testing.T) {+ client := dockerOrSkip(t)++ port := freeHighPort(t)+ name := fmt.Sprintf("khaos-restart-clean-%d", port)++ id := seedPublished(t, name, port)++ log := tempJournal(t)+ session := newSessionWith(+ inject.Env{Docker: client},+ log,+ time.Hour,+ inject.Lookup,+ )++ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ if err := session.Restart(+ ctx,+ dockerx.Target{ID: id, Name: name},+ ); err != nil {+ t.Fatalf("restart: %v", err)+ }++ if !running(name) {+ t.Fatal("the target did not come back up")+ }++ outstanding, err := journal.Outstanding(log.Path())+ if err != nil {+ t.Fatalf("outstanding: %v", err)+ }++ if len(outstanding) != 0 {+ t.Fatalf(+ "a completed restart left %d obligation(s) outstanding, which a reaper would act on forever",+ len(outstanding),+ )+ }+}++func freeHighPort(t *testing.T) int {+ t.Helper()++ listener, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Skipf("no free port: %v", err)+ }+ defer func() { _ = listener.Close() }()++ return listener.Addr().(*net.TCPAddr).Port+}++func seedPublished(t *testing.T, name string, port int) string {+ t.Helper()++ _ = exec.Command("docker", "rm", "-f", name).Run()++ out, err := exec.Command(+ "docker", "run", "-d", "--name", name,+ "-p", fmt.Sprintf("127.0.0.1:%d:7", port),+ restartImage, "sleep", "600",+ ).Output()+ if err != nil {+ t.Skipf("could not create the fixture: %v", err)+ }++ t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() })++ return strings.TrimSpace(string(out))+}++func running(name string) bool {+ out, err := exec.Command(+ "docker", "inspect", "-f", "{{.State.Running}}", name,+ ).Output()++ return err == nil && strings.TrimSpace(string(out)) == "true"+}++// The obligation is only worth journalling if recovery acts on it: a cold+// reclaim has to put back a target khaos took down and never brought up.+//+// Modelled the way it actually happens -- khaos cancelled or killed partway+// through the restart of a RUNNING target. The fixture ignores SIGTERM, so+// the stop takes its full timeout and the cancellation lands inside it.+func TestAReclaimRestoresATargetLeftDownByAFailedRestart(t *testing.T) {+ client := dockerOrSkip(t)++ port := freeHighPort(t)+ name := fmt.Sprintf("khaos-restart-reclaim-%d", port)++ id := seedPublished(t, name, port)++ if !running(name) {+ t.Skip("the fixture is not running")+ }++ log := tempJournal(t)+ env := inject.Env{Docker: client}+ session := newSessionWith(env, log, time.Hour, inject.Lookup)++ ctx, cancel := context.WithCancel(context.Background())++ done := make(chan error, 1)+ go func() {+ done <- session.Restart(ctx, dockerx.Target{ID: id, Name: name})+ }()++ // Well inside the stop's own timeout.+ time.Sleep(1500 * time.Millisecond)+ cancel()++ if err := <-done; err == nil {+ t.Skip("the restart completed before the cancellation landed")+ }++ // The daemon finishes the stop it was asked for, so the target is down+ // with khaos no longer able to bring it back.+ deadline := time.Now().Add(30 * time.Second)+ for running(name) && time.Now().Before(deadline) {+ time.Sleep(200 * time.Millisecond)+ }++ if running(name) {+ t.Skip("the target never went down, so there is no obligation to test")+ }++ sweep := reclaimWith(+ context.Background(),+ env,+ log,+ time.Now().UTC().Add(2*time.Hour),+ true,+ inject.Lookup,+ )++ if sweep.Failed() {+ t.Fatalf("the reclaim reported failures: %+v", sweep.Results)+ }++ if len(sweep.Results) == 0 {+ t.Fatal("the reclaim found nothing to act on")+ }++ if !running(name) {+ t.Fatal(+ "a reclaim ran over the restart obligation and the operator's target is still down",+ )+ }+}++// The obligation has to be DURABLE before the stop begins, not written once+// the outcome is known: khaos killed outright partway through leaves only+// what the journal already holds, and a stop with nothing recorded is a+// target down with no record that it owes a start.+func TestTheRestartObligationIsDurableBeforeTheStop(t *testing.T) {+ client := dockerOrSkip(t)++ port := freeHighPort(t)+ name := fmt.Sprintf("khaos-restart-durable-%d", port)++ id := seedPublished(t, name, port)++ log := tempJournal(t)+ session := newSessionWith(+ inject.Env{Docker: client},+ log,+ time.Hour,+ inject.Lookup,+ )++ ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)+ defer cancel()++ done := make(chan error, 1)+ go func() {+ done <- session.Restart(ctx, dockerx.Target{ID: id, Name: name})+ }()++ // The fixture ignores SIGTERM, so the stop is still running here.+ time.Sleep(1500 * time.Millisecond)++ outstanding, err := journal.Outstanding(log.Path())+ if err != nil {+ t.Fatalf("outstanding: %v", err)+ }++ if len(outstanding) == 0 {+ t.Error(+ "the stop is in flight and the journal holds no obligation; a khaos killed right now leaves the target down with nothing to recover from",+ )+ }++ if err := <-done; err != nil {+ t.Fatalf("restart: %v", err)+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.goindex d8e0beee..4cec7fdb 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go@@ -14,7 +14,6 @@ import ("github.com/CarterPerez-dev/khaos/internal/inject""github.com/CarterPerez-dev/khaos/internal/journal""github.com/CarterPerez-dev/khaos/internal/lease"- "github.com/CarterPerez-dev/khaos/internal/lock")const (@@ -125,6 +124,11 @@ func (s *Session) Inject(return err}+ // Remembered BEFORE the apply: an Apply that stops, renames and creates+ // containers is not instant, and a record outside the renewable set+ // cannot have its expiry extended while that work is in flight.+ s.remember(record)+applyErr := mutator.Apply(ctx, s.env, target, payload)record.At = time.Now().UTC()@@ -149,9 +153,7 @@ func (s *Session) Inject(func (s *Session) Renew() error {// The same settle lock Reclaim takes: read-then-append is two operations// and the reaper runs beside this.- if settle, err := lock.Acquire(s.log.Path() + settleSuffix); err == nil {- defer func() { _ = settle.Release() }()- }+ defer holdSettle(s.log.Path())()live, err := journal.Outstanding(s.log.Path())if err != nil {@@ -221,9 +223,21 @@ func (s *Session) settle(mutationID string, failure error) {}}+// remember is keyed on the mutation id: the same record arrives twice, once+// before its apply and once with the phase that apply produced, and two+// entries would renew a stale copy and revert the same fault twice.func (s *Session) remember(record journal.Record) {s.mu.Lock()defer s.mu.Unlock()++ for i := range s.mutations {+ if s.mutations[i].MutationID == record.MutationID {+ s.mutations[i] = record++ return+ }+ }+s.mutations = append(s.mutations, record)}@@ -239,7 +253,9 @@ func (s *Session) renewable(dropped := 0for i := range s.mutations {- if s.mutations[i].Phase != journal.PhaseApplied {+ // PENDING is renewable too: it is a mutation still inside its Apply.+ if s.mutations[i].Phase != journal.PhaseApplied &&+ s.mutations[i].Phase != journal.PhasePending {continue}@@ -252,7 +268,13 @@ func (s *Session) renewable(s.mutations[i].ExpiresAt = expirys.mutations[i].At = now- out = append(out, s.mutations[i])++ // Emitted as a RENEWAL while the session's copy keeps its real+ // phase, so the journal can tell holding from applying.+ renewal := s.mutations[i]+ renewal.Phase = journal.PhaseRenewed++ out = append(out, renewal)}return out, dropped@@ -318,9 +340,11 @@ func revertWith(// In place rather than recreated, which keeps the published port: a recreate// moves it under khaos's own probe mid-run.//-// It journals nothing, unlike every other mutation here, and that is correct-// rather than an omission: a stop followed by a start leaves the container-// where it was, so there is no state a cold-started revert would have to undo.+// The pair leaves the container where it was, but only once BOTH halves+// complete. A stop whose start never follows -- a failed start, a cancelled+// context, a dead khaos -- leaves the operator's target down, so the window+// between them carries a journalled obligation that a cold reclaim can act+// on.func (s *Session) Restart(ctx context.Context,target dockerx.Target,@@ -329,11 +353,77 @@ func (s *Session) Restart(return ErrNoDaemon}+ running, err := s.env.Docker.Running(ctx, target.ID)+ if err != nil {+ return err+ }++ payload, err := journal.EncodePayload(+ inject.RestartPayload{WasRunning: running},+ )+ if err != nil {+ return err+ }++ record := journal.Record{+ MutationID: journal.NewID(),+ RunID: s.runID,+ LeaseID: s.lease.ID(),+ Kind: inject.KindRestart,+ Daemon: s.daemon(),+ TargetID: target.ID,+ TargetName: target.Name,+ Payload: payload,+ Phase: journal.PhasePending,+ ExpiresAt: s.lease.ExpiresAt(),+ }++ if err := s.log.Append(record); err != nil {+ return err+ }++ s.remember(record)++ if err := s.settleRestart(ctx, record, target); err != nil {+ return err+ }++ return nil+}++// settleRestart performs the stop and the start, and leaves the record+// OUTSTANDING for anything it did not finish.+func (s *Session) settleRestart(+ ctx context.Context,+ record journal.Record,+ target dockerx.Target,+) error {+ record.At = time.Now().UTC()+if err := s.env.Docker.Stop(ctx, target.ID); err != nil {+ // Stopped or not is unknown here, so the obligation stands and a+ // reclaim decides by reading the container back.+ record.Phase = journal.PhaseFailed+ record.Detail = err.Error()+ s.remember(record)+ _ = s.log.Append(record)+return err}- return s.env.Docker.Start(ctx, target.ID)+ if err := s.env.Docker.Start(ctx, target.ID); err != nil {+ record.Phase = journal.PhaseApplied+ record.Detail = err.Error()+ s.remember(record)+ _ = s.log.Append(record)++ return err+ }++ record.Phase = journal.PhaseReverted+ s.remember(record)++ return s.log.Append(record)}func (s *Session) Confirm(diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.goindex 26046a3a..65586043 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go@@ -35,6 +35,12 @@ type fake struct {journalAtApply []journal.Recordpath stringunconfirmable bool++ // entered is closed when Apply begins and blocks until release is+ // closed, so a test can act on the journal with a mutation held INSIDE+ // its application.+ entered chan struct{}+ release chan struct{}}func (f *fake) Describe(@@ -63,6 +69,12 @@ func (f *fake) Apply(if f.path != "" {f.journalAtApply, _ = journal.Outstanding(f.path)}++ if f.entered != nil {+ close(f.entered)+ <-f.release+ }+return f.applyErr}@@ -462,3 +474,163 @@ func TestAnUnknownKindIsRefused(t *testing.T) {t.Fatalf("got %v, want ErrUnknownKind", err)}}++// A mutation settled by the reaper while its own Apply is still running+// leaves a live fault that no later recovery can see, because Outstanding+// treats the reversion as terminal.+//+// The window is held open by a channel rather than a sleep, so this fails for+// the reason it names rather than on a slow machine.+func TestAMutationBeingAppliedIsNotSettledByTheReaper(t *testing.T) {+ log := tempJournal(t)++ mutator := &fake{+ entered: make(chan struct{}),+ release: make(chan struct{}),+ }+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})++ // A lease far shorter than the application takes.+ session := newSessionWith(inject.Env{}, log, time.Millisecond, registry)++ done := make(chan error, 1)+ go func() {+ done <- session.Inject(context.Background(), fakeKind, target())+ }()++ <-mutator.entered++ // The reaper runs while Apply is still inside the mutator, with a clock+ // well past the lease.+ sweep := reclaimWith(+ context.Background(),+ inject.Env{},+ log,+ time.Now().UTC().Add(time.Hour),+ true,+ registry,+ )++ close(mutator.release)++ if err := <-done; err != nil {+ t.Fatalf("inject: %v", err)+ }++ if len(sweep.Results) == 0 {+ // The reaper left it alone, which is the outcome this test wants.+ // Nothing further to check.+ return+ }++ // It was reverted mid-apply. Then the fault MUST still be accounted for,+ // because Apply completed and the target is carrying it.+ live, err := journal.Outstanding(log.Path())+ if err != nil {+ t.Fatalf("outstanding: %v", err)+ }++ if len(live) == 0 {+ t.Fatalf(+ "the reaper reverted a mutation while it was still being applied, and the applied fault is now outstanding to nobody: %d record(s) reclaimed, 0 outstanding",+ len(sweep.Results),+ )+ }+}++// A mutation INSIDE its own Apply has to be renewable, or its durable expiry+// lapses while the work is in flight.+func TestAMutationIsRenewableWhileItIsStillBeingApplied(t *testing.T) {+ log := tempJournal(t)++ mutator := &fake{+ entered: make(chan struct{}),+ release: make(chan struct{}),+ }+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})++ session := newSessionWith(inject.Env{}, log, time.Hour, registry)++ done := make(chan error, 1)+ go func() {+ done <- session.Inject(context.Background(), fakeKind, target())+ }()++ <-mutator.entered++ before := expiryOf(t, log.Path())++ if err := session.Renew(); err != nil {+ t.Fatalf("renew: %v", err)+ }++ after := expiryOf(t, log.Path())++ close(mutator.release)++ if err := <-done; err != nil {+ t.Fatalf("inject: %v", err)+ }++ if !after.After(before) {+ t.Fatalf(+ "the expiry of a mutation still inside its Apply did not move (%s -> %s), so the reaper is free to revert it mid-application",+ before,+ after,+ )+ }+}++// And a renewal has to SAY it is a renewal: while it is indistinguishable+// from an application, a reversion can never be safely re-opened by a real+// one.+func TestARenewalIsJournaledAsARenewalNotAnApplication(t *testing.T) {+ log := tempJournal(t)+ registry := registryOf(map[string]inject.Mutator{fakeKind: &fake{}})++ session := newSessionWith(inject.Env{}, log, time.Hour, registry)++ if err := session.Inject(+ context.Background(),+ fakeKind,+ target(),+ ); err != nil {+ t.Fatalf("inject: %v", err)+ }++ if err := session.Renew(); err != nil {+ t.Fatalf("renew: %v", err)+ }++ records, _ := journal.Replay(log.Path())+ if len(records) == 0 {+ t.Fatal("the journal is empty")+ }++ last := records[len(records)-1]++ if last.Phase != journal.PhaseRenewed {+ t.Fatalf(+ "a renewal was journaled as %q, want %q: while a renewal is indistinguishable from an application, a reversion can never be safely re-opened by a real one",+ last.Phase,+ journal.PhaseRenewed,+ )+ }+}++// expiryOf reads the latest expiry the journal carries for any outstanding+// mutation.+func expiryOf(t *testing.T, path string) time.Time {+ t.Helper()++ records, _ := journal.Replay(path)++ var latest time.Time+ for _, record := range records {+ if record.ExpiresAt.After(latest) {+ latest = record.ExpiresAt+ }+ }++ return latest+}