Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇

fix(khaos): two ways the journal stopped describing what khaos had done

CarterPerez-dev04e4aca4bfaa
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
index 9dbe5a94..a337ccc9 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
@@ -130,6 +130,28 @@ func (c *Client) Endpoints(
return result.Container.NetworkSettings.Networks, nil
}
+// Running reports whether the container is up, which is how a restart
+// obligation knows there is still something to put back.
+func (c *Client) Running(
+ ctx context.Context,
+ containerID string,
+) (bool, error) {
+ result, err := c.api.ContainerInspect(
+ ctx,
+ containerID,
+ client.ContainerInspectOptions{},
+ )
+ if err != nil {
+ return false, fmt.Errorf(
+ errState,
+ ShortID(containerID),
+ c.classify(err),
+ )
+ }
+
+ return result.Container.State != nil && result.Container.State.Running, nil
+}
+
func (c *Client) Paused(ctx context.Context, containerID string) (bool, error) {
result, err := c.api.ContainerInspect(
ctx,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
index ea6f7ebc..5142187c 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
@@ -11,6 +11,7 @@ import (
"net"
"net/http"
"net/http/httptest"
+ "slices"
"strings"
"testing"
"time"
@@ -295,8 +296,12 @@ func TestInProcessKindsAreNotOfferedForDirectInjection(t *testing.T) {
)
}
+ // Obligations are the third category and are DERIVED too: khaos's own
+ // recovery bookkeeping, resolvable by Lookup and never offered.
+ obligations := ObligationKinds()
+
for _, kind := range Kinds() {
- wantDirect := true
+ wantDirect := !slices.Contains(obligations, kind)
for _, prefix := range prefixes {
if strings.HasPrefix(kind, prefix) {
wantDirect = false
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
index 355ed0a4..7ed8f285 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
@@ -24,6 +24,7 @@ import (
const (
KindPause = "container-pause"
+ KindRestart = "container-restart"
KindDisconnect = "network-disconnect"
errUnknownKind = "%w: %q (known kinds: %s)"
@@ -34,6 +35,8 @@ const (
confirmDisconnect = "reading the container back as detached from every network"
describePause = "freeze every process in %s so it stops answering without closing its connections"
+ describeRestart = "stop %s and start it again, so it initialises without a dependency it already had"
+ confirmRestart = "%s is stopped, read back off the daemon"
describeDisconnect = "detach %s from every network it is on, then reattach it with the same aliases, addresses and driver options"
)
@@ -100,6 +103,7 @@ type Mutator interface {
var registry = map[string]Mutator{
KindPause: pauseMutator{},
+ KindRestart: restartMutator{},
KindDisconnect: disconnectMutator{},
}
@@ -133,7 +137,7 @@ func Kinds() []string {
func DirectKinds() []string {
kinds := make([]string, 0, len(registry))
for kind, mutator := range registry {
- if inProcess(mutator) {
+ if inProcess(mutator) || internal(mutator) {
continue
}
kinds = append(kinds, kind)
@@ -155,6 +159,18 @@ func inProcess(mutator Mutator) bool {
return ok
}
+// obligation is a mutation khaos records so its OWN recovery can undo it,
+// never one an operator asks for. Declared by the same self-identifying
+// shape as perRun rather than by a list somebody has to remember.
+type obligation interface {
+ Obligation()
+}
+
+func internal(mutator Mutator) bool {
+ _, ok := mutator.(obligation)
+ return ok
+}
+
// PerRunPrefixes is every prefix belonging to a tier that lives inside a run,
// read out of the registry rather than restated.
func PerRunPrefixes() []string {
@@ -177,6 +193,23 @@ func PerRunPrefixes() []string {
return prefixes
}
+// ObligationKinds are khaos's own recovery bookkeeping, DERIVED from the
+// registry so the exclusion cannot drift from the mutators that declare it.
+// A hand-written list is how thirty-six pki- kinds once shipped as offerable.
+func ObligationKinds() []string {
+ kinds := make([]string, 0, len(registry))
+
+ for kind, mutator := range registry {
+ if internal(mutator) {
+ kinds = append(kinds, kind)
+ }
+ }
+
+ slices.Sort(kinds)
+
+ return kinds
+}
+
func IsDirect(kind string) bool {
return slices.Contains(DirectKinds(), kind)
}
@@ -444,3 +477,89 @@ func (disconnectMutator) Confirm(
}
return confirmDisconnect, true, nil
}
+
+// restartMutator carries the obligation a cold start creates between its stop
+// and its start. The pair is not a fault -- it leaves the container where it
+// was -- but only once BOTH halves complete, and a stop whose start never
+// follows leaves the operator's target down with nothing owed to anyone.
+type restartMutator struct{}
+
+func (restartMutator) Describe(_ Env, t dockerx.Target) string {
+ return fmt.Sprintf(describeRestart, t.Name)
+}
+
+// RestartPayload records the state the target was in BEFORE khaos stopped it,
+// so recovery starts only what khaos took down.
+type RestartPayload struct {
+ WasRunning bool `json:"was_running"`
+}
+
+func (restartMutator) Plan(
+ ctx context.Context,
+ env Env,
+ t dockerx.Target,
+) (json.RawMessage, error) {
+ running, err := env.Docker.Running(ctx, t.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ return journal.EncodePayload(RestartPayload{WasRunning: running})
+}
+
+func (restartMutator) Apply(
+ ctx context.Context,
+ env Env,
+ t dockerx.Target,
+ _ json.RawMessage,
+) error {
+ return env.Docker.Stop(ctx, t.ID)
+}
+
+// Revert puts back only a container khaos stopped and that is still down.
+func (restartMutator) Revert(
+ ctx context.Context,
+ env Env,
+ record journal.Record,
+) error {
+ var payload RestartPayload
+ if len(record.Payload) > 0 {
+ if err := record.DecodePayload(&payload); err != nil {
+ return err
+ }
+ }
+
+ if !payload.WasRunning {
+ return nil
+ }
+
+ running, err := env.Docker.Running(ctx, record.TargetID)
+ if err != nil {
+ return err
+ }
+
+ if running {
+ return nil
+ }
+
+ return env.Docker.Start(ctx, record.TargetID)
+}
+
+// Confirm reads back that the target is down, which is the only observable
+// half of a restart that a reclaim can act on.
+func (restartMutator) Confirm(
+ ctx context.Context,
+ env Env,
+ t dockerx.Target,
+) (string, bool, error) {
+ running, err := env.Docker.Running(ctx, t.ID)
+ if err != nil || running {
+ return "", false, err
+ }
+
+ return fmt.Sprintf(confirmRestart, t.Name), true, nil
+}
+
+// Obligation marks this as khaos's own bookkeeping: it is resolvable by
+// Lookup so a reclaim can revert it, and absent from `khaos inject`.
+func (restartMutator) Obligation() {}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry_test.go
new file mode 100644
index 00000000..d0b87016
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry_test.go
@@ -0,0 +1,30 @@
+// ©AngelaMos | 2026
+// registry_test.go
+
+package inject
+
+import (
+ "slices"
+ "testing"
+)
+
+// An obligation kind is khaos's own bookkeeping: a reclaim has to be able to
+// resolve it, and `khaos inject` must never offer it. Both halves matter --
+// unresolvable means a target stays down after a failed restart, and offered
+// means the CLI advertises a command that is not a fault anyone asked for.
+func TestAnObligationKindIsResolvableButNeverOffered(t *testing.T) {
+ if _, err := Lookup(KindRestart); err != nil {
+ t.Fatalf(
+ "%s does not resolve, so a reclaim cannot undo a restart khaos left half-done: %v",
+ KindRestart,
+ err,
+ )
+ }
+
+ if slices.Contains(DirectKinds(), KindRestart) {
+ t.Errorf(
+ "`khaos inject` offers %s, which is khaos's own recovery bookkeeping rather than a fault",
+ KindRestart,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go
index 1f5db72d..6ca08f30 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal.go
@@ -21,6 +21,11 @@ const (
PhaseApplied Phase = "applied"
PhaseReverted Phase = "reverted"
PhaseFailed Phase = "failed"
+
+ // PhaseRenewed is an expiry extension and nothing else. It is distinct
+ // from PhaseApplied because an application re-opens a mutation a reaper
+ // already reverted and a renewal must never be able to.
+ PhaseRenewed Phase = "renewed"
)
const (
@@ -100,7 +105,7 @@ func (r Record) Expired(now time.Time) bool {
func (r Record) Outstanding() bool {
return r.Phase == PhasePending || r.Phase == PhaseApplied ||
- r.Phase == PhaseFailed
+ r.Phase == PhaseFailed || r.Phase == PhaseRenewed
}
type Journal struct {
@@ -206,10 +211,17 @@ func Outstanding(path string) ([]Record, error) {
order = append(order, record.MutationID)
}
- // A completed reversion is TERMINAL: nothing re-applies a mutation ID,
- // and last-write-wins would let a stale renewal resurrect the record.
- if record.Phase == PhaseReverted {
+ // A reversion is terminal against a renewal, because last-write-wins
+ // would let a stale one resurrect the record. It is NOT terminal
+ // against an application: a reaper can revert a mutation whose Apply
+ // is still in flight, and that Apply leaves a live fault that has to
+ // stay recoverable.
+ switch record.Phase {
+ case PhaseReverted:
settled[record.MutationID] = true
+ case PhaseApplied:
+ delete(settled, record.MutationID)
+ case PhasePending, PhaseFailed, PhaseRenewed:
}
latest[record.MutationID] = record
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go
index 138c8890..d6a5f475 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/journal/journal_test.go
@@ -360,43 +360,66 @@ func TestIDsAreDistinct(t *testing.T) {
}
}
-// A completed reversion is terminal. Plain last-write-wins let a stale renewal
-// resurrect a fault the reaper had already undone, and khaos then renewed a
-// lease for a target nothing was holding.
-func TestAReversionCannotBeUndoneByALaterApplied(t *testing.T) {
- path := filepath.Join(t.TempDir(), "journal.ndjson")
-
- log, err := Open(path)
- if err != nil {
- t.Fatalf("open: %v", err)
- }
- defer func() { _ = log.Close() }()
-
- const id = "mutation-1"
-
- for _, record := range []Record{
- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhasePending},
- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},
- // The reaper wins the race and reverts.
- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseReverted},
- // A renewal that read the journal BEFORE that reversion lands late.
- {MutationID: id, Kind: "container-pause", TargetID: "abc", Phase: PhaseApplied},
+// A completed reversion is terminal against a RENEWAL and re-openable by an
+// APPLICATION, and the journal has to tell those apart: a stale renewal must
+// not resurrect a fault the reaper undid, while an Apply that completed after
+// the reaper reverted it leaves a LIVE fault that has to stay recoverable.
+func TestAReversionIsTerminalForARenewalAndNotForAnApplication(t *testing.T) {
+ for name, tc := range map[string]struct {
+ last Phase
+ want int
+ }{
+ "a stale renewal landing after the reversion": {
+ last: PhaseRenewed,
+ want: 0,
+ },
+ "an apply that completed after the reversion": {
+ last: PhaseApplied,
+ want: 1,
+ },
} {
- if err := log.Append(record); err != nil {
- t.Fatalf("append: %v", err)
- }
- }
-
- outstanding, err := Outstanding(path)
- if err != nil {
- t.Fatalf("outstanding: %v", err)
- }
-
- if len(outstanding) != 0 {
- t.Fatalf(
- "a reverted mutation came back as outstanding: %+v",
- outstanding,
- )
+ t.Run(name, func(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "journal.ndjson")
+
+ log, err := Open(path)
+ if err != nil {
+ t.Fatalf("open: %v", err)
+ }
+ defer func() { _ = log.Close() }()
+
+ const id = "mutation-1"
+
+ for _, phase := range []Phase{
+ PhasePending,
+ PhaseApplied,
+ // The reaper wins the race and reverts.
+ PhaseReverted,
+ tc.last,
+ } {
+ if err := log.Append(Record{
+ MutationID: id,
+ Kind: "container-pause",
+ TargetID: "abc",
+ Phase: phase,
+ }); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ }
+
+ outstanding, err := Outstanding(path)
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) != tc.want {
+ t.Fatalf(
+ "%d outstanding, want %d: a %s record following a reversion",
+ len(outstanding),
+ tc.want,
+ tc.last,
+ )
+ }
+ })
}
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go
index a1f48f00..81ce1aaa 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim.go
@@ -17,6 +17,38 @@ import (
// settleSuffix mirrors statedir.Paths.Settle.
const settleSuffix = ".settle"
+const (
+ // Contention on the settle lock is two short critical sections
+ // colliding, so it is worth waiting out rather than walking past.
+ settleAttempts = 25
+ settleBackoff = 20 * time.Millisecond
+)
+
+// holdSettle serialises read-then-append between a run and the reaper. The
+// returned release is always safe to call.
+//
+// A filesystem that cannot flock is a different answer from a lock somebody
+// holds: retrying it would stall every recovery there forever, so it
+// proceeds unlocked.
+func holdSettle(path string) func() {
+ for attempt := range settleAttempts {
+ settle, err := lock.Acquire(path + settleSuffix)
+ if err == nil {
+ return func() { _ = settle.Release() }
+ }
+
+ if !errors.Is(err, lock.ErrHeld) {
+ return func() {}
+ }
+
+ if attempt < settleAttempts-1 {
+ time.Sleep(settleBackoff)
+ }
+ }
+
+ return func() {}
+}
+
type Reclaimed struct {
Record journal.Record
Err error
@@ -53,9 +85,7 @@ func Reclaim(
now time.Time,
expiredOnly bool,
) Sweep {
- if settle, err := lock.Acquire(log.Path() + settleSuffix); err == nil {
- defer func() { _ = settle.Release() }()
- }
+ defer holdSettle(log.Path())()
return reclaimWith(ctx, env, log, now, expiredOnly, inject.Lookup)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go
index fcff8914..dbf62919 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/reclaim_test.go
@@ -5,10 +5,13 @@ package run
import (
"errors"
+ "path/filepath"
"strings"
"testing"
+ "time"
"github.com/CarterPerez-dev/khaos/internal/journal"
+ "github.com/CarterPerez-dev/khaos/internal/lock"
)
// A fault on another daemon is left alone AND left outstanding.
@@ -74,3 +77,42 @@ func TestASweepWillNotSettleAFaultFromAnotherDaemon(t *testing.T) {
)
}
}
+
+// The settle lock serialises read-then-append between a run and the reaper,
+// so ordinary contention has to be waited out rather than walked past.
+func TestTheSettleLockIsWaitedForRatherThanWalkedPast(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "journal.jsonl")
+
+ held, err := lock.Acquire(path + settleSuffix)
+ if err != nil {
+ t.Fatalf("seed the contention: %v", err)
+ }
+
+ entered := make(chan time.Time, 1)
+
+ go func() {
+ release := holdSettle(path)
+ entered <- time.Now()
+ release()
+ }()
+
+ // It must NOT have sailed straight through while the lock was held.
+ select {
+ case at := <-entered:
+ t.Fatalf(
+ "holdSettle returned at %s with the lock still held by someone else, so the critical section is not serialised",
+ at.Format(time.RFC3339Nano),
+ )
+ case <-time.After(60 * time.Millisecond):
+ }
+
+ if err := held.Release(); err != nil {
+ t.Fatalf("release: %v", err)
+ }
+
+ select {
+ case <-entered:
+ case <-time.After(2 * time.Second):
+ t.Fatal("holdSettle never acquired the lock after it was released")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/restart_obligation_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/restart_obligation_test.go
new file mode 100644
index 00000000..61089430
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/restart_obligation_test.go
@@ -0,0 +1,302 @@
+// ©AngelaMos | 2026
+// restart_obligation_test.go
+
+package run
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const restartImage = "alpine:3.22"
+
+func dockerOrSkip(t *testing.T) *dockerx.Client {
+ t.Helper()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ client, err := dockerx.Open(ctx)
+ if err != nil {
+ t.Skipf("no docker daemon: %v", err)
+ }
+
+ t.Cleanup(func() { _ = client.Close() })
+
+ return client
+}
+
+// A stop whose start cannot follow leaves the operator's target DOWN, and
+// nothing records that it owes a restart: Restart journalled nothing at all,
+// on the reasoning that a stop followed by a start leaves the container where
+// it was -- which holds only when both halves complete.
+//
+// The start is made to fail for real rather than simulated: the container
+// publishes a host port, the test holds that port, and the daemon refuses to
+// allocate it. Stop on an already-stopped container succeeds, so the window
+// is exact.
+func TestAStopWhoseStartFailsLeavesARestartObligation(t *testing.T) {
+ client := dockerOrSkip(t)
+
+ port := freeHighPort(t)
+ name := fmt.Sprintf("khaos-restart-obligation-%d", port)
+
+ id := seedPublished(t, name, port)
+
+ // Stop it, then take the port so the daemon cannot give it back.
+ if err := exec.Command("docker", "stop", name).Run(); err != nil {
+ t.Skipf("could not stop the fixture: %v", err)
+ }
+
+ blocker, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+ if err != nil {
+ t.Skipf("could not hold the port: %v", err)
+ }
+ defer func() { _ = blocker.Close() }()
+
+ log := tempJournal(t)
+ session := newSessionWith(
+ inject.Env{Docker: client},
+ log,
+ time.Hour,
+ inject.Lookup,
+ )
+
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ target := dockerx.Target{ID: id, Name: name}
+
+ if err := session.Restart(ctx, target); err == nil {
+ t.Fatal(
+ "the start succeeded while its port was held; the fixture did not reproduce the window",
+ )
+ }
+
+ if running(name) {
+ t.Fatal("the fixture is running, so the start did not actually fail")
+ }
+
+ outstanding, err := journal.Outstanding(log.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) == 0 {
+ t.Fatal(
+ "khaos stopped the operator's target, failed to start it again, and recorded no obligation to bring it back: a cold-started reclaim has nothing to act on",
+ )
+ }
+}
+
+// The control: a Restart that completes leaves NOTHING outstanding, or the
+// obligation above would simply be a fault khaos never clears.
+func TestACompletedRestartLeavesNoObligation(t *testing.T) {
+ client := dockerOrSkip(t)
+
+ port := freeHighPort(t)
+ name := fmt.Sprintf("khaos-restart-clean-%d", port)
+
+ id := seedPublished(t, name, port)
+
+ log := tempJournal(t)
+ session := newSessionWith(
+ inject.Env{Docker: client},
+ log,
+ time.Hour,
+ inject.Lookup,
+ )
+
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ if err := session.Restart(
+ ctx,
+ dockerx.Target{ID: id, Name: name},
+ ); err != nil {
+ t.Fatalf("restart: %v", err)
+ }
+
+ if !running(name) {
+ t.Fatal("the target did not come back up")
+ }
+
+ outstanding, err := journal.Outstanding(log.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) != 0 {
+ t.Fatalf(
+ "a completed restart left %d obligation(s) outstanding, which a reaper would act on forever",
+ len(outstanding),
+ )
+ }
+}
+
+func freeHighPort(t *testing.T) int {
+ t.Helper()
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Skipf("no free port: %v", err)
+ }
+ defer func() { _ = listener.Close() }()
+
+ return listener.Addr().(*net.TCPAddr).Port
+}
+
+func seedPublished(t *testing.T, name string, port int) string {
+ t.Helper()
+
+ _ = exec.Command("docker", "rm", "-f", name).Run()
+
+ out, err := exec.Command(
+ "docker", "run", "-d", "--name", name,
+ "-p", fmt.Sprintf("127.0.0.1:%d:7", port),
+ restartImage, "sleep", "600",
+ ).Output()
+ if err != nil {
+ t.Skipf("could not create the fixture: %v", err)
+ }
+
+ t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() })
+
+ return strings.TrimSpace(string(out))
+}
+
+func running(name string) bool {
+ out, err := exec.Command(
+ "docker", "inspect", "-f", "{{.State.Running}}", name,
+ ).Output()
+
+ return err == nil && strings.TrimSpace(string(out)) == "true"
+}
+
+// The obligation is only worth journalling if recovery acts on it: a cold
+// reclaim has to put back a target khaos took down and never brought up.
+//
+// Modelled the way it actually happens -- khaos cancelled or killed partway
+// through the restart of a RUNNING target. The fixture ignores SIGTERM, so
+// the stop takes its full timeout and the cancellation lands inside it.
+func TestAReclaimRestoresATargetLeftDownByAFailedRestart(t *testing.T) {
+ client := dockerOrSkip(t)
+
+ port := freeHighPort(t)
+ name := fmt.Sprintf("khaos-restart-reclaim-%d", port)
+
+ id := seedPublished(t, name, port)
+
+ if !running(name) {
+ t.Skip("the fixture is not running")
+ }
+
+ log := tempJournal(t)
+ env := inject.Env{Docker: client}
+ session := newSessionWith(env, log, time.Hour, inject.Lookup)
+
+ ctx, cancel := context.WithCancel(context.Background())
+
+ done := make(chan error, 1)
+ go func() {
+ done <- session.Restart(ctx, dockerx.Target{ID: id, Name: name})
+ }()
+
+ // Well inside the stop's own timeout.
+ time.Sleep(1500 * time.Millisecond)
+ cancel()
+
+ if err := <-done; err == nil {
+ t.Skip("the restart completed before the cancellation landed")
+ }
+
+ // The daemon finishes the stop it was asked for, so the target is down
+ // with khaos no longer able to bring it back.
+ deadline := time.Now().Add(30 * time.Second)
+ for running(name) && time.Now().Before(deadline) {
+ time.Sleep(200 * time.Millisecond)
+ }
+
+ if running(name) {
+ t.Skip("the target never went down, so there is no obligation to test")
+ }
+
+ sweep := reclaimWith(
+ context.Background(),
+ env,
+ log,
+ time.Now().UTC().Add(2*time.Hour),
+ true,
+ inject.Lookup,
+ )
+
+ if sweep.Failed() {
+ t.Fatalf("the reclaim reported failures: %+v", sweep.Results)
+ }
+
+ if len(sweep.Results) == 0 {
+ t.Fatal("the reclaim found nothing to act on")
+ }
+
+ if !running(name) {
+ t.Fatal(
+ "a reclaim ran over the restart obligation and the operator's target is still down",
+ )
+ }
+}
+
+// The obligation has to be DURABLE before the stop begins, not written once
+// the outcome is known: khaos killed outright partway through leaves only
+// what the journal already holds, and a stop with nothing recorded is a
+// target down with no record that it owes a start.
+func TestTheRestartObligationIsDurableBeforeTheStop(t *testing.T) {
+ client := dockerOrSkip(t)
+
+ port := freeHighPort(t)
+ name := fmt.Sprintf("khaos-restart-durable-%d", port)
+
+ id := seedPublished(t, name, port)
+
+ log := tempJournal(t)
+ session := newSessionWith(
+ inject.Env{Docker: client},
+ log,
+ time.Hour,
+ inject.Lookup,
+ )
+
+ ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+ defer cancel()
+
+ done := make(chan error, 1)
+ go func() {
+ done <- session.Restart(ctx, dockerx.Target{ID: id, Name: name})
+ }()
+
+ // The fixture ignores SIGTERM, so the stop is still running here.
+ time.Sleep(1500 * time.Millisecond)
+
+ outstanding, err := journal.Outstanding(log.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(outstanding) == 0 {
+ t.Error(
+ "the stop is in flight and the journal holds no obligation; a khaos killed right now leaves the target down with nothing to recover from",
+ )
+ }
+
+ if err := <-done; err != nil {
+ t.Fatalf("restart: %v", err)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
index d8e0beee..4cec7fdb 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session.go
@@ -14,7 +14,6 @@ import (
"github.com/CarterPerez-dev/khaos/internal/inject"
"github.com/CarterPerez-dev/khaos/internal/journal"
"github.com/CarterPerez-dev/khaos/internal/lease"
- "github.com/CarterPerez-dev/khaos/internal/lock"
)
const (
@@ -125,6 +124,11 @@ func (s *Session) Inject(
return err
}
+ // Remembered BEFORE the apply: an Apply that stops, renames and creates
+ // containers is not instant, and a record outside the renewable set
+ // cannot have its expiry extended while that work is in flight.
+ s.remember(record)
+
applyErr := mutator.Apply(ctx, s.env, target, payload)
record.At = time.Now().UTC()
@@ -149,9 +153,7 @@ func (s *Session) Inject(
func (s *Session) Renew() error {
// The same settle lock Reclaim takes: read-then-append is two operations
// and the reaper runs beside this.
- if settle, err := lock.Acquire(s.log.Path() + settleSuffix); err == nil {
- defer func() { _ = settle.Release() }()
- }
+ defer holdSettle(s.log.Path())()
live, err := journal.Outstanding(s.log.Path())
if err != nil {
@@ -221,9 +223,21 @@ func (s *Session) settle(mutationID string, failure error) {
}
}
+// remember is keyed on the mutation id: the same record arrives twice, once
+// before its apply and once with the phase that apply produced, and two
+// entries would renew a stale copy and revert the same fault twice.
func (s *Session) remember(record journal.Record) {
s.mu.Lock()
defer s.mu.Unlock()
+
+ for i := range s.mutations {
+ if s.mutations[i].MutationID == record.MutationID {
+ s.mutations[i] = record
+
+ return
+ }
+ }
+
s.mutations = append(s.mutations, record)
}
@@ -239,7 +253,9 @@ func (s *Session) renewable(
dropped := 0
for i := range s.mutations {
- if s.mutations[i].Phase != journal.PhaseApplied {
+ // PENDING is renewable too: it is a mutation still inside its Apply.
+ if s.mutations[i].Phase != journal.PhaseApplied &&
+ s.mutations[i].Phase != journal.PhasePending {
continue
}
@@ -252,7 +268,13 @@ func (s *Session) renewable(
s.mutations[i].ExpiresAt = expiry
s.mutations[i].At = now
- out = append(out, s.mutations[i])
+
+ // Emitted as a RENEWAL while the session's copy keeps its real
+ // phase, so the journal can tell holding from applying.
+ renewal := s.mutations[i]
+ renewal.Phase = journal.PhaseRenewed
+
+ out = append(out, renewal)
}
return out, dropped
@@ -318,9 +340,11 @@ func revertWith(
// In place rather than recreated, which keeps the published port: a recreate
// moves it under khaos's own probe mid-run.
//
-// It journals nothing, unlike every other mutation here, and that is correct
-// rather than an omission: a stop followed by a start leaves the container
-// where it was, so there is no state a cold-started revert would have to undo.
+// The pair leaves the container where it was, but only once BOTH halves
+// complete. A stop whose start never follows -- a failed start, a cancelled
+// context, a dead khaos -- leaves the operator's target down, so the window
+// between them carries a journalled obligation that a cold reclaim can act
+// on.
func (s *Session) Restart(
ctx context.Context,
target dockerx.Target,
@@ -329,11 +353,77 @@ func (s *Session) Restart(
return ErrNoDaemon
}
+ running, err := s.env.Docker.Running(ctx, target.ID)
+ if err != nil {
+ return err
+ }
+
+ payload, err := journal.EncodePayload(
+ inject.RestartPayload{WasRunning: running},
+ )
+ if err != nil {
+ return err
+ }
+
+ record := journal.Record{
+ MutationID: journal.NewID(),
+ RunID: s.runID,
+ LeaseID: s.lease.ID(),
+ Kind: inject.KindRestart,
+ Daemon: s.daemon(),
+ TargetID: target.ID,
+ TargetName: target.Name,
+ Payload: payload,
+ Phase: journal.PhasePending,
+ ExpiresAt: s.lease.ExpiresAt(),
+ }
+
+ if err := s.log.Append(record); err != nil {
+ return err
+ }
+
+ s.remember(record)
+
+ if err := s.settleRestart(ctx, record, target); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// settleRestart performs the stop and the start, and leaves the record
+// OUTSTANDING for anything it did not finish.
+func (s *Session) settleRestart(
+ ctx context.Context,
+ record journal.Record,
+ target dockerx.Target,
+) error {
+ record.At = time.Now().UTC()
+
if err := s.env.Docker.Stop(ctx, target.ID); err != nil {
+ // Stopped or not is unknown here, so the obligation stands and a
+ // reclaim decides by reading the container back.
+ record.Phase = journal.PhaseFailed
+ record.Detail = err.Error()
+ s.remember(record)
+ _ = s.log.Append(record)
+
return err
}
- return s.env.Docker.Start(ctx, target.ID)
+ if err := s.env.Docker.Start(ctx, target.ID); err != nil {
+ record.Phase = journal.PhaseApplied
+ record.Detail = err.Error()
+ s.remember(record)
+ _ = s.log.Append(record)
+
+ return err
+ }
+
+ record.Phase = journal.PhaseReverted
+ s.remember(record)
+
+ return s.log.Append(record)
}
func (s *Session) Confirm(
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go
index 26046a3a..65586043 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/run/session_test.go
@@ -35,6 +35,12 @@ type fake struct {
journalAtApply []journal.Record
path string
unconfirmable bool
+
+ // entered is closed when Apply begins and blocks until release is
+ // closed, so a test can act on the journal with a mutation held INSIDE
+ // its application.
+ entered chan struct{}
+ release chan struct{}
}
func (f *fake) Describe(
@@ -63,6 +69,12 @@ func (f *fake) Apply(
if f.path != "" {
f.journalAtApply, _ = journal.Outstanding(f.path)
}
+
+ if f.entered != nil {
+ close(f.entered)
+ <-f.release
+ }
+
return f.applyErr
}
@@ -462,3 +474,163 @@ func TestAnUnknownKindIsRefused(t *testing.T) {
t.Fatalf("got %v, want ErrUnknownKind", err)
}
}
+
+// A mutation settled by the reaper while its own Apply is still running
+// leaves a live fault that no later recovery can see, because Outstanding
+// treats the reversion as terminal.
+//
+// The window is held open by a channel rather than a sleep, so this fails for
+// the reason it names rather than on a slow machine.
+func TestAMutationBeingAppliedIsNotSettledByTheReaper(t *testing.T) {
+ log := tempJournal(t)
+
+ mutator := &fake{
+ entered: make(chan struct{}),
+ release: make(chan struct{}),
+ }
+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})
+
+ // A lease far shorter than the application takes.
+ session := newSessionWith(inject.Env{}, log, time.Millisecond, registry)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- session.Inject(context.Background(), fakeKind, target())
+ }()
+
+ <-mutator.entered
+
+ // The reaper runs while Apply is still inside the mutator, with a clock
+ // well past the lease.
+ sweep := reclaimWith(
+ context.Background(),
+ inject.Env{},
+ log,
+ time.Now().UTC().Add(time.Hour),
+ true,
+ registry,
+ )
+
+ close(mutator.release)
+
+ if err := <-done; err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ if len(sweep.Results) == 0 {
+ // The reaper left it alone, which is the outcome this test wants.
+ // Nothing further to check.
+ return
+ }
+
+ // It was reverted mid-apply. Then the fault MUST still be accounted for,
+ // because Apply completed and the target is carrying it.
+ live, err := journal.Outstanding(log.Path())
+ if err != nil {
+ t.Fatalf("outstanding: %v", err)
+ }
+
+ if len(live) == 0 {
+ t.Fatalf(
+ "the reaper reverted a mutation while it was still being applied, and the applied fault is now outstanding to nobody: %d record(s) reclaimed, 0 outstanding",
+ len(sweep.Results),
+ )
+ }
+}
+
+// A mutation INSIDE its own Apply has to be renewable, or its durable expiry
+// lapses while the work is in flight.
+func TestAMutationIsRenewableWhileItIsStillBeingApplied(t *testing.T) {
+ log := tempJournal(t)
+
+ mutator := &fake{
+ entered: make(chan struct{}),
+ release: make(chan struct{}),
+ }
+ registry := registryOf(map[string]inject.Mutator{fakeKind: mutator})
+
+ session := newSessionWith(inject.Env{}, log, time.Hour, registry)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- session.Inject(context.Background(), fakeKind, target())
+ }()
+
+ <-mutator.entered
+
+ before := expiryOf(t, log.Path())
+
+ if err := session.Renew(); err != nil {
+ t.Fatalf("renew: %v", err)
+ }
+
+ after := expiryOf(t, log.Path())
+
+ close(mutator.release)
+
+ if err := <-done; err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ if !after.After(before) {
+ t.Fatalf(
+ "the expiry of a mutation still inside its Apply did not move (%s -> %s), so the reaper is free to revert it mid-application",
+ before,
+ after,
+ )
+ }
+}
+
+// And a renewal has to SAY it is a renewal: while it is indistinguishable
+// from an application, a reversion can never be safely re-opened by a real
+// one.
+func TestARenewalIsJournaledAsARenewalNotAnApplication(t *testing.T) {
+ log := tempJournal(t)
+ registry := registryOf(map[string]inject.Mutator{fakeKind: &fake{}})
+
+ session := newSessionWith(inject.Env{}, log, time.Hour, registry)
+
+ if err := session.Inject(
+ context.Background(),
+ fakeKind,
+ target(),
+ ); err != nil {
+ t.Fatalf("inject: %v", err)
+ }
+
+ if err := session.Renew(); err != nil {
+ t.Fatalf("renew: %v", err)
+ }
+
+ records, _ := journal.Replay(log.Path())
+ if len(records) == 0 {
+ t.Fatal("the journal is empty")
+ }
+
+ last := records[len(records)-1]
+
+ if last.Phase != journal.PhaseRenewed {
+ t.Fatalf(
+ "a renewal was journaled as %q, want %q: while a renewal is indistinguishable from an application, a reversion can never be safely re-opened by a real one",
+ last.Phase,
+ journal.PhaseRenewed,
+ )
+ }
+}
+
+// expiryOf reads the latest expiry the journal carries for any outstanding
+// mutation.
+func expiryOf(t *testing.T, path string) time.Time {
+ t.Helper()
+
+ records, _ := journal.Replay(path)
+
+ var latest time.Time
+ for _, record := range records {
+ if record.ExpiresAt.After(latest) {
+ latest = record.ExpiresAt
+ }
+ }
+
+ return latest
+}