Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(khaos): the environment faults, on a tier that renames the original instead of rebuilding it

CarterPerez-dev1a289858ac99
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
index 5a22fb27..d1389473 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
@@ -56,6 +56,10 @@ const (
envSecretPath = "KHAOS_RANGE_SECRET_PATH"
envSecretMode = "KHAOS_RANGE_SECRET_MODE"
+ // Set on the fixture the environment faults target, and empty on the one
+ // the file faults target: whichever is set is where the secret comes from.
+ envSecretVar = "KHAOS_RANGE_SECRET_VAR"
+
defaultSecretPath = "/run/khaos-secrets/hmac"
envCacheTTL = "KHAOS_RANGE_CACHE_TTL"
@@ -168,6 +172,7 @@ func roleHandler(role string) (http.Handler, error) {
return newSecretGateway(
envOr(envSecretPath, defaultSecretPath),
envOr(envSecretMode, SecretUnchecked),
+ os.Getenv(envSecretVar),
).routes(), nil
case roleDB:
return newDBGateway(
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go
index e273f793..7b591d65 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/secretgateway.go
@@ -42,10 +42,29 @@ const (
type secretGateway struct {
path string
mode string
+
+ // variable names the environment variable the secret arrives in, for the
+ // fixture the environment faults target. It is read on EVERY request for
+ // the same reason the file is: a value read once at startup would make the
+ // fault invisible - except that these faults replace the container, so the
+ // fresh process reads the broken environment anyway. Reading it per
+ // request costs nothing and keeps one code path.
+ variable string
+}
+
+func newSecretGateway(path, mode, variable string) *secretGateway {
+ return &secretGateway{path: path, mode: mode, variable: variable}
}
-func newSecretGateway(path, mode string) *secretGateway {
- return &secretGateway{path: path, mode: mode}
+// secret is whichever source this fixture was configured with. The env source
+// does not distinguish unset from empty, which is the whole point of the pair
+// of faults that target it: os.Getenv returns "" for both.
+func (g *secretGateway) secret() ([]byte, error) {
+ if g.variable != "" {
+ return []byte(os.Getenv(g.variable)), nil
+ }
+
+ return os.ReadFile(g.path)
}
func (g *secretGateway) routes() *http.ServeMux {
@@ -71,7 +90,7 @@ func (g *secretGateway) routes() *http.ServeMux {
// issue hands out a credential signed with whatever secret the service holds
// right now, which is how khaos gets a valid one without reading the file.
func (g *secretGateway) issue(w http.ResponseWriter, _ *http.Request) {
- secret, err := os.ReadFile(g.path)
+ secret, err := g.secret()
if err != nil {
w.Header().Set(headerReason, reasonSecretUnreadable)
w.WriteHeader(http.StatusServiceUnavailable)
@@ -99,7 +118,7 @@ func (g *secretGateway) protected(w http.ResponseWriter, r *http.Request) {
return
}
- secret, err := os.ReadFile(g.path)
+ secret, err := g.secret()
if err != nil {
w.Header().Set(headerReason, reasonSecretUnreadable)
w.WriteHeader(http.StatusServiceUnavailable)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index cd6baa24..3aceff55 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -765,6 +765,7 @@ func rangeSupplies() []string {
experiment.ParamHealthURL,
experiment.ParamReadyURL,
experiment.ParamSecretPath,
+ experiment.ParamEnvVar,
}
}
@@ -781,6 +782,7 @@ const (
tierCred
tierPool
tierSecret
+ tierEnv
)
func tierOf(kind string) tier {
@@ -793,6 +795,8 @@ func tierOf(kind string) tier {
return tierTLS
case inject.NeedsResolver(kind):
return tierDNS
+ case inject.NeedsEnv(kind):
+ return tierEnv
case inject.NeedsSecret(kind):
return tierSecret
case inject.IsPoolFault(kind):
@@ -834,6 +838,7 @@ type tierAnswers struct {
healthURL string
readyURL string
secretPath string
+ variable string
}
func answersFor(lab rangelab.Lab, at tier) tierAnswers {
@@ -891,6 +896,14 @@ func answersFor(lab rangelab.Lab, at tier) tierAnswers {
answers.target = rangelab.SecretGatewayName
answers.secretPath = rangelab.SecretPath
+ // A SECOND fixture, whose secret is in its environment. It is not a mode
+ // of the file one because these faults replace the container, and doing
+ // that to the file fixture would take the volume's experiments with it.
+ case tierEnv:
+ answers.url = lab.EnvProtectedURL()
+ answers.target = rangelab.EnvGatewayName
+ answers.variable = rangelab.SecretVariable
+
default:
// The direct tier's URL and target are the values set above.
}
@@ -943,6 +956,7 @@ func rangeDefaults(
experiment.ParamHealthURL: tiered.healthURL,
experiment.ParamReadyURL: tiered.readyURL,
experiment.ParamSecretPath: tiered.secretPath,
+ experiment.ParamEnvVar: tiered.variable,
} {
if value != "" {
defaults[name] = value
@@ -1070,6 +1084,9 @@ func requireTier(lab rangelab.Lab, at tier) error {
case tierSecret:
return lab.RequireSecretTier()
+ case tierEnv:
+ return lab.RequireEnvTier()
+
default:
// The direct tier needs only the gateway Discover already found.
}
@@ -1235,6 +1252,11 @@ func standUpTiers(
func() (func(), error) {
return standUpSecretTier(ctx, spec, target, &env)
},
+ func() (func(), error) {
+ env.EnvPlan = spec.Env
+
+ return nil, nil
+ },
func() (func(), error) {
return standUpCredTier(
ctx,
@@ -1732,11 +1754,22 @@ func askForCredentials(
if want(experiment.ParamToken) {
issue := lab.IssueToken
- if at == tierSecret {
+ switch at {
+ case tierSecret:
if err := lab.RequireSecretTier(); err != nil {
return err
}
issue = lab.IssueSecretToken
+
+ case tierEnv:
+ if err := lab.RequireEnvTier(); err != nil {
+ return err
+ }
+ issue = lab.IssueEnvToken
+
+ default:
+ // Every other tier's credential comes from the range's own
+ // identity provider through the direct gateway.
}
token, err := issue(ctx)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
index ae6b922a..c9a74630 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -178,6 +178,7 @@ func allTiers() []tier {
tierCred,
tierPool,
tierSecret,
+ tierEnv,
}
}
@@ -197,6 +198,8 @@ func tierName(at tier) string {
return "at the service whose pool holds the credential"
case tierSecret:
return "at the file the service verifies with"
+ case tierEnv:
+ return "at the variable the service verifies with"
default:
return "direct"
}
@@ -225,6 +228,7 @@ func fullLab() rangelab.Lab {
Stateful: true,
DBGatewayPort: 30010,
SecretPort: 30011,
+ EnvPort: 30012,
}
}
@@ -792,6 +796,7 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
tierCred: "",
tierPool: lab.DBProtectedURL(),
tierSecret: lab.SecretProtectedURL(),
+ tierEnv: lab.EnvProtectedURL(),
}
seen := map[string]tier{}
@@ -855,6 +860,12 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
case tierSecret:
wantTarget = rangelab.SecretGatewayName
+ // A SECOND secret fixture, whose secret is in its environment.
+ // These faults REPLACE the container, and doing that to the file
+ // fixture would take the volume's experiments with it.
+ case tierEnv:
+ wantTarget = rangelab.EnvGatewayName
+
default:
// The direct and proxy tiers fault the dependency itself.
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
index 8a390141..057a3e0e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
@@ -42,7 +42,7 @@ const (
flagPoolMode = "pool-mode"
flagSecretMode = "secret-mode"
- mintedSecrets = 3
+ mintedSecrets = 4
rangeShort = "bring up a target stack that is deliberately breakable"
rangeLong = `The range is a small stack khaos ships so you have something to break before you
@@ -203,6 +203,11 @@ func mintRangeSecrets(lab *rangelab.Lab) error {
lab.DBPassword = secrets[1]
lab.DBAppPassword = secrets[2]
+ // The HMAC secret the environment fixture verifies with. The file fixture
+ // gets its own from the range IMAGE, because a volume seeded from the
+ // image is what lets it be mounted read-only.
+ lab.SecretValue = secrets[3]
+
return nil
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/recreate.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/recreate.go
new file mode 100644
index 00000000..37f91e85
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/recreate.go
@@ -0,0 +1,349 @@
+// ©AngelaMos | 2026
+// recreate.go
+
+package dockerx
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "maps"
+ "net/netip"
+ "slices"
+ "strings"
+
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/network"
+ "github.com/moby/moby/client"
+)
+
+const (
+ // HeldSuffix is where the ORIGINAL container goes while a replacement runs
+ // under its name. khaos never destroys it: a recreate from an inspection
+ // loses the anonymous volume with its data stranded, every prior log line,
+ // and the container id that invariant 8 re-verifies against. A rename
+ // keeps all three and makes the revert a rename back.
+ HeldSuffix = "-khaos-held"
+
+ // HeldVarPrefix carries a variable's original value inside the container
+ // while khaos is pretending it is absent, so the value never reaches
+ // khaos's journal or its state directory. Same rule as the secret tier's
+ // aside copy.
+ HeldVarPrefix = "KHAOS_HELD_"
+
+ envAssign = "="
+
+ errCapture = "read %s back before replacing it: %w"
+ errNoConfig = "%w: the daemon returned no configuration for %s, so khaos cannot build a replacement for it"
+ errRename = "rename %s to %s: %w"
+ errRecreate = "create the replacement for %s: %w"
+ errStartReplaced = "start the replacement for %s: %w"
+ errStopContainer = "stop %s before replacing it: %w"
+ errStartContainer = "start %s: %w"
+ errRemoveContainer = "remove %s: %w"
+
+ errImageEnv = "read the environment %s bakes into its own image: %w"
+ errNoVariable = "%w: %s does not carry %s in its environment, so there is nothing for khaos to take away"
+ errImageDefines = "%w: %s is defined by the IMAGE %s as well as by the container, so a replacement created without it gets the image's value back and khaos would report a fault the daemon had undone. Shadow it with the empty-value fault instead, or target a service that does not bake it in"
+)
+
+var ErrNoContainerConfig = errors.New(
+ "the daemon returned no configuration to rebuild from",
+)
+
+var ErrVariableAbsent = errors.New(
+ "that variable is not in the target's environment",
+)
+
+var ErrImageDefinesVariable = errors.New(
+ "the image defines that variable, so removing the container's copy changes nothing",
+)
+
+// Spec is everything needed to recreate one container. It is captured from the
+// daemon rather than assembled, so nothing khaos forgot to copy can go missing.
+type Spec struct {
+ Name string
+ Config *container.Config
+ HostConfig *container.HostConfig
+ Networks map[string]*network.EndpointSettings
+}
+
+// Capture reads a container back. The per-network EndpointSettings travel whole
+// and are cleaned of the daemon's own identifiers: reconnecting with only a
+// network NAME silently drops the static address, the MAC and every alias, and
+// on a compose project the service name lives in the aliases, so every peer
+// addressing it by name would get NXDOMAIN forever.
+func (c *Client) Capture(
+ ctx context.Context,
+ containerID string,
+) (Spec, error) {
+ result, err := c.api.ContainerInspect(
+ ctx,
+ containerID,
+ client.ContainerInspectOptions{},
+ )
+ if err != nil {
+ return Spec{}, fmt.Errorf(
+ errCapture,
+ ShortID(containerID),
+ c.classify(err),
+ )
+ }
+
+ raw := result.Container
+
+ if raw.Config == nil || raw.HostConfig == nil {
+ return Spec{}, fmt.Errorf(
+ errNoConfig,
+ ErrNoContainerConfig,
+ ShortID(containerID),
+ )
+ }
+
+ spec := Spec{
+ Name: strings.TrimPrefix(raw.Name, namePrefix),
+ Config: raw.Config,
+ HostConfig: raw.HostConfig,
+ Networks: map[string]*network.EndpointSettings{},
+ }
+
+ if raw.NetworkSettings == nil {
+ return spec, nil
+ }
+
+ for name, settings := range raw.NetworkSettings.Networks {
+ if settings == nil {
+ continue
+ }
+
+ // Deep-copied through the API's own helper, then cleaned of the
+ // daemon's operational fields. IPAMConfig, Aliases and MacAddress are
+ // kept: those are what the replacement has to be created WITH, and
+ // reconnecting with only a network name drops all three.
+ copied := settings.Copy()
+ copied.EndpointID = ""
+ copied.NetworkID = ""
+ copied.Gateway = netip.Addr{}
+ copied.IPAddress = netip.Addr{}
+ copied.IPv6Gateway = netip.Addr{}
+ copied.GlobalIPv6Address = netip.Addr{}
+ copied.IPPrefixLen = 0
+ copied.GlobalIPv6PrefixLen = 0
+
+ // The short id is in here and it is about to change.
+ copied.DNSNames = nil
+
+ spec.Networks[name] = copied
+ }
+
+ return spec, nil
+}
+
+// Rename is how the original is kept. The replacement takes the name, so every
+// peer addressing the service by name reaches the replacement.
+func (c *Client) Rename(ctx context.Context, containerID, name string) error {
+ if _, err := c.api.ContainerRename(
+ ctx,
+ containerID,
+ client.ContainerRenameOptions{NewName: name},
+ ); err != nil {
+ return fmt.Errorf(
+ errRename,
+ ShortID(containerID),
+ name,
+ c.classify(err),
+ )
+ }
+
+ return nil
+}
+
+// Recreate creates and starts a container from a captured spec with a replaced
+// environment, and returns its id. The caller has already renamed the original
+// aside, so this is additive: nothing has been destroyed when it fails.
+func (c *Client) Recreate(
+ ctx context.Context,
+ spec Spec,
+ environ []string,
+) (string, error) {
+ config := *spec.Config
+ config.Env = environ
+
+ created, err := c.api.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Name: spec.Name,
+ Config: &config,
+ HostConfig: spec.HostConfig,
+ NetworkingConfig: &network.NetworkingConfig{
+ EndpointsConfig: spec.Networks,
+ },
+ })
+ if err != nil {
+ return "", fmt.Errorf(errRecreate, spec.Name, c.classify(err))
+ }
+
+ if _, err := c.api.ContainerStart(
+ ctx,
+ created.ID,
+ client.ContainerStartOptions{},
+ ); err != nil {
+ return created.ID, fmt.Errorf(
+ errStartReplaced,
+ spec.Name,
+ c.classify(err),
+ )
+ }
+
+ return created.ID, nil
+}
+
+// ImageEnv is what the IMAGE bakes in, which is not the same question as what
+// the container carries. The daemon merges the image's environment into any key
+// a create request omits, so a variable defined in both places cannot be taken
+// away by omission.
+func (c *Client) ImageEnv(ctx context.Context, image string) ([]string, error) {
+ result, err := c.api.ImageInspect(ctx, image)
+ if err != nil {
+ return nil, fmt.Errorf(errImageEnv, image, c.classify(err))
+ }
+
+ if result.Config == nil {
+ return nil, nil
+ }
+
+ return result.Config.Env, nil
+}
+
+// EnvValue reads one variable out of a captured environment, taking the LAST
+// occurrence. Docker deduplicates on the whole `KEY=VALUE` string rather than
+// on the key, so a shadowed variable appears twice and the last one is what the
+// process sees - khaos has to read it the same way or it mis-reports its own
+// fault.
+func EnvValue(environ []string, name string) (string, bool) {
+ value := ""
+ found := false
+
+ for _, pair := range environ {
+ key, rest, ok := strings.Cut(pair, envAssign)
+ if !ok || key != name {
+ continue
+ }
+
+ value = rest
+ found = true
+ }
+
+ return value, found
+}
+
+// WithoutVariable removes every occurrence of a key, which is what an unset
+// needs: leaving one behind would shadow the removal.
+func WithoutVariable(environ []string, name string) []string {
+ out := make([]string, 0, len(environ))
+
+ for _, pair := range environ {
+ if key, _, ok := strings.Cut(pair, envAssign); ok && key == name {
+ continue
+ }
+
+ out = append(out, pair)
+ }
+
+ return out
+}
+
+// ShadowedEmpty appends `NAME=` rather than replacing the original entry, which
+// is what makes the fault reversible without khaos holding the value: the
+// revert removes the LAST occurrence and the original is still underneath.
+func ShadowedEmpty(environ []string, name string) []string {
+ return append(slices.Clone(environ), name+envAssign)
+}
+
+// HeldFor is where an unset keeps the original value: inside the replacement's
+// own environment, under a name the target does not read.
+func HeldFor(name string) string {
+ return HeldVarPrefix + name
+}
+
+// ImageDefines reports whether the image bakes this variable in, which is the
+// one precondition `env_unset` cannot work around.
+func ImageDefines(imageEnv []string, name string) bool {
+ _, found := EnvValue(imageEnv, name)
+
+ return found
+}
+
+// HeldName is the name the original container is renamed to.
+func HeldName(name string) string {
+ return name + HeldSuffix
+}
+
+// Variables is every key a captured environment carries, for a refusal that
+// names what IS there rather than only what is not.
+func Variables(environ []string) []string {
+ seen := map[string]bool{}
+
+ for _, pair := range environ {
+ if key, _, ok := strings.Cut(pair, envAssign); ok {
+ seen[key] = true
+ }
+ }
+
+ return slices.Sorted(maps.Keys(seen))
+}
+
+// Stop stops a container before it is renamed aside, because two containers
+// cannot both hold a network's static address and a service's aliases. A
+// container that is already stopped is success.
+func (c *Client) Stop(ctx context.Context, containerID string) error {
+ if _, err := c.api.ContainerStop(
+ ctx,
+ containerID,
+ client.ContainerStopOptions{},
+ ); err != nil {
+ return fmt.Errorf(
+ errStopContainer,
+ ShortID(containerID),
+ c.classify(err),
+ )
+ }
+
+ return nil
+}
+
+// Start brings the original back after a revert has renamed it into place.
+func (c *Client) Start(ctx context.Context, containerID string) error {
+ if _, err := c.api.ContainerStart(
+ ctx,
+ containerID,
+ client.ContainerStartOptions{},
+ ); err != nil {
+ return fmt.Errorf(
+ errStartContainer,
+ ShortID(containerID),
+ c.classify(err),
+ )
+ }
+
+ return nil
+}
+
+// Remove removes a container khaos CREATED, and only ever the replacement it
+// made itself. A container that is already gone is success, because a revert
+// runs again after a reclaim.
+func (c *Client) Remove(ctx context.Context, containerID string) error {
+ _, err := c.api.ContainerRemove(
+ ctx,
+ containerID,
+ client.ContainerRemoveOptions{Force: true},
+ )
+ if err == nil {
+ return nil
+ }
+
+ classified := c.classify(err)
+ if errors.Is(classified, ErrContainerGone) {
+ return nil
+ }
+
+ return fmt.Errorf(errRemoveContainer, ShortID(containerID), classified)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/recreate_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/recreate_test.go
new file mode 100644
index 00000000..be81a4ae
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/recreate_test.go
@@ -0,0 +1,139 @@
+// ©AngelaMos | 2026
+// recreate_test.go
+
+package dockerx
+
+import (
+ "slices"
+ "testing"
+)
+
+// TestEnvValueTakesTheLastOccurrence is the control for the one thing the
+// environment faults mis-report if read naively. Docker deduplicates on the
+// whole `KEY=VALUE` string rather than on the key, so a shadowed variable
+// appears TWICE in an inspect and the process sees the last one. Measured on
+// khaos's own range: the replacement's environment carried the original value
+// and an empty shadow, in that order.
+func TestEnvValueTakesTheLastOccurrence(t *testing.T) {
+ environ := []string{
+ "PATH=/usr/bin",
+ "KHAOS_SECRET=original",
+ "OTHER=keep",
+ "KHAOS_SECRET=",
+ }
+
+ value, found := EnvValue(environ, "KHAOS_SECRET")
+ if !found {
+ t.Fatal("the variable was not found at all")
+ }
+
+ if value != "" {
+ t.Fatalf(
+ "EnvValue = %q, want the LAST occurrence, which is empty",
+ value,
+ )
+ }
+
+ if _, found := EnvValue(environ, "ABSENT"); found {
+ t.Error("a variable that is not there was reported as present")
+ }
+}
+
+// TestWithoutVariableRemovesEVERYOccurrence matters for the same reason: one
+// left behind would shadow the removal and khaos would report a fault it had
+// not injected.
+func TestWithoutVariableRemovesEveryOccurrence(t *testing.T) {
+ environ := []string{
+ "KHAOS_SECRET=original",
+ "PATH=/usr/bin",
+ "KHAOS_SECRET=",
+ "KHAOS_SECRET=again",
+ }
+
+ out := WithoutVariable(environ, "KHAOS_SECRET")
+
+ if _, found := EnvValue(out, "KHAOS_SECRET"); found {
+ t.Fatalf("the variable survived removal: %v", out)
+ }
+
+ if !slices.Contains(out, "PATH=/usr/bin") {
+ t.Error("removal took another variable with it")
+ }
+}
+
+// TestShadowedEmptyAppendsRatherThanReplacing is what makes the empty fault
+// reversible without khaos holding the value: the original is still underneath,
+// so the revert only has to drop what was appended.
+func TestShadowedEmptyAppendsRatherThanReplacing(t *testing.T) {
+ environ := []string{"KHAOS_SECRET=original", "PATH=/usr/bin"}
+
+ out := ShadowedEmpty(environ, "KHAOS_SECRET")
+
+ if len(out) != len(environ)+1 {
+ t.Fatalf("len = %d, want %d", len(out), len(environ)+1)
+ }
+
+ if out[len(out)-1] != "KHAOS_SECRET=" {
+ t.Fatalf("the shadow is %q, want an empty assignment", out[len(out)-1])
+ }
+
+ if !slices.Contains(out, "KHAOS_SECRET=original") {
+ t.Error(
+ "the original was replaced rather than shadowed, so the revert would have nothing to restore from",
+ )
+ }
+
+ // And the process reads the shadow.
+ if value, _ := EnvValue(out, "KHAOS_SECRET"); value != "" {
+ t.Errorf("the shadow does not win: EnvValue = %q", value)
+ }
+
+ // The input is not mutated, or a second fault would compound the first.
+ if len(environ) != 2 {
+ t.Errorf("ShadowedEmpty mutated its input: %v", environ)
+ }
+}
+
+// TestImageDefinesIsTheOnePreconditionAnUnsetCannotWorkAround pins the measured
+// reason `env_unset` refuses: the daemon merges the image's environment into
+// any key a create request omits, so a variable defined in both places cannot
+// be taken away by omission and khaos would report a fault the daemon had
+// undone for it.
+func TestImageDefinesIsTheOnePrecondition(t *testing.T) {
+ imageEnv := []string{"PATH=/usr/bin", "APP_SECRET=baked-in"}
+
+ if !ImageDefines(imageEnv, "APP_SECRET") {
+ t.Error("a variable the image defines was not recognised")
+ }
+
+ if ImageDefines(imageEnv, "CONTAINER_ONLY") {
+ t.Error("a variable only the container carries was blamed on the image")
+ }
+}
+
+func TestHeldNamesAreDerivedRatherThanInvented(t *testing.T) {
+ if got := HeldName("shop-api"); got != "shop-api"+HeldSuffix {
+ t.Errorf("HeldName = %q", got)
+ }
+
+ if got := HeldFor("APP_SECRET"); got != HeldVarPrefix+"APP_SECRET" {
+ t.Errorf("HeldFor = %q", got)
+ }
+}
+
+func TestVariablesNamesWhatIsThere(t *testing.T) {
+ environ := []string{
+ "PATH=/usr/bin",
+ "B=2",
+ "A=1",
+ "B=3",
+ "malformed-without-equals",
+ }
+
+ got := Variables(environ)
+ want := []string{"A", "B", "PATH"}
+
+ if !slices.Equal(got, want) {
+ t.Fatalf("Variables = %v, want %v", got, want)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
index 4f54427d..cdfd7d81 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
@@ -75,6 +75,10 @@ type Shape struct {
// only place that has the target's own inspection.
Secret *inject.SecretPlan
+ // The environment tier's plan, which is the only fault class that replaces
+ // the container.
+ Env *inject.EnvPlan
+
// Separate fields because the target addresses khaos by a name khaos cannot
// see from its own listener, so neither can be derived from the other.
AuthorityBase string
@@ -285,6 +289,7 @@ func Build(
DNSHost: shape.DNSHost,
Cred: shape.Cred,
Secret: shape.Secret,
+ Env: shape.Env,
IdleFor: shape.IdleFor,
Act: shape.Act,
ActDetail: shape.ActDetail,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/env.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/env.go
new file mode 100644
index 00000000..ba0f9465
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/env.go
@@ -0,0 +1,141 @@
+// ©AngelaMos | 2026
+// env.go
+
+package experiment
+
+import (
+ "context"
+ "crypto/rand"
+ "fmt"
+
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+ "github.com/CarterPerez-dev/khaos/internal/secretfile"
+)
+
+const (
+ ParamEnvVar = "variable"
+
+ NameEnvUnset = "credential.env_unset"
+ NameEnvEmpty = "credential.env_empty"
+
+ usageEnvVar = "the environment variable holding the secret, which khaos takes away by replacing the container; the original is kept inside the replacement so khaos never writes it anywhere"
+
+ usageTargetEnv = "the consenting container whose environment you want to break; khaos keeps the original container aside and renames it back on the way out"
+
+ summaryEnvUnset = "replace a service with one that does not carry its secret at all, which is what a deploy with a missing variable does"
+
+ summaryEnvEmpty = "replace a service with one whose secret is present and EMPTY, which os.Getenv cannot distinguish from absent"
+
+ conditionEnvUnset = "%s being restarted without %s in its environment"
+
+ conditionEnvEmpty = "%s being restarted with %s present and empty"
+
+ responseEnvUnset = "refusing to serve at all rather than starting healthy and verifying credentials against a secret it never read"
+
+ responseEnvEmpty = "refusing every credential, because an empty secret is a secret equal to the empty string and HMAC computes happily with one"
+
+ defectEnvUnset = "replacing it with a container that does not carry that variable"
+
+ defectEnvEmpty = "replacing it with a container whose variable is present and empty"
+
+ caveatEnvHeld = "the replacement kept verifying credentials; khaos replaced the container, so this is the service's behaviour on a FRESH start with that environment rather than anything it did while running"
+)
+
+// envPolarity matches the secret family's: a service that verifies a credential
+// against a secret it could not have read is not verifying anything.
+var envPolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a service that starts healthy and keeps accepting credentials without the secret it needs to check them has stopped being a check",
+}
+
+var envParams = []Param{
+ {Name: ParamTarget, Usage: usageTargetEnv, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamToken, Usage: usageTokenSecret, Required: true},
+ {Name: ParamEnvVar, Usage: usageEnvVar, Required: true},
+}
+
+func buildEnv(
+ kind inject.EnvKind,
+ condition, defect string,
+) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ if err := inject.ValidEnvKind(kind); err != nil {
+ return Shape{}, err
+ }
+
+ target := params.Get(ParamTarget)
+ url := params.Get(ParamURL)
+ variable := params.Get(ParamEnvVar)
+
+ return Shape{
+ Condition: fmt.Sprintf(condition, target, variable),
+ Target: target,
+ Probes: probe.Pair{
+ Valid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeValidName,
+ URL: url,
+ Credential: probe.Valid,
+ Mint: memoised(params.Get(ParamToken)),
+ }),
+ Invalid: probe.NewHTTP(probe.HTTPOptions{
+ Name: probeEmptyKeyName,
+ Describe: fmt.Sprintf(describeEmptyKey, url),
+ URL: url,
+ Credential: probe.Invalid,
+ Mint: func(context.Context) (string, error) {
+ return secretfile.Token(nil, rand.Text()), nil
+ },
+ }),
+ },
+ Env: &inject.EnvPlan{Kind: kind, Variable: variable},
+
+ Defect: defect,
+ HeldCaveat: caveatEnvHeld,
+
+ // A service that cannot read its secret answers 503 rather than
+ // refusing a credential, and for this control that IS failing
+ // closed.
+ ClosedWhen: []probe.Disposition{probe.Unavailable},
+ }, nil
+ }
+}
+
+func init() {
+ catalogAdd(Definition{
+ Name: NameEnvUnset,
+ Family: FamilyCredential,
+ Summary: summaryEnvUnset,
+ Hypothesis: Hypothesis{
+ Condition: "a service being restarted without the variable holding its secret",
+ Response: responseEnvUnset,
+ },
+ Polarity: envPolarity,
+ FaultKind: inject.EnvFaultKind(inject.EnvUnset),
+ Params: envParams,
+ Build: buildEnv(
+ inject.EnvUnset,
+ conditionEnvUnset,
+ defectEnvUnset,
+ ),
+ })
+
+ catalogAdd(Definition{
+ Name: NameEnvEmpty,
+ Family: FamilyCredential,
+ Summary: summaryEnvEmpty,
+ Hypothesis: Hypothesis{
+ Condition: "a service being restarted with that variable present and empty",
+ Response: responseEnvEmpty,
+ },
+ Polarity: envPolarity,
+ FaultKind: inject.EnvFaultKind(inject.EnvEmpty),
+ Params: envParams,
+ Build: buildEnv(
+ inject.EnvEmpty,
+ conditionEnvEmpty,
+ defectEnvEmpty,
+ ),
+ })
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
index 14861eab..7e3a097a 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -135,6 +135,10 @@ type Experiment struct {
// container that mounts only the mount containing it.
Secret *inject.SecretPlan
+ // Env replaces the container, because a container's environment is fixed
+ // at creation and the update endpoint silently ignores it.
+ Env *inject.EnvPlan
+
Defect string
HeldCaveat string
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/envfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/envfault.go
new file mode 100644
index 00000000..765bf5f5
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/envfault.go
@@ -0,0 +1,398 @@
+// ©AngelaMos | 2026
+// envfault.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+type EnvKind string
+
+const (
+ // EnvUnset takes the variable away. Deliverable only where the IMAGE does
+ // not also define it, because the daemon merges the image's environment
+ // into any key a create request omits.
+ EnvUnset EnvKind = "unset"
+
+ // EnvEmpty shadows it with `NAME=`, which is the more interesting of the
+ // two: os.Getenv cannot tell it from unset and os.LookupEnv can, so a
+ // required-field check written with the wrong one does not fire.
+ EnvEmpty EnvKind = "empty"
+
+ KindEnvPrefix = "env-"
+
+ errNoEnvPlan = "%w: %s needs to be told which variable to take away"
+
+ errUnknownEnvKind = "%w: %q (known environment faults: %s)"
+
+ describeEnvUnset = "replace %s with a container that does not carry %s at all, keeping the original aside so the revert is a rename rather than a rebuild"
+
+ describeEnvEmpty = "replace %s with a container whose %s is present and EMPTY, which os.Getenv cannot tell from absent"
+
+ confirmEnvUnset = "the replacement's own environment no longer carries %s, and the original is held as %s"
+
+ errNoVariableThere = "%w: %s does not carry %s in its environment, so there is nothing for khaos to take away. It carries: %s"
+
+ errImageWouldRestore = "%w: %s is defined by the IMAGE %s as well as by the container, so a replacement created without it gets the image's value back and khaos would report a fault the daemon had undone. Use the empty-value fault, which shadows it instead, or target a service that does not bake it in"
+
+ confirmEnvEmpty = "the replacement's own environment carries %s as an empty value, read from the LAST occurrence the way the process reads it"
+)
+
+var ErrNoEnvPlan = errors.New(
+ "this mutation has no environment variable to act on",
+)
+
+var ErrUnknownEnvKind = errors.New(
+ "no environment fault is registered under that name",
+)
+
+func EnvKinds() []string {
+ return []string{string(EnvEmpty), string(EnvUnset)}
+}
+
+func EnvFaultKind(kind EnvKind) string {
+ return KindEnvPrefix + string(kind)
+}
+
+// EnvPlan is what the revert needs, and it carries no value: the original is
+// held inside the replacement's own environment under a name the target does
+// not read, so the journal records names only.
+type EnvPlan struct {
+ Kind EnvKind `json:"kind"`
+ Variable string `json:"variable"`
+
+ // Held is the name the ORIGINAL container was renamed to. The revert
+ // removes the replacement and renames this back, which is why neither the
+ // anonymous volume, the logs nor the original id are lost.
+ Held string `json:"held"`
+
+ // OriginalID is the container khaos resolved in the plan, and ReplacedID
+ // is what it created. Invariant 8 re-verifies by immutable id, and this
+ // fault class mints a new one, so the handover is recorded rather than
+ // left for the rollback to guess.
+ OriginalID string `json:"original_id"`
+ ReplacedID string `json:"replaced_id,omitempty"`
+}
+
+// envMutator is the only tier that REPLACES the target. Everything else mutates
+// a dependency, a file or a credential; a container's environment is fixed at
+// creation and `ContainerUpdate` accepts an Env field, returns HTTP 200 with no
+// warning, and changes nothing.
+type envMutator struct {
+ kind EnvKind
+}
+
+func init() {
+ for _, name := range EnvKinds() {
+ kind := EnvKind(name)
+ registry[EnvFaultKind(kind)] = envMutator{kind: kind}
+ }
+}
+
+func (m envMutator) Describe(env Env, target dockerx.Target) string {
+ variable := env.EnvPlan.Variable
+ if variable == "" {
+ variable = string(m.kind)
+ }
+
+ if m.kind == EnvEmpty {
+ return fmt.Sprintf(describeEnvEmpty, target.Name, variable)
+ }
+
+ return fmt.Sprintf(describeEnvUnset, target.Name, variable)
+}
+
+// Plan refuses both ways this fault can be a no-op: a variable the target does
+// not carry, and one the IMAGE defines as well, where a replacement created
+// without it gets the image's value back.
+func (m envMutator) Plan(
+ ctx context.Context,
+ env Env,
+ target dockerx.Target,
+) (json.RawMessage, error) {
+ if env.EnvPlan == nil || env.EnvPlan.Variable == "" {
+ return nil, fmt.Errorf(
+ errNoEnvPlan,
+ ErrNoEnvPlan,
+ EnvFaultKind(m.kind),
+ )
+ }
+
+ plan := env.EnvPlan
+
+ if plan.Kind != m.kind {
+ return nil, fmt.Errorf(
+ errWrongTrapKind,
+ ErrTrapMismatch,
+ plan.Kind,
+ m.kind,
+ )
+ }
+
+ spec, err := env.Docker.Capture(ctx, target.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, carried := dockerx.EnvValue(
+ spec.Config.Env,
+ plan.Variable,
+ ); !carried {
+ return nil, fmt.Errorf(
+ errNoVariableThere,
+ dockerx.ErrVariableAbsent,
+ target.Name,
+ plan.Variable,
+ strings.Join(dockerx.Variables(spec.Config.Env), ", "),
+ )
+ }
+
+ if m.kind == EnvUnset {
+ imageEnv, err := env.Docker.ImageEnv(ctx, spec.Config.Image)
+ if err != nil {
+ return nil, err
+ }
+
+ if dockerx.ImageDefines(imageEnv, plan.Variable) {
+ return nil, fmt.Errorf(
+ errImageWouldRestore,
+ dockerx.ErrImageDefinesVariable,
+ plan.Variable,
+ spec.Config.Image,
+ )
+ }
+ }
+
+ plan.Held = dockerx.HeldName(spec.Name)
+ plan.OriginalID = target.ID
+
+ return journal.EncodePayload(*plan)
+}
+
+// Apply renames the original aside and creates the replacement under its name,
+// in that order, so nothing is destroyed: a failed create leaves the original
+// present and the revert renames it back.
+func (m envMutator) Apply(
+ ctx context.Context,
+ env Env,
+ target dockerx.Target,
+ raw json.RawMessage,
+) error {
+ var plan EnvPlan
+ if err := json.Unmarshal(raw, &plan); err != nil {
+ return err
+ }
+
+ spec, err := env.Docker.Capture(ctx, target.ID)
+ if err != nil {
+ return err
+ }
+
+ environ := m.replacement(spec.Config.Env, plan.Variable)
+
+ // Stopped before the rename, because two containers cannot both hold the
+ // network's static address and the service's aliases.
+ if err := env.Docker.Stop(ctx, target.ID); err != nil {
+ return err
+ }
+
+ if err := env.Docker.Rename(ctx, target.ID, plan.Held); err != nil {
+ return err
+ }
+
+ replaced, err := env.Docker.Recreate(ctx, spec, environ)
+ if err != nil {
+ // The original is still there under its held name, so the revert can
+ // put it back whatever happened here.
+ if replaced != "" {
+ _ = env.Docker.Remove(context.WithoutCancel(ctx), replaced)
+ }
+
+ return err
+ }
+
+ if env.EnvPlan != nil {
+ env.EnvPlan.Held = plan.Held
+ env.EnvPlan.OriginalID = plan.OriginalID
+ env.EnvPlan.ReplacedID = replaced
+ }
+
+ return nil
+}
+
+// replacement is the one line that differs between the two faults.
+func (m envMutator) replacement(environ []string, variable string) []string {
+ if m.kind == EnvEmpty {
+ return dockerx.ShadowedEmpty(environ, variable)
+ }
+
+ value, _ := dockerx.EnvValue(environ, variable)
+
+ // The original value stays INSIDE the container, under a name the target
+ // does not read, so khaos can restore it from a cold start without ever
+ // writing it to disk.
+ held := dockerx.HeldFor(variable) + "=" + value
+
+ return append(
+ dockerx.WithoutVariable(environ, variable),
+ held,
+ )
+}
+
+// Confirm reads the REPLACEMENT's own environment back from the daemon, which
+// is the only authority on what the process was started with.
+func (m envMutator) Confirm(
+ ctx context.Context,
+ env Env,
+ target dockerx.Target,
+) (string, bool, error) {
+ if env.EnvPlan == nil {
+ return "", false, fmt.Errorf(
+ errNoEnvPlan,
+ ErrNoEnvPlan,
+ EnvFaultKind(m.kind),
+ )
+ }
+
+ plan := env.EnvPlan
+
+ environ, err := env.Docker.ContainerEnv(ctx, m.subject(plan, target))
+ if err != nil {
+ return "", false, err
+ }
+
+ value, carried := environ[plan.Variable]
+
+ if m.kind == EnvEmpty {
+ if !carried || value != "" {
+ return "", false, nil
+ }
+
+ return fmt.Sprintf(confirmEnvEmpty, plan.Variable), true, nil
+ }
+
+ if carried {
+ return "", false, nil
+ }
+
+ if _, held := environ[dockerx.HeldFor(plan.Variable)]; !held {
+ return "", false, nil
+ }
+
+ return fmt.Sprintf(
+ confirmEnvUnset,
+ plan.Variable,
+ dockerx.HeldFor(plan.Variable),
+ ), true, nil
+}
+
+// subject is the replacement when there is one, which is the identity handover
+// invariant 8 needs: the id khaos resolved in the plan is not the id it is
+// measuring by the time this runs.
+func (m envMutator) subject(plan *EnvPlan, target dockerx.Target) string {
+ if plan.ReplacedID != "" {
+ return plan.ReplacedID
+ }
+
+ return target.ID
+}
+
+// Revert removes the replacement and renames the original back, so the
+// anonymous volume, every prior log line and the original container id all
+// survive a fault that would otherwise have destroyed all three.
+func (m envMutator) Revert(
+ ctx context.Context,
+ env Env,
+ record journal.Record,
+) error {
+ var plan EnvPlan
+
+ if len(record.Payload) == 0 {
+ return fmt.Errorf(
+ errNoEnvPlan,
+ ErrNoEnvPlan,
+ EnvFaultKind(m.kind),
+ )
+ }
+
+ if err := record.DecodePayload(&plan); err != nil {
+ return err
+ }
+
+ if env.EnvPlan != nil && env.EnvPlan.ReplacedID != "" {
+ plan.ReplacedID = env.EnvPlan.ReplacedID
+ }
+
+ if env.Docker == nil || plan.Held == "" {
+ return fmt.Errorf(errNoEnvPlan, ErrNoEnvPlan, EnvFaultKind(m.kind))
+ }
+
+ var failures []error
+
+ // A cold start finds the replacement by NAME, because the original's name
+ // is what it took and the id was never journaled until it existed.
+ replaced := plan.ReplacedID
+ if replaced == "" {
+ replaced = strings.TrimSuffix(plan.Held, dockerx.HeldSuffix)
+ }
+
+ if err := env.Docker.Remove(ctx, replaced); err != nil &&
+ !errors.Is(err, dockerx.ErrContainerGone) {
+ failures = append(failures, err)
+ }
+
+ original := strings.TrimSuffix(plan.Held, dockerx.HeldSuffix)
+
+ if err := env.Docker.Rename(ctx, plan.Held, original); err != nil {
+ return errors.Join(append(failures, err)...)
+ }
+
+ // Started by the name it now has, not by the one it had a line ago. The
+ // first version started plan.Held, which the rename had just taken away,
+ // so the original came back STOPPED and the next run reported the range as
+ // missing the fixture.
+ if err := env.Docker.Start(ctx, original); err != nil {
+ failures = append(failures, err)
+ }
+
+ return errors.Join(failures...)
+}
+
+// NeedsEnv tells the run this mutation replaces the target, derived from the
+// registry rather than from the kind's spelling.
+func NeedsEnv(kind string) bool {
+ mutator, ok := registry[kind]
+ if !ok {
+ return false
+ }
+
+ _, isEnv := mutator.(envMutator)
+
+ return isEnv
+}
+
+// ValidEnvKind refuses an invented fault before anything is captured.
+func ValidEnvKind(kind EnvKind) error {
+ if slices.Contains(EnvKinds(), string(kind)) {
+ return nil
+ }
+
+ return fmt.Errorf(
+ errUnknownEnvKind,
+ ErrUnknownEnvKind,
+ kind,
+ strings.Join(EnvKinds(), ", "),
+ )
+}
+
+// Prefix declares this tier's kinds to the registry.
+func (envMutator) Prefix() string { return KindEnvPrefix }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
index bca2e364..7973a665 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
@@ -67,6 +67,10 @@ type Env struct {
CredTrap dbx.Trap
CredPlan CredPlan
+ // The only tier that REPLACES the target, so its plan carries the identity
+ // handover invariant 8 needs.
+ EnvPlan *EnvPlan
+
// A POINTER, like every other tier's per-run state: Plan measures the size
// the target had and Confirm reads it back, and a copy would hand
// Confirm a zero - which read as "the file did not shrink" and aborted
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
index 7910d515..73c18e80 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
@@ -59,6 +59,7 @@ const (
RedisName = "khaos-range-redis"
DBGatewayName = "khaos-range-gateway-db"
SecretGatewayName = "khaos-range-gateway-secret"
+ EnvGatewayName = "khaos-range-gateway-env"
LabelRange = "khaos.range"
FailModeOpen = "open"
@@ -102,6 +103,13 @@ const (
EnvPoolMode = "KHAOS_RANGE_POOL_MODE"
EnvSecretPort = "KHAOS_RANGE_SECRET_PORT"
EnvSecretMode = "KHAOS_RANGE_SECRET_MODE"
+ EnvSecretValue = "KHAOS_RANGE_SECRET_VALUE"
+ EnvEnvPort = "KHAOS_RANGE_ENV_PORT"
+
+ // SecretVariable is the variable the environment faults take away, which
+ // khaos names rather than discovers: the fixture is configured to read its
+ // secret from exactly this one.
+ SecretVariable = "KHAOS_RANGE_SECRET_VALUE"
// AppRole is created by `range up` rather than by the engine's init hook,
// because the hook needs a file in the image and the data directory is a
@@ -198,6 +206,8 @@ const (
errSeedRole = "khaos could not create the range's %s role on its own engine: %w"
+ errNoEnvGateway = "%w: this experiment replaces a container to break its ENVIRONMENT, and the running range has no %s; bring the range down and up again to create it"
+
errNoSecretGateway = "%w: this experiment breaks a secret FILE, and the running range has no %s; bring the range down and up again to create it"
errNoStatefulTier = "%w: this experiment acts on a real database engine, and the running range has none. khaos does not replace these dependencies with its own code, because the finding IS what three correct engines do, so the stack has to be brought up with them: `khaos range down && khaos range up --with-db`"
@@ -228,6 +238,8 @@ type Lab struct {
DBAppPassword string
DBGatewayPort int
SecretPort int
+ EnvPort int
+ SecretValue string
// Read back from the TLS gateway's own environment, never remembered.
Dependency string
@@ -259,6 +271,10 @@ func Locate(ports Ports) (Lab, error) {
DependencyPort: ports.Dependency,
DNSPort: ports.DNS,
+
+ DBGatewayPort: ports.DBGateway,
+ SecretPort: ports.Secret,
+ EnvPort: ports.Env,
}, nil
}
@@ -276,6 +292,16 @@ type Ports struct {
// The resolver publishes NOTHING: it listens on 53 inside its own network
// namespace, which is why that tier needs no capability.
DNS int
+
+ // The M9 fixtures. These have to be CHOSEN here like every other host
+ // port: leaving them unset published `127.0.0.1:0:8080`, which hands the
+ // choice to the kernel's ephemeral range. khaos then neither collision
+ // checks them nor keeps them stable - and the environment faults recreate
+ // the container, so the port moved under khaos's own probe mid-run and the
+ // valid stream reported the fixture as unreachable.
+ DBGateway int
+ Secret int
+ Env int
}
// FreePorts picks every host port the range needs in one go, all distinct.
@@ -309,6 +335,9 @@ func FreePorts() (Ports, error) {
&chosen.Authority,
&chosen.Dependency,
&chosen.DNS,
+ &chosen.DBGateway,
+ &chosen.Secret,
+ &chosen.Env,
} {
port, err := pick()
if err != nil {
@@ -458,6 +487,8 @@ func (l Lab) Up(
EnvPoolMode+"="+poolMode,
EnvSecretPort+"="+strconv.Itoa(l.SecretPort),
EnvSecretMode+"="+secretMode,
+ EnvSecretValue+"="+l.SecretValue,
+ EnvEnvPort+"="+strconv.Itoa(l.EnvPort),
)
args := []string{"up", "--build", "--detach", "--wait"}
@@ -546,6 +577,8 @@ func (l Lab) Down(ctx context.Context, out io.Writer) error {
EnvDBAppPassword+"="+teardownPassword,
EnvDBGatewayPort+"="+strconv.Itoa(teardownPort),
EnvSecretPort+"="+strconv.Itoa(teardownPort),
+ EnvSecretValue+"="+teardownPassword,
+ EnvEnvPort+"="+strconv.Itoa(teardownPort),
)
// The profile is ALWAYS passed on the way down. Compose only removes the
@@ -777,59 +810,62 @@ func Discover(ctx context.Context, reader PortReader) (Lab, error) {
lab.readStatefulTier(running)
- if err = lab.readPooledTier(ctx, reader, running); err != nil {
- return Lab{}, err
- }
-
- if err = lab.readSecretTier(ctx, reader, running); err != nil {
+ if err = lab.readM9Fixtures(ctx, reader, running); err != nil {
return Lab{}, err
}
return lab, nil
}
-// readPooledTier reads the pooled fixture's published port, which is the one
-// service in the stateful tier khaos probes over HTTP.
-func (l *Lab) readPooledTier(
+// readM9Fixtures reads the three gateways M9 added. Each one's absence is not
+// fatal, for the same reason as every other gateway: the experiments that do
+// not need it must still work.
+func (l *Lab) readM9Fixtures(
ctx context.Context,
reader PortReader,
running map[string]dockerx.Target,
) error {
- gateway, ok := running[DBGatewayName]
- if !ok {
- return nil
- }
+ for name, into := range map[string]*int{
+ DBGatewayName: &l.DBGatewayPort,
+ SecretGatewayName: &l.SecretPort,
+ EnvGatewayName: &l.EnvPort,
+ } {
+ gateway, ok := running[name]
+ if !ok {
+ continue
+ }
- port, err := reader.PublishedPort(ctx, gateway.ID, GatewayInternalPort)
- if err != nil {
- return err
- }
+ port, err := reader.PublishedPort(ctx, gateway.ID, GatewayInternalPort)
+ if err != nil {
+ return err
+ }
- l.DBGatewayPort = port
+ *into = port
+ }
return nil
}
-// readSecretTier reads the secret fixture's published port. Its absence is not
-// fatal: the experiments that do not need it must still work.
-func (l *Lab) readSecretTier(
- ctx context.Context,
- reader PortReader,
- running map[string]dockerx.Target,
-) error {
- gateway, ok := running[SecretGatewayName]
- if !ok {
- return nil
- }
+func (l Lab) EnvGatewayURL() string {
+ return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.EnvPort))
+}
- port, err := reader.PublishedPort(ctx, gateway.ID, GatewayInternalPort)
- if err != nil {
- return err
- }
+func (l Lab) EnvProtectedURL() string {
+ return l.EnvGatewayURL() + protectedPath
+}
- l.SecretPort = port
+// IssueEnvToken asks the env-sourced fixture for a credential signed with
+// whatever secret its environment currently holds.
+func (l Lab) IssueEnvToken(ctx context.Context) (string, error) {
+ return l.issueFrom(ctx, l.EnvGatewayURL()+secretTokenPath)
+}
- return nil
+func (l Lab) RequireEnvTier() error {
+ if l.EnvPort != 0 {
+ return nil
+ }
+
+ return fmt.Errorf(errNoEnvGateway, ErrRangeIncomplete, EnvGatewayName)
}
func (l Lab) SecretGatewayURL() string {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
index 00eb0b6d..3c70929d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
@@ -524,6 +524,42 @@ services:
restart: "no"
+ # The same verifier with its secret in the ENVIRONMENT rather than on a
+ # volume, which is the fixture the two `credential.env_*` faults target. It
+ # is a second service rather than a mode of the first because those faults
+ # REPLACE the container: khaos renames the original aside and creates a
+ # replacement under its name, and doing that to the file fixture would take
+ # the volume's experiments down with it.
+ #
+ # The value is minted per `range up` for the same reason as every other
+ # range secret, and khaos never reads it: it asks this service for a
+ # credential, and holds the original inside the replacement it creates.
+ gateway-env:
+ container_name: khaos-range-gateway-env
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["secret"]
+ environment:
+ KHAOS_RANGE_SECRET_VAR: "KHAOS_RANGE_SECRET_VALUE"
+ KHAOS_RANGE_SECRET_VALUE: "${KHAOS_RANGE_SECRET_VALUE:?khaos mints this; run `khaos range up`}"
+ KHAOS_RANGE_SECRET_MODE: "${KHAOS_RANGE_SECRET_MODE:-unchecked}"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_ENV_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ networks:
+ range:
+ ipv4_address: 10.244.244.17
+ healthcheck:
+ test: ["CMD", "/usr/local/bin/khaos-range", "alivecheck"]
+ interval: 2s
+ timeout: 2s
+ retries: 15
+ restart: "no"
+
networks:
range:
name: khaos-range
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
index 8fee56de..3453de9a 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
@@ -229,6 +229,16 @@ leftover_roles() {
# The pool configuration is fixed at container creation, deliberately, so a
# gate spanning all three has to restart the range. A fixture that could be
# reconfigured mid-run would let khaos change the target it is measuring.
+held_containers() {
+ docker ps -a --format '{{.Names}}' |
+ grep -c -- '-khaos-held' | tr -d '[:space:]' || true
+}
+
+env_of() {
+ docker inspect "$1" --format '{{range .Config.Env}}{{println .}}{{end}}' \
+ 2>/dev/null | grep -E "^$2=" | tail -1 || true
+}
+
start_range() {
local pool="${1:-unbounded}"
local secret="${2:-unchecked}"
@@ -549,6 +559,120 @@ if start_range unbounded unchecked; then
"private to the container"
fi
+echo "== the environment: the one fault class that REPLACES the container =="
+
+# A container's environment is fixed at creation. `ContainerUpdate` accepts an
+# Env field, returns HTTP 200 with no warning, and changes nothing - measured -
+# so these two replace the container, and the mechanism was chosen to keep the
+# ORIGINAL rather than to rebuild it: a recreate from an inspection loses the
+# anonymous volume with its data stranded, every prior log line, and the
+# container id invariant 8 re-verifies against. khaos renames the original
+# aside and renames it back on the way out.
+#
+# The two faults are INDISTINGUISHABLE to a service that reads with os.Getenv,
+# which is the whole teaching point, so both are expected to read the same
+# against each fixture mode.
+declare -A WANT_ENV=(
+ [unchecked]="deviated / PERMITTED"
+ [validated]="deviated / DENIED"
+)
+
+declare -A WANT_ENV_EXIT=(
+ [unchecked]=2
+ [validated]=0
+)
+
+for secret in unchecked validated; do
+ if ! start_range unbounded "$secret"; then
+ continue
+ fi
+
+ for experiment in credential.env_empty credential.env_unset; do
+ run_experiment "$experiment"
+
+ label="${experiment#credential.} against a ${secret} verifier"
+
+ assert_verdict "$label" "${WANT_ENV[$secret]}"
+ assert_exit "$label" "${WANT_ENV_EXIT[$secret]}"
+ assert_confirmed "$label"
+ assert_control_held "$label"
+ done
+
+ # The original has to be back, running, under its own name - and nothing
+ # held aside. A revert that renamed it back and forgot to start it left the
+ # next run reporting the range as missing the fixture entirely.
+ if docker ps --filter name=khaos-range-gateway-env --filter status=running \
+ --format '{{.Names}}' | grep -q 'khaos-range-gateway-env$'; then
+ pass "the original container is back and running after the revert"
+ else
+ fail "the original container did not come back running"
+ fi
+
+ held="$(held_containers)"
+ if [[ "$held" == "0" ]]; then
+ pass "no container is left renamed aside"
+ else
+ fail "${held} container(s) left renamed aside"
+ fi
+
+ # The variable is back with its ORIGINAL value, not an empty one: the
+ # revert restores the container rather than patching the environment.
+ restored="$(env_of khaos-range-gateway-env KHAOS_RANGE_SECRET_VALUE)"
+ if [[ -n "$restored" && "$restored" != "KHAOS_RANGE_SECRET_VALUE=" ]]; then
+ pass "the variable is back with its original value"
+ else
+ fail "the variable came back as ${restored:-<absent>}"
+ fi
+done
+
+echo "== an environment fault khaos cannot deliver is refused by name =="
+
+if start_range unbounded unchecked; then
+ port="$(docker port khaos-range-gateway-env 8080 2>/dev/null |
+ head -1 | cut -d: -f2)"
+ token="$(curl -s --max-time 5 -X POST \
+ "http://127.0.0.1:${port}/secret-token" 2>/dev/null |
+ sed -E 's/.*"token":"([^"]+)".*/\1/')"
+
+ # A variable the IMAGE defines as well. The daemon merges the image's
+ # environment into any key a create request omits, so a replacement made
+ # without it gets the image's value back and khaos would report a fault the
+ # daemon had undone for it.
+ run_unmeasured credential.env_unset \
+ --target khaos-range-gateway-env \
+ --url "http://127.0.0.1:${port}/protected" \
+ --token "$token" \
+ --variable PATH
+
+ if [[ "$last_code" != 0 ]]; then
+ pass "a variable the image defines is refused"
+ else
+ fail "khaos reported an unset the daemon would have undone"
+ fi
+
+ assert_report_has \
+ "the refusal names the mechanism and the fault to use instead" \
+ "gets the image's value back"
+
+ # A variable that is not there at all. Nothing to take away, and the
+ # refusal names what IS there so the operator can see the typo.
+ run_unmeasured credential.env_unset \
+ --target khaos-range-gateway-env \
+ --url "http://127.0.0.1:${port}/protected" \
+ --token "$token" \
+ --variable KHAOS_NOT_A_VARIABLE
+
+ if [[ "$last_code" != 0 ]]; then
+ pass "a variable the target does not carry is refused"
+ else
+ fail "khaos took away a variable that was never there"
+ fi
+
+ assert_report_has \
+ "the refusal names the variables the target DOES carry" \
+ "It carries:"
+fi
+
echo "== the pooled faults put BOTH containers to the refusal predicates =="
# khaos measures one container and performs the rotation at another, so the