Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

fix(khaos): the six audit findings, closed rather than carried, and the rotation they cleared the way for

CarterPerez-dev3f49be219f24
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go
index aa6fd6df..edcc0c92 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go
@@ -34,6 +34,25 @@ const (
// step 4 names as the retry storm.
AuthzRetries = "retries"
+ // AuthzCached is also CORRECT, and it exists because khaos's own verdict
+ // machinery was wrong about it. It differs from `distinguishes` in two ways
+ // that are ordinary in real services and that khaos had never been shown
+ // together: it caches an authorization decision for a second, and it
+ // expresses a refusal as 503 rather than 403, the way a service that maps
+ // "my dependency would not authorize this" onto an upstream error does.
+ //
+ // That produces one acceptance from the cache followed by refusals khaos
+ // reads as `Unavailable`, in both fault windows. khaos reported it
+ // `deviated / PERMITTED` with "this target's fallback is to allow",
+ // because its order-sensitive counts reset on `Rejected` alone while this
+ // experiment DECLARES a 503 as its fail-closed outcome. This mode is the
+ // input the gate needs to be able to reject.
+ AuthzCached = "cached"
+
+ // Short enough to lapse several times inside a fault window, which is what
+ // makes the cached acceptance the FIRST sample rather than the whole run.
+ authzCacheFor = time.Second
+
headerAuthzReason = "X-Khaos-Authz"
reasonNotAuthenticated = "this service never issued that credential, which is its own check and not the dependency's"
@@ -48,6 +67,10 @@ const (
reasonCollapsed = "the authorization dependency DENIED it and this mode read that as no answer, so it allowed it"
+ reasonCachedAllow = "this caller's own authorization-decision cache still holds an allow for that credential"
+
+ reasonCachedRefusal = "this caller's own authorization-decision cache still holds a refusal for that credential"
+
// Three attempts, because the finding is the ask COUNT and one retry would
// make it indistinguishable from a second probe arriving.
authzAttempts = 3
@@ -71,6 +94,18 @@ type authzGateway struct {
mu sync.Mutex
issued map[string]bool
+
+ // decided is this caller's own authorization-decision cache, and only the
+ // cached mode writes to it.
+ decided map[string]verdictAt
+}
+
+// verdictAt is one cached authorization decision. A caller that caches is
+// ordinary; what khaos was wrong about is how the cache's first sample reads
+// once the refusals that follow it are 503s rather than 403s.
+type verdictAt struct {
+ allow bool
+ at time.Time
}
func newAuthzGateway(dependency, mode string) *authzGateway {
@@ -79,7 +114,47 @@ func newAuthzGateway(dependency, mode string) *authzGateway {
mode: mode,
client: &http.Client{Timeout: upstreamTimeout},
issued: map[string]bool{},
+ decided: map[string]verdictAt{},
+ }
+}
+
+// refusalStatus is how THIS caller says no. A 503 is as common as a 403 in
+// service-to-service code -- a service that maps a dependency's refusal onto an
+// upstream error -- and khaos has to read both as the same refusal.
+func (g *authzGateway) refusalStatus() int {
+ if g.mode == AuthzCached {
+ return http.StatusServiceUnavailable
+ }
+
+ return http.StatusForbidden
+}
+
+// cachedVerdict answers from this caller's own cache while the entry is fresh.
+func (g *authzGateway) cachedVerdict(credential string) (bool, bool) {
+ if g.mode != AuthzCached {
+ return false, false
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ held, found := g.decided[credential]
+ if !found || time.Since(held.at) > authzCacheFor {
+ return false, false
}
+
+ return held.allow, true
+}
+
+func (g *authzGateway) remember(credential string, allow bool) {
+ if g.mode != AuthzCached {
+ return
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ g.decided[credential] = verdictAt{allow: allow, at: time.Now()}
}
func (g *authzGateway) routes() *http.ServeMux {
@@ -144,13 +219,31 @@ func (g *authzGateway) protected(w http.ResponseWriter, r *http.Request) {
return
}
- w.WriteHeader(http.StatusForbidden)
+ w.WriteHeader(g.refusalStatus())
}
// decide is the whole fixture. One branch reads the dependency's three answers
// as three; the second reads "not 200" as "no answer"; the third reads them
// correctly and asks again every time it does not like one.
func (g *authzGateway) decide(credential string) (bool, string) {
+ if allow, fresh := g.cachedVerdict(credential); fresh {
+ if allow {
+ return true, reasonCachedAllow
+ }
+
+ return false, reasonCachedRefusal
+ }
+
+ allow, reason := g.readDependency(credential)
+
+ g.remember(credential, allow)
+
+ return allow, reason
+}
+
+// readDependency asks and applies this mode's READING of the answer, which is
+// the one line the whole experiment is about.
+func (g *authzGateway) readDependency(credential string) (bool, string) {
status, reached := g.ask(credential)
switch g.mode {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go
index c0d4e7ea..5317153a 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go
@@ -10,6 +10,7 @@ import (
"strings"
"sync/atomic"
"testing"
+ "time"
)
// stubAuthz is khaos's side of the experiment, reduced to what the fixture can
@@ -96,6 +97,7 @@ func TestEveryModeAuthenticatesLocally(t *testing.T) {
AuthzDistinguishes,
AuthzCollapses,
AuthzRetries,
+ AuthzCached,
} {
t.Run(mode, func(t *testing.T) {
gateway, dependency := authzFixture(t, mode)
@@ -239,6 +241,95 @@ func TestTheCollapseNamesWhichAnswerItIgnored(t *testing.T) {
}
}
+// The cached mode is the input khaos's own verdict machinery was wrong about,
+// so the fixture has to actually produce that sequence: one acceptance from the
+// cache, then refusals expressed as 503. Both halves matter -- the cache, and
+// the status -- because khaos's order-sensitive counts reset on a REFUSAL and
+// a 503 is only a refusal for a control that declared it.
+func TestTheCachedModeServesOnceThenRefusesWith503(t *testing.T) {
+ gateway, dependency := authzFixture(t, AuthzCached)
+ credential := issued(t, gateway)
+
+ if got := askAuthz(t, gateway, credential); got != http.StatusOK {
+ t.Fatalf("an allowed credential returned %d, want 200", got)
+ }
+
+ dependency.status.Store(http.StatusForbidden)
+
+ // Inside the cache window the caller answers from its own cache and never
+ // asks, which is what puts the acceptance FIRST.
+ dependency.asked.Store(0)
+
+ if got := askAuthz(t, gateway, credential); got != http.StatusOK {
+ t.Fatalf(
+ "within the cache window the caller answered %d; a cached allow is what makes the first sample an acceptance",
+ got,
+ )
+ }
+
+ if asked := dependency.asked.Load(); asked != 0 {
+ t.Fatalf(
+ "the caller asked %d time(s) inside its own cache window, so it holds no cache and this mode cannot produce the sequence it exists for",
+ asked,
+ )
+ }
+
+ time.Sleep(authzCacheFor + 100*time.Millisecond)
+
+ if got := askAuthz(
+ t,
+ gateway,
+ credential,
+ ); got != http.StatusServiceUnavailable {
+ t.Fatalf(
+ "once the cache lapsed the caller answered %d, want %d: this mode expresses its refusal as an upstream error, which is the half khaos read wrong",
+ got,
+ http.StatusServiceUnavailable,
+ )
+ }
+
+ if asked := dependency.asked.Load(); asked == 0 {
+ t.Fatal("the caller never asked after its cache lapsed")
+ }
+
+ // And it is still CORRECT: it refuses, it does not serve a credential the
+ // dependency denied.
+ if got := askAuthz(
+ t,
+ gateway,
+ credential,
+ ); got == http.StatusOK {
+ t.Fatal(
+ "the cached mode served a credential its dependency had denied, which would make it a second collapse rather than a correct caller",
+ )
+ }
+}
+
+// Only the cached mode caches, or every other mode's measured behaviour moves.
+func TestNoOtherModeHoldsADecisionCache(t *testing.T) {
+ for _, mode := range []string{
+ AuthzDistinguishes,
+ AuthzCollapses,
+ AuthzRetries,
+ } {
+ t.Run(mode, func(t *testing.T) {
+ gateway, dependency := authzFixture(t, mode)
+ credential := issued(t, gateway)
+
+ askAuthz(t, gateway, credential)
+ dependency.asked.Store(0)
+ askAuthz(t, gateway, credential)
+
+ if asked := dependency.asked.Load(); asked == 0 {
+ t.Fatalf(
+ "%s answered a second request without asking its dependency, so it is caching and its measured ask-rate is not what khaos thinks it is",
+ mode,
+ )
+ }
+ })
+ }
+}
+
// A caller that cannot reach its dependency at all is the branch every network
// fault already reaches, and the correct modes must refuse there too.
func TestAnUnreachableDependencyIsRefusedByTheCorrectModes(t *testing.T) {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go
index ebf429bd..6e7fd6e1 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go
@@ -35,7 +35,7 @@ const (
reasonDenied = "khaos's authorization dependency denies this credential: a definitive verdict, with full reachability"
reasonOutage = "khaos's authorization dependency cannot answer; this is the absence of a decision, not a decision"
- subjectAuthz = "an authorization service that allows every credential it was not told to deny"
+ SubjectAuthz = "an authorization service that allows every credential it was not told to deny"
askTimeout = 2 * time.Second
shutdownGrace = 2 * time.Second
@@ -101,7 +101,7 @@ func New() *Dependency {
// wildcard bind would put an allow-everything authorization service on every
// interface the host has.
func (d *Dependency) Listen(addr string) error {
- if err := bindguard.Check(addr, subjectAuthz); err != nil {
+ if err := bindguard.Check(addr, SubjectAuthz); err != nil {
return err
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go
index 1d3be2fc..8ae280c2 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go
@@ -31,6 +31,8 @@ const (
errBadAddr = "%w: %q is not a host and port (%v)"
errOffBox = "%w: %q is not on loopback or on a Docker bridge, so it is reachable from outside this host and a container cannot reach it anyway; bind the bridge gateway the daemon reports"
+
+ errUnmanaged = "%w: %q is not a gateway of any network the Docker daemon manages, and %s must not be reachable from anything but the target under test. khaos checked %d network gateway(s). A kernel bridge is not enough: a CNI pod bridge, a swarm gateway bridge and a host interface enslaved to a bridge all look like one, and the last of those is the host's own routable address"
)
// ErrUnsafeBind is one sentinel for every listener khaos opens. Each tier
@@ -86,6 +88,50 @@ func Check(listenAddr, subject string) error {
return nil
}
+// Managed is the narrow check, and Check is the structural one. Check asks
+// "could a container reach this and nothing off the host"; this asks "is this
+// an address DOCKER manages", which is the question `/sys/class/net/<if>/bridge`
+// cannot answer: that file exists for every kernel bridge device, including a
+// CNI pod bridge, a swarm gateway bridge, every user-defined Docker network's
+// bridge, and -- on a host whose LAN interface is enslaved to one -- the
+// routable LAN address itself. Check accepted all of those while its own
+// comment claimed nothing but the target could reach the listener.
+//
+// gateways comes from the daemon, so this lives wherever a Docker client does
+// and the structural check stays in the packages that bind. An empty set is a
+// refusal rather than a pass: khaos could not establish the allowed addresses,
+// and binding anyway is the behaviour this exists to stop.
+func Managed(listenAddr, subject string, gateways []string) error {
+ host, _, err := net.SplitHostPort(listenAddr)
+ if err != nil {
+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)
+ }
+
+ ip := net.ParseIP(host)
+ if ip == nil {
+ return fmt.Errorf(errWildcard, ErrUnsafeBind, listenAddr, subject)
+ }
+
+ if ip.IsLoopback() {
+ return nil
+ }
+
+ for _, gateway := range gateways {
+ if managed := net.ParseIP(gateway); managed != nil &&
+ managed.Equal(ip) {
+ return nil
+ }
+ }
+
+ return fmt.Errorf(
+ errUnmanaged,
+ ErrUnsafeBind,
+ listenAddr,
+ subject,
+ len(gateways),
+ )
+}
+
func bridgeHosts(ip net.IP) (bool, error) {
interfaces, err := net.Interfaces()
if err != nil {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go
index 315b6f5f..8ffdbf67 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go
@@ -81,3 +81,83 @@ func TestAWildcardRefusalNamesTheSubject(t *testing.T) {
)
}
}
+
+// Managed is the check a kernel bridge cannot satisfy. Measured on the
+// development host when this was written: a CNI pod bridge at 10.42.0.1, a
+// swarm gateway bridge at 172.23.0.1 and forty-nine user-defined bridges, one
+// of them carrying nine containers of an unrelated project -- every one of them
+// carrying the sysfs marker Check reads, and none of them an address khaos
+// should serve an allow-everything listener on.
+func TestManagedTakesTheAllowedSetFromTheDaemon(t *testing.T) {
+ gateways := []string{"172.17.0.1", "10.244.244.1"}
+
+ for _, row := range []struct {
+ name string
+ addr string
+ set []string
+ want bool
+ }{
+ {
+ name: "a gateway the daemon manages",
+ addr: "172.17.0.1:45999",
+ set: gateways,
+ want: true,
+ },
+ {
+ name: "another one it manages",
+ addr: "10.244.244.1:45999",
+ set: gateways,
+ want: true,
+ },
+ {
+ name: "loopback, which needs no daemon",
+ addr: "127.0.0.1:45999",
+ set: nil,
+ want: true,
+ },
+ {
+ name: "a CNI pod bridge, which carries the sysfs marker",
+ addr: "10.42.0.1:45999",
+ set: gateways,
+ want: false,
+ },
+ {
+ name: "a host interface enslaved to a bridge",
+ addr: "192.168.1.183:45999",
+ set: gateways,
+ want: false,
+ },
+ {
+ name: "an empty set is a refusal, not a pass",
+ addr: "172.17.0.1:45999",
+ set: nil,
+ want: false,
+ },
+ {
+ name: "a wildcard, which names no address at all",
+ addr: "0.0.0.0:45999",
+ set: gateways,
+ want: false,
+ },
+ } {
+ t.Run(row.name, func(t *testing.T) {
+ err := Managed(row.addr, subject, row.set)
+
+ if row.want && err != nil {
+ t.Fatalf("%s was refused: %v", row.addr, err)
+ }
+
+ if row.want {
+ return
+ }
+
+ if err == nil {
+ t.Fatalf("%s was accepted", row.addr)
+ }
+
+ if !errors.Is(err, ErrUnsafeBind) {
+ t.Fatalf("%s: err = %v, want ErrUnsafeBind", row.addr, err)
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index e3d89238..f21ed690 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -11,20 +11,24 @@ import (
"maps"
"net"
"net/url"
+ "os"
"slices"
"strconv"
"strings"
"time"
+ "github.com/CarterPerez-dev/khaos/internal/authz"
"github.com/CarterPerez-dev/khaos/internal/bindguard"
"github.com/CarterPerez-dev/khaos/internal/dbx"
"github.com/CarterPerez-dev/khaos/internal/dnsrv"
"github.com/CarterPerez-dev/khaos/internal/dockerx"
"github.com/CarterPerez-dev/khaos/internal/experiment"
+ "github.com/CarterPerez-dev/khaos/internal/idp"
"github.com/CarterPerez-dev/khaos/internal/inject"
"github.com/CarterPerez-dev/khaos/internal/inject/proxy"
"github.com/CarterPerez-dev/khaos/internal/journal"
"github.com/CarterPerez-dev/khaos/internal/lease"
+ "github.com/CarterPerez-dev/khaos/internal/pki"
"github.com/CarterPerez-dev/khaos/internal/rangelab"
"github.com/CarterPerez-dev/khaos/internal/render"
"github.com/CarterPerez-dev/khaos/internal/run"
@@ -128,6 +132,15 @@ enforcing in the first place.`
msgAuthzUp = "khaos is answering authorization decisions on %s, allowing every credential until it is told otherwise"
+ // Every parameter can also arrive in the environment, which is the only
+ // channel that does not put a credential in the world-readable process
+ // table. Same convention as the resolver nonce.
+ paramEnv = "%s [or %s]"
+
+ envParamPrefix = "KHAOS_"
+ flagWordSep = "-"
+ envWordSep = "_"
+
errNoSubjectSecret = "%s does not carry %s in its environment, so khaos cannot read the credential its own connection pool authenticated with, and a rotation it cannot undo is not one khaos will perform"
errTargetOffNetwork = "%s is not attached to %s, so it cannot be using the chaos resolver that serves it and khaos could not count its queries: %w"
@@ -297,11 +310,16 @@ func experimentFlags(cmd *cobra.Command, definition experiment.Definition) {
Duration(flagSampleEvery, experiment.DefaultSampleInterval, sampleUsage)
}
+// paramUsage names the environment variable too, because a channel an operator
+// cannot discover is a channel nobody uses -- and for a credential it is the
+// only one that keeps the value out of the world-readable process table.
func paramUsage(param experiment.Param) string {
- if param.Default == "" {
- return param.Usage
+ usage := param.Usage
+ if param.Default != "" {
+ usage = fmt.Sprintf(paramDefault, usage, param.Default)
}
- return fmt.Sprintf(paramDefault, param.Usage, param.Default)
+
+ return fmt.Sprintf(paramEnv, usage, envFor(param.Name))
}
type inputs struct {
@@ -324,6 +342,17 @@ func experimentInputs(
supplied := map[string]string{}
for _, param := range definition.Params {
if !cmd.Flags().Changed(param.Name) {
+ // A FLAG is not the only way in, and for a credential it is the
+ // worst one: /proc/<pid>/cmdline is world-readable (mode 444,
+ // measured) while /proc/<pid>/environ is 400, which is why the
+ // chaos resolver's nonce has always been an environment variable
+ // rather than an argument. An experiment can require two live
+ // credentials, so argv-only put both of an operator's secrets in
+ // the process table and in shell history.
+ if value, given := os.LookupEnv(envFor(param.Name)); given {
+ supplied[param.Name] = value
+ }
+
continue
}
@@ -343,6 +372,20 @@ func experimentInputs(
return inputs{Project: project, Supplied: supplied, Timing: timing}, nil
}
+// envFor is the environment variable that supplies a parameter when its flag is
+// absent. Derived from the parameter's own name rather than listed, so a new
+// parameter gets the channel without anyone remembering to add it:
+// `--revocable-token` reads `KHAOS_REVOCABLE_TOKEN`.
+//
+// Precedence is flag, then environment, then the range, then the declaration's
+// default. The environment beats the range because both are the operator
+// speaking and the range is only a convenience.
+func envFor(param string) string {
+ return envParamPrefix + strings.ToUpper(
+ strings.ReplaceAll(param, flagWordSep, envWordSep),
+ )
+}
+
func experimentTiming(cmd *cobra.Command) (experiment.Timing, error) {
durations := map[string]time.Duration{}
@@ -1212,6 +1255,26 @@ func startProxy(
// startProvider binds khaos's own identity provider on the Docker bridge gateway
// at the port the issuer names, never a wildcard: the target addresses the issuer
// as `host.docker.internal`, which does not resolve here.
+// providerListen is where the provider will bind, computed before the bind so
+// the guard and the listener are given the same string.
+func providerListen(
+ ctx context.Context,
+ reader rangelab.PortReader,
+ spec experiment.Experiment,
+) (string, error) {
+ port, err := portOf(spec.IdP.Issuer())
+ if err != nil {
+ return "", err
+ }
+
+ gateway, err := reader.BridgeGateway(ctx)
+ if err != nil {
+ return "", err
+ }
+
+ return net.JoinHostPort(gateway, strconv.Itoa(port)), nil
+}
+
func startProvider(
ctx context.Context,
out io.Writer,
@@ -1220,18 +1283,11 @@ func startProvider(
) error {
issuer := spec.IdP.Issuer()
- port, err := portOf(issuer)
- if err != nil {
- return err
- }
-
- gateway, err := reader.BridgeGateway(ctx)
+ listen, err := providerListen(ctx, reader, spec)
if err != nil {
return err
}
- listen := net.JoinHostPort(gateway, strconv.Itoa(port))
-
if err := spec.IdP.Listen(listen); err != nil {
return fmt.Errorf(errIdPBind, listen, err)
}
@@ -1274,10 +1330,10 @@ func standUpTiers(
return standUpIdPTier(ctx, out, work.Docker, spec, &env)
},
func() (func(), error) {
- return standUpPKITier(out, spec, &env)
+ return standUpPKITier(ctx, out, work.Docker, spec, &env)
},
func() (func(), error) {
- return standUpAuthzTier(out, spec, &env)
+ return standUpAuthzTier(ctx, out, work.Docker, spec, &env)
},
func() (func(), error) {
return standUpDNSTier(ctx, out, work.Docker, spec, target, &env)
@@ -1316,6 +1372,26 @@ func standUpTiers(
return env, closeAll, nil
}
+// managedBind is the NARROW half of the bind guard and it lives here because
+// the allowed set comes from the daemon. Each binding package carries the
+// structural half -- no wildcard, no privileged port, loopback or a bridge --
+// and a kernel bridge is not enough: measured on a development host, that
+// includes a CNI pod bridge, a swarm gateway bridge, forty-nine user-defined
+// bridges carrying other projects' containers, and on a host whose LAN
+// interface is enslaved to a bridge, the routable LAN address itself.
+func managedBind(
+ ctx context.Context,
+ docker *dockerx.Client,
+ addr, subject string,
+) error {
+ gateways, err := docker.Gateways(ctx)
+ if err != nil {
+ return err
+ }
+
+ return bindguard.Managed(addr, subject, gateways)
+}
+
func standUpProxyTier(
ctx context.Context,
out io.Writer,
@@ -1327,6 +1403,15 @@ func standUpProxyTier(
return nil, nil
}
+ if err := managedBind(
+ ctx,
+ docker,
+ spec.ProxyListen,
+ proxy.SubjectProxy,
+ ); err != nil {
+ return nil, err
+ }
+
fault, err := startProxy(ctx, out, docker, *spec)
if err != nil {
return nil, err
@@ -1349,6 +1434,20 @@ func standUpIdPTier(
return nil, nil
}
+ listen, err := providerListen(ctx, docker, *spec)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := managedBind(
+ ctx,
+ docker,
+ listen,
+ idp.SubjectProvider,
+ ); err != nil {
+ return nil, err
+ }
+
if err := startProvider(ctx, out, docker, *spec); err != nil {
return nil, err
}
@@ -1360,7 +1459,9 @@ func standUpIdPTier(
}
func standUpPKITier(
+ ctx context.Context,
out io.Writer,
+ docker *dockerx.Client,
spec *experiment.Experiment,
env *inject.Env,
) (func(), error) {
@@ -1368,6 +1469,15 @@ func standUpPKITier(
return nil, nil
}
+ for addr, subject := range map[string]string{
+ spec.AuthorityListen: pki.SubjectRevocation,
+ spec.AuthorityTLS: pki.SubjectDependency,
+ } {
+ if err := managedBind(ctx, docker, addr, subject); err != nil {
+ return nil, err
+ }
+ }
+
if err := startAuthority(out, *spec); err != nil {
return nil, err
}
@@ -1383,7 +1493,9 @@ func standUpPKITier(
// through the whole run, or the baseline measures a dependency that is down and
// the fault window is the only one where it is up.
func standUpAuthzTier(
+ ctx context.Context,
out io.Writer,
+ docker *dockerx.Client,
spec *experiment.Experiment,
env *inject.Env,
) (func(), error) {
@@ -1391,6 +1503,15 @@ func standUpAuthzTier(
return nil, nil
}
+ if err := managedBind(
+ ctx,
+ docker,
+ spec.AuthzListen,
+ authz.SubjectAuthz,
+ ); err != nil {
+ return nil, err
+ }
+
if err := spec.Authz.Listen(spec.AuthzListen); err != nil {
// The guard refuses BEFORE the bind is attempted, so the port-collision
// advice below is wrong for the one refusal an operator will actually
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
index 39250fe4..f06bb715 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -918,3 +918,96 @@ func answerableFor(definition experiment.Definition) []string {
return out
}
+
+// A credential supplied as a FLAG is in /proc/<pid>/cmdline, which is
+// world-readable (mode 444, measured), while /proc/<pid>/environ is 400. That
+// is why the chaos resolver's nonce has always been an environment variable
+// rather than an argument, and `credential.apikey_revoked` is the first
+// experiment to require TWO live credentials, so argv-only put both of an
+// operator's secrets in the process table and in shell history.
+func TestAParameterCanArriveInTheEnvironmentInsteadOfArgv(t *testing.T) {
+ definition := outage(t)
+
+ t.Setenv("KHAOS_TOKEN", "a-credential-from-the-environment")
+
+ cmd := newRunCmd()
+ cmd.SetArgs([]string{definition.Name})
+
+ in, err := experimentInputs(
+ experimentCmdFor(t, cmd, definition),
+ definition,
+ )
+ if err != nil {
+ t.Fatalf("inputs: %v", err)
+ }
+
+ if in.Supplied[experiment.ParamToken] != "a-credential-from-the-environment" {
+ t.Fatalf(
+ "the environment channel was ignored: %q; a credential has no other way in that keeps it out of the process table",
+ in.Supplied[experiment.ParamToken],
+ )
+ }
+}
+
+// The FLAG still wins, because an operator who typed one meant it, and the
+// environment must not silently override what they just said.
+func TestAFlagBeatsTheEnvironment(t *testing.T) {
+ definition := outage(t)
+
+ t.Setenv("KHAOS_TOKEN", "from-the-environment")
+
+ cmd := newRunCmd()
+ sub := experimentCmdFor(t, cmd, definition)
+
+ if err := sub.Flags().
+ Set(experiment.ParamToken, "from-the-flag"); err != nil {
+ t.Fatalf("set the flag: %v", err)
+ }
+
+ in, err := experimentInputs(sub, definition)
+ if err != nil {
+ t.Fatalf("inputs: %v", err)
+ }
+
+ if in.Supplied[experiment.ParamToken] != "from-the-flag" {
+ t.Fatalf(
+ "the environment overrode a flag the operator typed: %q",
+ in.Supplied[experiment.ParamToken],
+ )
+ }
+}
+
+// And the mapping is DERIVED from the parameter's name, so a new parameter gets
+// the channel without anyone remembering to add it.
+func TestTheEnvironmentNameIsDerivedFromTheParameter(t *testing.T) {
+ for param, want := range map[string]string{
+ experiment.ParamToken: "KHAOS_TOKEN",
+ experiment.ParamRevocable: "KHAOS_REVOCABLE_TOKEN",
+ experiment.ParamAuthzListen: "KHAOS_AUTHZ_LISTEN",
+ experiment.ParamInvalidToken: "KHAOS_INVALID_TOKEN",
+ } {
+ if got := envFor(param); got != want {
+ t.Fatalf("envFor(%q) = %q, want %q", param, got, want)
+ }
+ }
+}
+
+// experimentCmdFor finds the per-experiment subcommand, which is where the
+// parameters are declared as flags.
+func experimentCmdFor(
+ t *testing.T,
+ root *cobra.Command,
+ definition experiment.Definition,
+) *cobra.Command {
+ t.Helper()
+
+ for _, sub := range root.Commands() {
+ if sub.Name() == definition.Name {
+ return sub
+ }
+ }
+
+ t.Fatalf("no subcommand for %s", definition.Name)
+
+ return nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
index 577aa492..ab70925c 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
@@ -30,6 +30,10 @@ var ErrUnknownPoolMode = errors.New(
"that is not a pool configuration the range fixture understands",
)
+var ErrUnknownDNSMode = errors.New(
+ "that is not a resolution behaviour the range gateway understands",
+)
+
var ErrUnknownAuthzMode = errors.New(
"that is not an authorization-error handling the range fixture understands",
)
@@ -46,6 +50,7 @@ const (
flagPoolMode = "pool-mode"
flagSecretMode = "secret-mode"
flagAuthzMode = "authz-mode"
+ flagDNSMode = "dns-mode"
mintedSecrets = 4
@@ -74,7 +79,9 @@ running.`
quietUsage = "suppress compose build and start output"
secretModeUsage = "how the range's secret fixture handles the secret it reads. validated checks the length after every read, which is the correct handling and the fix for CVE-2019-20933; unchecked lets an empty value flow into the crypto, which is that CVE's own shape. An unrecognised value is refused rather than quietly leaving the fixture correct"
- authzModeUsage = "how the range's service-to-service caller reads its authorization dependency's answers. distinguishes reads allow, deny and unknown as three outcomes, which is correct; collapses reads anything that is not a 200 as an absence of an answer and allows it, which is research 07's F20 and the single most common fail-open in service-to-service code; retries reaches the right verdict the wrong way, asking again every time it does not like an answer, which hammers the dependency with a credential it has already been told is dead. An unrecognised value is refused rather than quietly leaving the caller correct"
+ dnsModeUsage = "how the range's DNS gateway resolves its dependency's name. reresolve resolves on every request, which is a client that opens a fresh connection per call; pinned resolves once and reuses the connection, which is the DEFAULT behaviour of a Go service and the shape that never sees a zone rotation. Both are real, and `dns.ttl_rotate` needs both to assert its verdict in two directions. An unrecognised value is refused rather than quietly leaving the gateway re-resolving"
+
+ authzModeUsage = "how the range's service-to-service caller reads its authorization dependency's answers. distinguishes reads allow, deny and unknown as three outcomes, which is correct; collapses reads anything that is not a 200 as an absence of an answer and allows it, which is research 07's F20 and the single most common fail-open in service-to-service code; retries reaches the right verdict the wrong way, asking again every time it does not like an answer, which hammers the dependency with a credential it has already been told is dead; cached is correct as well and is the shape khaos's own verdict machinery was wrong about, holding a one-second decision cache and expressing its refusals as 503 rather than 403. An unrecognised value is refused rather than quietly leaving the caller correct"
poolModeUsage = "how the stateful tier's pooled fixture configures its connection pool. unbounded is the DEFAULT configuration of a Go service rather than a broken one, because database/sql closes a connection for age only when SetConnMaxLifetime is positive; bounded sets a lifetime, which is the knob that turns a rotation into a revocation; held checks one connection out and never returns it, which no lifetime bounds at all"
@@ -92,6 +99,7 @@ running.`
errUnknownPoolMode = "%w: %q (known pool modes: %s)"
errUnknownSecretMode = "%w: %q (known secret modes: %s)"
errUnknownAuthzMode = "%w: %q (known authorization modes: %s)"
+ errUnknownDNSMode = "%w: %q (known DNS modes: %s)"
msgStatefulUp = "the stateful tier is up: PostgreSQL, MySQL, MariaDB and Redis, none of them published to a host port"
msgPortHint = "host port %d was chosen at random; pass --port to pin it"
@@ -163,6 +171,10 @@ func newRangeUpCmd() *cobra.Command {
if err != nil {
return err
}
+ dnsMode, err := cmd.Flags().GetString(flagDNSMode)
+ if err != nil {
+ return err
+ }
return runRangeUp(
cmd.Context(),
cmd.OutOrStdout(),
@@ -176,6 +188,7 @@ func newRangeUpCmd() *cobra.Command {
pool: poolMode,
secret: secretMode,
authz: authzMode,
+ dns: dnsMode,
},
)
},
@@ -198,6 +211,7 @@ func newRangeUpCmd() *cobra.Command {
rangelab.AuthzCollapses,
authzModeUsage,
)
+ cmd.Flags().String(flagDNSMode, rangelab.DNSReresolve, dnsModeUsage)
return cmd
}
@@ -209,6 +223,7 @@ type rangeModes struct {
pool string
secret string
authz string
+ dns string
}
// mintRangeSecrets gives the range a fresh nonce and a fresh database password
@@ -263,11 +278,30 @@ func knownPoolModes() []string {
// knownAuthzModes is validated against the whole set for the same reason as
// the other three: `collapses` is the DEFAULT here, so a typo would hand the
// experiment a correct caller and report the defect as absent.
+// knownDNSModes is validated by name for the same reason as the rest: the
+// default leaves the gateway re-resolving, so a typo would hand `dns.ttl_rotate`
+// the one target shape that always follows a rotation and the pinning direction
+// would never be measured.
+// reresolveFor turns the mode into the value the gateway's own environment
+// carries, which is a bare boolean string the fixture reads.
+func reresolveFor(mode string) string {
+ if mode == rangelab.DNSPinned {
+ return "false"
+ }
+
+ return "true"
+}
+
+func knownDNSModes() []string {
+ return []string{rangelab.DNSReresolve, rangelab.DNSPinned}
+}
+
func knownAuthzModes() []string {
return []string{
rangelab.AuthzDistinguishes,
rangelab.AuthzCollapses,
rangelab.AuthzRetries,
+ rangelab.AuthzCached,
}
}
@@ -357,9 +391,43 @@ func validateModes(failMode string, modes rangeModes) error {
modes.authz, strings.Join(knownAuthzModes(), listSep))
}
+ if !slices.Contains(knownDNSModes(), modes.dns) {
+ return fmt.Errorf(errUnknownDNSMode, ErrUnknownDNSMode,
+ modes.dns, strings.Join(knownDNSModes(), listSep))
+ }
+
return nil
}
+// alsoBoundOn is every address BESIDES loopback that a port has to be free on.
+// Four of the tiers bind the Docker bridge gateway rather than loopback, so a
+// port khaos proved free on 127.0.0.1 can still be taken where they will bind
+// it. A daemon that will not answer is not fatal: the free check degrades to
+// loopback and the tier's own bind refusal names the port.
+func alsoBoundOn(ctx context.Context, work *workspace.Workspace) []string {
+ gateway, err := work.Docker.BridgeGateway(ctx)
+ if err != nil {
+ return nil
+ }
+
+ return []string{gateway}
+}
+
+// sayRangeIsUp is the operator-facing summary, split out so the function that
+// builds the range is about building it.
+func sayRangeIsUp(out io.Writer, failMode string, port int, withDB bool) {
+ event(out, markOK, theme.StatusOK, msgRangeUp)
+ note(out, fmt.Sprintf(msgFailMode, failMode))
+ note(out, fmt.Sprintf(msgPortHint, port))
+
+ if withDB {
+ note(out, msgStatefulUp)
+ }
+
+ note(out, msgTokenHint)
+ fmt.Fprintln(out)
+}
+
func runRangeUp(
ctx context.Context,
out io.Writer,
@@ -394,9 +462,11 @@ func runRangeUp(
pinned := port != 0
var lab rangelab.Lab
+ also := alsoBoundOn(ctx, work)
+
for attempt := range portAttempts {
// Only the direct gateway's port is pinnable; the others are plumbing.
- ports, err := rangelab.FreePorts()
+ ports, err := rangelab.FreePorts(also...)
if err != nil {
return err
}
@@ -424,6 +494,7 @@ func runRangeUp(
modes.pool,
modes.secret,
modes.authz,
+ reresolveFor(modes.dns),
withDB,
)
if err == nil {
@@ -437,15 +508,7 @@ func runRangeUp(
note(out, fmt.Sprintf(msgPortRetry, port))
}
- event(out, markOK, theme.StatusOK, msgRangeUp)
- note(out, fmt.Sprintf(msgFailMode, failMode))
- note(out, fmt.Sprintf(msgPortHint, lab.Port))
- if withDB {
- note(out, msgStatefulUp)
- }
-
- note(out, msgTokenHint)
- fmt.Fprintln(out)
+ sayRangeIsUp(out, failMode, lab.Port, withDB)
// Read back from the containers, which are the only authority on where
// they will actually dial and which issuer they will trust.
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go
index ab6147dc..3a74753f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go
@@ -74,6 +74,21 @@ func (c *Client) Arm(ctx context.Context, trap Trap) (Stats, error) {
return c.call(ctx, http.MethodPost, PathArm, body)
}
+// Rotate moves a name's address in the zone. A non-2xx is an error rather than
+// a silent no-op, which is what lets a failed rotation VOID the run instead of
+// leaving khaos measuring a name nobody moved.
+func (c *Client) Rotate(
+ ctx context.Context,
+ name, address string,
+) (Stats, error) {
+ body, err := json.Marshal(Rotation{Name: name, Address: address})
+ if err != nil {
+ return Stats{}, err
+ }
+
+ return c.call(ctx, http.MethodPost, PathRotate, body)
+}
+
func (c *Client) Disarm(ctx context.Context) (Stats, error) {
return c.call(ctx, http.MethodPost, PathDisarm, nil)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go
index af79114a..2edc8d19 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go
@@ -26,6 +26,7 @@ const (
PathArm = "/arm"
PathDisarm = "/disarm"
+ PathRotate = "/rotate"
PathState = "/state"
PathHealthz = "/healthz"
@@ -65,6 +66,7 @@ func (r *Resolver) ServeControl(listen string) error {
mux.HandleFunc(PathState, r.guard(r.handleState))
mux.HandleFunc(PathArm, r.guard(r.handleArm))
mux.HandleFunc(PathDisarm, r.guard(r.handleDisarm))
+ mux.HandleFunc(PathRotate, r.guard(r.handleRotate))
server := &http.Server{
Handler: mux,
@@ -152,6 +154,39 @@ func (r *Resolver) handleState(w http.ResponseWriter, _ *http.Request) {
r.writeJSON(w, r.Stats())
}
+// handleRotate moves a name's address in the zone and echoes the state, whose
+// Zone map is read back rather than remembered -- so khaos's observable can name
+// the address the rotation replaced rather than the one it asked for.
+func (r *Resolver) handleRotate(w http.ResponseWriter, req *http.Request) {
+ var rotation Rotation
+
+ decoder := json.NewDecoder(
+ http.MaxBytesReader(w, req.Body, controlBodyLimit),
+ )
+ if err := decoder.Decode(&rotation); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ previous, err := r.Rotate(rotation.Name, rotation.Address)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusUnprocessableEntity)
+ return
+ }
+
+ stats := r.Stats()
+ stats.Rotated = previous
+
+ r.writeJSON(w, stats)
+}
+
+// Rotation is one zone change, and it is a different request shape from a Trap
+// on purpose: arming a fault and moving the zone are different acts.
+type Rotation struct {
+ Name string `json:"name"`
+ Address string `json:"address"`
+}
+
// handleArm installs a fault and echoes the resulting state. The echo says
// which branch was chosen, not what a target would receive, so it is never
// what a fault is confirmed by.
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go
index 17c6b178..d8e9d93a 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go
@@ -18,6 +18,10 @@ const (
testUpstream = "upstream.elsewhere.test."
testUpAddr = "10.88.88.8"
loopbackAny = "127.0.0.1:0"
+
+ // Distinct from DefaultWrongAddress so the oracle census can tell an armed
+ // rotation from an armed redirect.
+ testRotatedAddress = "10.99.99.9"
)
// testUpstreamServer is a resolver khaos is NOT authoritative for, so the
@@ -116,6 +120,13 @@ func startResolver(t *testing.T) (*Resolver, *Client) {
testControl,
)
+ // The zone SNAPSHOT, taken before any fault, exactly as the CLI takes it
+ // when it attaches to a running resolver. Without it the oracle has no
+ // record of what a name answered before khaos touched it, which the
+ // rotation's confirmation needs and which wrong-address's observable has
+ // been printing as "unknown" in every test until now.
+ client.Zone = resolver.Zone()
+
return resolver, client
}
@@ -152,6 +163,13 @@ func deliverableKinds() []Kind {
func trapFor(kind Kind) Trap {
trap := Trap{Kind: kind, Host: testHost}
+ // A rotation needs a destination, and a DIFFERENT one from wrong-address's
+ // default: asking the wrong-address oracle about an armed rotation has to
+ // REFUTE, and it cannot if both move the name to the same place.
+ if kind == AddressRotated {
+ trap.Address = testRotatedAddress
+ }
+
if kind == SlowResponse {
// Well above the oracle's differential margin and well under the
// embedded resolver's ceiling, so the control is not deciding on noise.
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go
index 7bfb475f..1028ec37 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go
@@ -29,11 +29,13 @@ const (
observeServFail = "answering SERVFAIL for %s while still resolving %s"
observeSlow = "answering %s correctly after %s, against %s for the control on the same resolver"
observeWrong = "answering %s with %v instead of %s"
- observeChain = "answering %s with a %d-hop CNAME ending at %s"
- observeLoop = "answering %s with a CNAME cycle through %v"
- observeTrunc = "setting the truncation bit for %s over UDP with an empty answer, and serving %v completely over TCP"
- observeTruncNo = "setting the truncation bit for %s with an empty answer over UDP and over TCP, so no transport ever completes"
- observeTTL = "answering %s with a TTL of %d, against %d for %s on the same resolver"
+
+ observeRotated = "%s now answers %v where the zone served %s before khaos moved it, from the resolver's own honest path rather than from a fault, so a target that asks again follows the rotation"
+ observeChain = "answering %s with a %d-hop CNAME ending at %s"
+ observeLoop = "answering %s with a CNAME cycle through %v"
+ observeTrunc = "setting the truncation bit for %s over UDP with an empty answer, and serving %v completely over TCP"
+ observeTruncNo = "setting the truncation bit for %s with an empty answer over UDP and over TCP, so no transport ever completes"
+ observeTTL = "answering %s with a TTL of %d, against %d for %s on the same resolver"
refuteControl = "the control name %s did not resolve through this resolver (%v), so the resolver is broken rather than lying"
refuteNoFault = "%s still resolves to %v through this resolver, so nothing is armed against it"
@@ -44,6 +46,9 @@ const (
refuteSlowWrong = "%s answered %v late instead of %s, so the answer was changed as well as delayed and this fault changes only the timing"
refuteWrong = "%s answered %v, which does not contain %s"
refuteWrongIndirect = "%s answered the right address through CNAMEs %v: that is the chain fault, not a direct redirect"
+ refuteRotated = "%s answered %v, which does not contain the address khaos rotated it to, %s"
+ refuteRotatedUnknown = "%s answered the rotated address and khaos holds no record of what the zone served before the move, so it cannot show a rotation happened rather than a name that always pointed there"
+ refuteRotatedSame = "%s answered %s both before and after the rotation, so nothing moved"
refuteChain = "%s answered %d CNAME hops, and a chain needs %d"
refuteChainCycle = "%s answered CNAMEs %v, which cycle: that is the loop fault, not a chain"
refuteTruncReach = "%s could not be reached over TCP at all (%v), which is an unreachable resolver rather than one that truncates on both transports"
@@ -116,6 +121,9 @@ func (c *Client) Observe(trap Trap) (string, bool, error) {
case TTLZero, TTLLong:
return ttlObservation(trap, faulted, control, c.ControlName)
+ case AddressRotated:
+ return rotatedObservation(trap, faulted, c.honest(trap.Host))
+
default:
return refutation(refuteUnknown, string(trap.Kind))
}
@@ -178,6 +186,48 @@ func wrongObservation(
return fmt.Sprintf(observeWrong, trap.Host, addresses, honest), true, nil
}
+// rotatedObservation is the rotation's own observable, and it is DISTINCT from
+// wrong-address's on purpose: that one answers wrongly while the zone stays
+// put, and this one moved the zone, so the honest path serves the new address
+// too. Naming both addresses is what tells an operator which act khaos
+// performed.
+//
+// The snapshot the client holds was taken before the fault, so it is the
+// address the rotation replaced. Requiring the two to DIFFER is the read-back
+// half of the confirmation: Rotate already refuses a move to the address a name
+// already answers with, and this proves the resolver served the move.
+func rotatedObservation(
+ trap Trap,
+ faulted answer,
+ before string,
+) (string, bool, error) {
+ addresses := faulted.addresses()
+ want := trap.address()
+
+ if !slices.Contains(addresses, want) {
+ return refutation(refuteRotated, trap.Host, addresses, want)
+ }
+
+ if hops := faulted.cnames(); len(hops) > 0 {
+ return refutation(refuteWrongIndirect, trap.Host, hops)
+ }
+
+ if before == "" {
+ return refutation(refuteRotatedUnknown, trap.Host)
+ }
+
+ if before == want {
+ return refutation(refuteRotatedSame, trap.Host, want)
+ }
+
+ return fmt.Sprintf(
+ observeRotated,
+ trap.Host,
+ addresses,
+ before,
+ ), true, nil
+}
+
func (c *Client) honest(host string) string {
if c.Zone == nil {
return ""
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go
index 691eb29b..8a8d86d2 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go
@@ -142,10 +142,25 @@ func rettl(msg *dns.Msg, ttl uint32) *dns.Msg {
// validAddress refuses a malformed WrongAddress magnitude before the trap arms,
// where it would answer an empty NOERROR and read as a target that resolved
// nothing.
+// validAddress requires IPv4, and that is narrower than "parses as an IP" for a
+// measured reason. Every khaos probe asks for an A record, and addressAnswer
+// answers an empty NOERROR when the family asked for is not the family of the
+// address it was given -- correctly, since inventing an A record from an IPv6
+// address builds nothing (measured: `dns: bad A A`). So an IPv6 address passes
+// a parse check, arms cleanly, and then makes khaos serve an EMPTY ANSWER while
+// reporting that it armed a redirect. Refusing it here means the operator finds
+// out when they ask for it rather than reading a verdict about a fault khaos
+// never served.
func validAddress(address string) error {
- if net.ParseIP(address) == nil {
+ parsed := net.ParseIP(address)
+ if parsed == nil {
return fmt.Errorf(errBadAddress, ErrBadAddress, address)
}
+
+ if parsed.To4() == nil {
+ return fmt.Errorf(errNotV4, ErrBadAddress, address)
+ }
+
return nil
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go
index 324ba438..e6a472e7 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go
@@ -47,6 +47,7 @@ const (
errControlName = "%w: %q is not in the zone, so a confirmation could never use it as a control"
errFaultedHost = "%w: %s is the confirmation control and cannot also be the faulted name"
errOutsideZone = "%w: %s is not a name this resolver is authoritative for (zone: %s)"
+ errSameAddress = "%w: %s already answers with %s, so rotating it there changes nothing and khaos would measure a rotation that never happened"
errNoZoneTarget = "%w: this resolver was given no confirmation control name"
)
@@ -70,6 +71,10 @@ var ErrControlIsHost = errors.New(
"the confirmation control cannot be the name under fault",
)
+var ErrSameAddress = errors.New(
+ "a rotation to the address a name already answers with is not a rotation",
+)
+
var ErrOutsideZone = errors.New(
"this resolver is not authoritative for that name",
)
@@ -102,6 +107,7 @@ type Resolver struct {
mu sync.Mutex
trap Trap
+ rotated rotation
servers []*dns.Server
arrival map[string]int64
queries map[string]int64
@@ -311,6 +317,26 @@ func (r *Resolver) Arm(trap Trap) error {
}
}
+ // The one trap whose arming moves the ZONE rather than installing a wrong
+ // answer, so the honest path serves the new address afterwards and a target
+ // that asks again follows it. The previous address is recorded here, which
+ // is what makes Disarm a restore rather than a guess.
+ // INERT skips the rotation and keeps everything else, which is the whole
+ // point of the control: an inert arm that still moved the zone would be
+ // confirmed by an oracle reading the served answer, and the one negative
+ // control this package relies on would pass against a mutation it was
+ // built to catch. Measured -- it did.
+ if trap.Kind == AddressRotated && !trap.Inert {
+ previous, err := r.Rotate(trap.Host, trap.Address)
+ if err != nil {
+ return err
+ }
+
+ r.mu.Lock()
+ r.rotated = rotation{name: trap.Host, from: previous}
+ r.mu.Unlock()
+ }
+
r.mu.Lock()
r.trap = trap
r.mu.Unlock()
@@ -338,10 +364,74 @@ func (r *Resolver) inZone(host string) error {
)
}
+// Rotate changes the ADDRESS a name answers with, which is a different act from
+// arming a fault: a fault is a wrong answer the resolver gives while the zone
+// stays put, and this moves the zone itself. The honest answer is served from
+// the same table afterwards, so a target that asks again gets the new address
+// and nothing else about the resolver changes.
+//
+// It is the operational act an operator performs for DNS-based failover or
+// endpoint revocation, and the finding khaos measures is whether it reached the
+// target at all.
+func (r *Resolver) Rotate(name, address string) (string, error) {
+ if err := validAddress(address); err != nil {
+ return "", err
+ }
+
+ canonical := dns.CanonicalName(name)
+
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ previous, ok := r.zone[canonical]
+ if !ok {
+ names := make([]string, 0, len(r.zone))
+ for zoneName := range r.zone {
+ names = append(names, zoneName)
+ }
+
+ return "", fmt.Errorf(
+ errOutsideZone,
+ ErrOutsideZone,
+ name,
+ joinWith(sorted(names), ", "),
+ )
+ }
+
+ if previous == address {
+ return "", fmt.Errorf(errSameAddress, ErrSameAddress, name, address)
+ }
+
+ r.zone[canonical] = address
+
+ return previous, nil
+}
+
+// Disarm removes the fault and RESTORES a rotation, because khaos has to leave
+// the zone as it found it: a rotation that outlived the run would be khaos
+// changing an operator's resolution permanently.
func (r *Resolver) Disarm() {
r.mu.Lock()
+ moved := r.rotated
+ r.rotated = rotation{}
r.trap = Trap{}
r.mu.Unlock()
+
+ if moved.name == "" {
+ return
+ }
+
+ // Best effort and deliberately not reported: Disarm has no error channel,
+ // and the restore can only fail for a name that is no longer in the zone,
+ // which this resolver has no path to.
+ _, _ = r.Rotate(moved.name, moved.from)
+}
+
+// rotation is what a restore needs: the name and the address it answered with
+// before khaos moved it.
+type rotation struct {
+ name string
+ from string
}
func (r *Resolver) armed() Trap {
@@ -377,6 +467,10 @@ type Stats struct {
// address the fault replaced.
Zone map[string]string `json:"zone"`
+ // Rotated is the address a rotation REPLACED, echoed by that one route so
+ // khaos's observable can name what the zone used to say.
+ Rotated string `json:"rotated,omitempty"`
+
// Only the queries khaos was AUTHORITATIVE for, keyed by source and name.
// An arrival count alone goes non-zero for a target resolving anything else.
Queries map[string]int64 `json:"queries"`
@@ -502,6 +596,13 @@ func (r *Resolver) serveFault(
Address: trap.address(),
}, chainAnswer)
+ case AddressRotated:
+ // The HONEST answer, deliberately: this trap moved the zone, so the
+ // honest path already serves the new address. Synthesizing from the
+ // trap instead would make it a wrong-answer fault and khaos would be
+ // serving a redirect while reporting a rotation.
+ return r.synthesize(req, question, honest, addressAnswer)
+
case CNAMELoop:
return r.synthesize(req, question, honest, loopAnswer)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/rotate_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/rotate_test.go
new file mode 100644
index 00000000..d20b1f0e
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/rotate_test.go
@@ -0,0 +1,107 @@
+// ©AngelaMos | 2026
+// rotate_test.go
+
+package dnsrv
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/miekg/dns"
+)
+
+// A rotation moves the ZONE, which is a different act from arming a fault: a
+// fault is a wrong answer given while the zone stays put. Research 05 section
+// 6.5 is the polarity this serves.
+func TestARotationChangesWhatTheNameAnswersWith(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ const rotated = "10.244.244.77"
+
+ previous, err := resolver.Rotate(testHost, rotated)
+ if err != nil {
+ t.Fatalf("rotate: %v", err)
+ }
+
+ if previous != testAddress {
+ t.Fatalf(
+ "the rotation reported %q as the address it replaced, want %q: khaos's observable names what the zone USED to say",
+ previous,
+ testAddress,
+ )
+ }
+
+ if got := resolver.Stats().Zone[dns.CanonicalName(testHost)]; got != rotated {
+ t.Fatalf("after the rotation the zone says %q, want %q", got, rotated)
+ }
+}
+
+// A rotation to the address a name already answers with is not a rotation, and
+// accepting it would let khaos measure a change that never happened -- the same
+// reason the credential tier's inert control must be REJECTED by confirmation.
+func TestARotationThatChangesNothingIsRefused(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ if _, err := resolver.Rotate(
+ testHost,
+ testAddress,
+ ); !errors.Is(err, ErrSameAddress) {
+ t.Fatalf("rotating to the current address returned %v", err)
+ }
+}
+
+// And a name outside the zone, for the same reason arming one is refused: it
+// would succeed, answer nothing, and report a survival against nothing.
+func TestARotationOutsideTheZoneIsRefused(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ if _, err := resolver.Rotate(
+ "not-in-this-zone.invalid.",
+ "10.244.244.77",
+ ); !errors.Is(err, ErrOutsideZone) {
+ t.Fatalf("rotating a name outside the zone returned %v", err)
+ }
+}
+
+func TestARotationToAnUnusableAddressIsRefused(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ for _, address := range []string{"", "not-an-address", "::1"} {
+ if _, err := resolver.Rotate(testHost, address); err == nil {
+ t.Fatalf("rotating to %q was accepted", address)
+ }
+ }
+}
+
+// The control surface carries it, nonce-guarded like every other route, and a
+// failure is an ERROR rather than a silent no-op: that is what lets a failed
+// rotation void a run instead of leaving khaos measuring a name nobody moved.
+func TestTheControlSurfaceRotatesAndRefusesWithoutTheNonce(t *testing.T) {
+ _, client := startResolver(t)
+
+ stats, err := client.Rotate(context.Background(), testHost, "10.244.244.78")
+ if err != nil {
+ t.Fatalf("rotate through the control surface: %v", err)
+ }
+
+ if stats.Rotated != testAddress {
+ t.Fatalf(
+ "the echo reports %q as the replaced address, want %q",
+ stats.Rotated,
+ testAddress,
+ )
+ }
+
+ wrong := NewClient(client.DNS, client.Control, "not-the-nonce", testControl)
+
+ if _, err := wrong.Rotate(
+ context.Background(),
+ testHost,
+ "10.244.244.79",
+ ); err == nil {
+ t.Fatal(
+ "a rotation was accepted without the nonce, so anything that can reach the control surface can move the zone mid-run",
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go
index 8d624b1b..413ea659 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go
@@ -57,6 +57,15 @@ const (
TTLZero Kind = "ttl-zero"
TTLLong Kind = "ttl-long"
+ // AddressRotated MOVES THE ZONE rather than answering it wrongly, which is
+ // what makes it the only trap whose arming changes what the HONEST path
+ // serves. It is the operational act behind DNS-based failover and endpoint
+ // revocation, and the finding is whether it reached the target at all --
+ // research 05 section 6.5 carries the polarity, which is NOT inverted: the
+ // rotation points at the same endpoint `wrong-address` uses, so a target
+ // that follows it starts trusting a host on the strength of a name.
+ AddressRotated Kind = "address-rotated"
+
// REFUSED BY KHAOS: the embedded resolver rewrites both to SERVFAIL, and a
// real library branches on the rcode, so delivering one under its own name
// would be a wrong measurement rather than a fault.
@@ -93,6 +102,8 @@ const (
describeTTLZero = "answer %s correctly with a TTL of zero, so nothing may cache it"
describeTTLLong = "answer %s correctly with a TTL of %d seconds, so a cache pins it for a week"
+ describeRotated = "MOVE %s's dependency to %s in the zone itself, the way an operator performs a DNS failover or an endpoint revocation, and measure whether the change reaches it at all"
+
describeRefused = "answer %s REFUSED, which khaos cannot deliver"
describeTimeout = "drop %s's query entirely, which khaos cannot deliver"
@@ -105,6 +116,8 @@ const (
errDelayRange = "%w: %s was asked for a %s delay, and the ceiling is %s"
errNoHost = "%w: %s needs the dependency's name, and none was given"
errBadAddress = "%w: %q is not an IP address"
+ errNoRotation = "%w: %s needs the address to move the name TO; a rotation with no destination is not a rotation"
+ errNotV4 = "%w: %q is an IPv6 address, and this resolver answers the A queries khaos's probes send; an IPv6 answer to an A query is an empty NOERROR, so khaos would serve nothing while reporting that it armed a redirect"
// Each names the mechanism, what the target would really have received, and
// what to ask for instead.
@@ -128,6 +141,10 @@ var ErrDelayRange = errors.New(
var ErrNoHost = errors.New("this DNS fault needs a name to act on")
+var ErrNoRotation = errors.New(
+ "a rotation needs the address it moves the name to",
+)
+
var ErrBadAddress = errors.New("that is not an IP address")
// Trap travels through the journal, so every field has to be enough to
@@ -159,6 +176,7 @@ func Kinds() []string {
string(TruncationNoTCP),
string(TTLZero),
string(TTLLong),
+ string(AddressRotated),
string(Refused),
string(Timeout),
}
@@ -211,6 +229,8 @@ func (t Trap) Describe(target string) string {
return fmt.Sprintf(describeTTLZero, target)
case TTLLong:
return fmt.Sprintf(describeTTLLong, target, LongTTL)
+ case AddressRotated:
+ return fmt.Sprintf(describeRotated, target, t.address())
case Refused:
return fmt.Sprintf(describeRefused, target)
case Timeout:
@@ -271,6 +291,19 @@ func (t Trap) ValidateShape() error {
}
}
+ // A rotation needs its destination at SHAPE time rather than defaulting to
+ // one, because moving a name to an address khaos chose for it is not an
+ // operational act anybody performs.
+ if t.Kind == AddressRotated {
+ if t.Address == "" {
+ return fmt.Errorf(errNoRotation, ErrNoRotation, string(t.Kind))
+ }
+
+ if err := validAddress(t.Address); err != nil {
+ return err
+ }
+ }
+
return nil
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
index 9a4ac000..9dbe5a94 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go
@@ -31,6 +31,7 @@ const (
errNoNetwork = "%s has no network address at all, so khaos has nothing to dial it on"
errBridge = "read the gateway of the default bridge network: %w"
errNoBridge = "the default bridge network reports no gateway address, so khaos cannot find an address a container can reach it on"
+ errGateways = "list the networks the daemon manages, to find the addresses khaos may bind a listener on: %w"
bridgeNetwork = "bridge"
envPairParts = 2
@@ -256,6 +257,33 @@ func (c *Client) BridgeGateway(ctx context.Context) (string, error) {
return "", errors.New(errNoBridge)
}
+// Gateways is every address DOCKER manages as a network gateway, which is a
+// narrower question than "is this a bridge". `/sys/class/net/<if>/bridge`
+// exists for every kernel bridge device: measured on this host, that includes
+// a CNI pod bridge, a swarm overlay's gateway bridge and forty-nine
+// user-defined ones carrying unrelated projects' containers -- and on a host
+// whose LAN interface is enslaved to a bridge, the routable LAN address itself.
+// khaos binds listeners that answer whatever asks them, so the allowed set has
+// to come from the daemon rather than from sysfs.
+func (c *Client) Gateways(ctx context.Context) ([]string, error) {
+ networks, err := c.api.NetworkList(ctx, client.NetworkListOptions{})
+ if err != nil {
+ return nil, fmt.Errorf(errGateways, err)
+ }
+
+ addresses := []string{}
+
+ for _, network := range networks.Items {
+ for _, config := range network.IPAM.Config {
+ if config.Gateway.IsValid() {
+ addresses = append(addresses, config.Gateway.String())
+ }
+ }
+ }
+
+ return addresses, nil
+}
+
// classify turns a daemon NotFound into ErrContainerGone, so it is only valid
// where the MISSING THING IS THE CONTAINER; NetworkConnect returns NotFound for
// a missing NETWORK, so see classifyNetwork.
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
index 0cf583c1..66000b1d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
@@ -99,6 +99,11 @@ type Shape struct {
ClosedWhen []probe.Disposition
+ // RotatedWhat names a zone rotation, and it is the one observable read as a
+ // finding when it is ZERO: a target that never asked again never saw the
+ // rotation, which is a revocation that did not take effect.
+ RotatedWhat string
+
// The liveness guard for a fault whose landing the probe streams cannot see.
Consulted func() (int64, error)
ConsultedWhat string
@@ -312,6 +317,7 @@ func Build(
CacheRequired: shape.CacheRequired,
DeliveredWhat: shape.DeliveredWhat,
DeliveredHint: shape.DeliveredHint,
+ RotatedWhat: shape.RotatedWhat,
DNSTrap: shape.DNSTrap,
ResolverNetwork: shape.ResolverNetwork,
DNSHost: shape.DNSHost,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go
index 31052bb7..0ff8a9f2 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go
@@ -241,7 +241,10 @@ func dispositionOf(result dbx.Result) (probe.Disposition, string) {
case dbx.Refused, dbx.Locked, dbx.Expired, dbx.MustReset, dbx.Terminated:
return probe.Rejected, result.Detail
case dbx.Unreachable:
- return probe.Unavailable, result.Detail
+ // An engine khaos could not reach ANSWERED nothing, which is not the
+ // same observation as a service answering 503 - and the pooled
+ // experiments declare the second as their fail-closed outcome.
+ return probe.Unreachable, result.Detail
default:
return probe.ProbeFailed, result.Detail
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go
index 5122396d..03f5b7ee 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go
@@ -19,6 +19,7 @@ const (
NameDNSTruncationNoTCP = "dns.truncation_no_tcp"
NameDNSTTLZero = "dns.ttl_zero"
NameDNSTTLLong = "dns.ttl_long"
+ NameDNSTTLRotate = "dns.ttl_rotate"
NameDNSRefused = "dns.refused"
NameDNSTimeout = "dns.timeout"
)
@@ -44,6 +45,8 @@ const (
summaryTTLLong = "answer the dependency's name correctly with a week-long TTL and measure whether the target honours it, which is how long a stale answer would keep being trusted after the dependency moved"
+ summaryTTLRotate = "MOVE the dependency's address in the zone, the way an operator performs a DNS failover or an endpoint revocation, and measure whether the change reaches the target at all -- a client still talking to the address that was rotated away is a revocation that never took effect"
+
summaryRefused = "REFUSED through Docker's embedded resolver, which khaos refuses to approximate; the resolver rewrites it to SERVFAIL and a real library branches on the rcode"
summaryTimeout = "a dropped query through Docker's embedded resolver, which khaos refuses to approximate; the resolver answers SERVFAIL itself at a measured 4.001s and a client never sees a socket timeout"
@@ -105,6 +108,13 @@ var dnsCatalogue = []dnsSpec{
fault: dnsrv.TTLLong,
needsCache: true,
},
+ {
+ name: NameDNSTTLRotate,
+ summary: summaryTTLRotate,
+ fault: dnsrv.AddressRotated,
+ extra: []Param{dnsAddressParam},
+ rotates: true,
+ },
{
name: NameDNSRefused,
summary: summaryRefused,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go
index b72f7d2f..189c8fcd 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go
@@ -42,6 +42,8 @@ const (
deliveredResolve = "re-resolve its dependency's name"
+ rotatedWhat = "moved %s to %s in the zone"
+
// What to check when a DNS fault was armed, confirmed and never met. It
// lives here rather than in the verdict's shared sentence because the only
// other tier carrying that guard reaches it for an unrelated reason.
@@ -96,6 +98,12 @@ type dnsSpec struct {
// than a clean survival it never earned.
needsCache bool
+ // rotates MOVES the zone rather than answering it wrongly, and it is exempt
+ // from the delivery guard for the same reason the TTL pair is: a target
+ // that never asks again IS the measurement here, not a void run. Research
+ // 05 section 6.5 carries the polarity.
+ rotates bool
+
extra []Param
}
@@ -160,6 +168,19 @@ func buildDNS(spec dnsSpec) func(*Params) (Shape, error) {
shape.CacheRequired = cacheRequiredTTL
}
+ // The rotation's own observable. A target that never asks again did not
+ // see the rotation, which is a finding khaos must state and must not
+ // call a bypass: it is still talking to the honest endpoint it used
+ // during the baseline, so no credential got through that would not
+ // have.
+ if spec.rotates {
+ shape.RotatedWhat = fmt.Sprintf(
+ rotatedWhat,
+ host,
+ trap.Address,
+ )
+ }
+
return shape, nil
}
}
@@ -206,7 +227,7 @@ func heldCaveatFor(spec dnsSpec) string {
// deliveredFor is derived from the same needsCache declaration that turns on the
// cache reading, so the two cannot disagree about which faults are about caching.
func deliveredFor(spec dnsSpec) string {
- if spec.needsCache {
+ if spec.needsCache || spec.rotates {
return ""
}
@@ -214,7 +235,7 @@ func deliveredFor(spec dnsSpec) string {
}
func hintFor(spec dnsSpec) string {
- if spec.needsCache {
+ if spec.needsCache || spec.rotates {
return ""
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
index 24d4c539..8caed1aa 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
@@ -314,6 +314,7 @@ func (e Engine) measureFault(
observation.Delivered = after - before
observation.DeliveredWhat = spec.DeliveredWhat
observation.DeliveredHint = spec.DeliveredHint
+ observation.RotatedWhat = spec.RotatedWhat
observation.Refusals = refusedAfter - refusedBefore
// NOT folded into CounterUnreadable, which voids the run: this count feeds
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
index ff0e5d36..21bb1c5b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -157,6 +157,10 @@ type Experiment struct {
Consulted func() (int64, error)
ConsultedWhat string
+ // RotatedWhat names a zone rotation whose arrival khaos measures by whether
+ // the target asked again at all.
+ RotatedWhat string
+
// Refusals is the count of definitive denials the dependency issued, which
// only the credential under test is ever answered with.
Refusals func() (int64, error)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
index 8450e39c..b5adbaed 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go
@@ -231,11 +231,16 @@ func TestAnUnreachableEndpointIsNotMistakenForARejection(t *testing.T) {
outcome := spec.Probes.Invalid.Run(context.Background())
- if outcome.Disposition != probe.Unavailable {
+ // Unreachable rather than Unavailable since the dispositions were split:
+ // nothing ANSWERED here. Reading it as a rejection would make an outage
+ // look like a control doing its job, and reading it as an answered 503
+ // would let a control that declares a 503 its fail-closed outcome cover a
+ // target that had died.
+ if outcome.Disposition != probe.Unreachable {
t.Fatalf(
- "a connection failure was classified as %s; reading it as a rejection would "+
- "make an outage look like a control doing its job",
+ "a connection failure was classified as %s, want %s",
outcome.Disposition,
+ probe.Unreachable,
)
}
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/rotation_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/rotation_test.go
new file mode 100644
index 00000000..919e738b
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/rotation_test.go
@@ -0,0 +1,169 @@
+// ©AngelaMos | 2026
+// rotation_test.go
+
+package experiment
+
+import (
+ "strings"
+ "testing"
+)
+
+const movedWhat = "moved dependency.khaos.test. to 10.244.244.20 in the zone"
+
+func rotated(queriesDuringFault int64) Observation {
+ return Observation{
+ Baseline: healthyBaseline(),
+ Fault: heldFaultWindow(),
+ Recovery: healthyRecovery(),
+ FaultConfirmed: true,
+ RotatedWhat: movedWhat,
+ ConsultedWhat: ConsultsResolver,
+ Consulted: 2,
+ Delivered: queriesDuringFault,
+ }
+}
+
+// The whole question for the one fault that MOVES the zone: did the change
+// reach the target at all. khaos states it in BOTH directions, so a clean run
+// is an answer rather than silence.
+func TestTheRotationIsReportedInBothDirections(t *testing.T) {
+ followed := Judge(rotated(74), dnsPolarity)
+
+ if !hasReason(followed, "so the rotation reached it") {
+ t.Fatalf(
+ "a target that asked again was not told the rotation reached it: %v",
+ followed.Reasons,
+ )
+ }
+
+ unseen := Judge(rotated(0), dnsPolarity)
+
+ if !hasReason(unseen, "ZERO during the fault window") {
+ t.Fatalf(
+ "a target that never asked again was not told the rotation missed it: %v",
+ unseen.Reasons,
+ )
+ }
+
+ // Both carry the counts and the act, or an operator cannot audit either.
+ for _, verdict := range []Verdict{followed, unseen} {
+ if !hasReason(verdict, movedWhat) {
+ t.Fatalf(
+ "the reason does not name what khaos moved: %v",
+ verdict.Reasons,
+ )
+ }
+ }
+}
+
+// AND IT IS NOT A BYPASS. No credential got through that would not have before:
+// the target is still reaching the endpoint it reached during the baseline.
+// Reporting PERMITTED here would be khaos claiming a bypass it did not observe,
+// which is the overstatement this family's polarity exists to prevent.
+func TestARotationThatMissedTheTargetIsNotCalledABypass(t *testing.T) {
+ verdict := Judge(rotated(0), dnsPolarity)
+
+ if verdict.Direction != Held {
+ t.Fatalf(
+ "a target that never saw the rotation was judged %s: %v",
+ verdict.Direction,
+ verdict.Reasons,
+ )
+ }
+
+ if verdict.Defect {
+ t.Fatalf("it was reported as a defect: %v", verdict.Reasons)
+ }
+
+ if !hasReason(verdict, "NOT a credential bypass") {
+ t.Fatalf(
+ "khaos did not say it was reporting no bypass, so a reader has to infer it: %v",
+ verdict.Reasons,
+ )
+ }
+}
+
+// And an experiment that performed no rotation must be told nothing about one,
+// or every other fault in every family gains a sentence about a zone it never
+// touched.
+func TestAnExperimentThatRotatedNothingSaysNothingAboutIt(t *testing.T) {
+ observation := Observation{
+ Baseline: healthyBaseline(),
+ Fault: heldFaultWindow(),
+ Recovery: healthyRecovery(),
+ FaultConfirmed: true,
+ }
+
+ for _, reason := range Judge(observation, dnsPolarity).Reasons {
+ if strings.Contains(reason, "in the zone") ||
+ strings.Contains(reason, "rotation") {
+ t.Fatalf(
+ "an experiment that rotated nothing was told about one: %q",
+ reason,
+ )
+ }
+ }
+}
+
+// The rotation is EXEMPT from the delivery guard, and that exemption is the
+// whole design: window one's guard reads "the target never met the fault" as a
+// void run, and here not asking again IS the measurement.
+func TestTheRotationIsExemptFromTheDeliveryGuard(t *testing.T) {
+ // The BUILT experiment, not a hand-made observation: the exemption lives in
+ // the builder, so an observation written by hand proves nothing about it.
+ // Measured by gutting, which stayed green against the hand-made version.
+ definition, err := Lookup(NameDNSTTLRotate)
+ if err != nil {
+ t.Fatalf("lookup: %v", err)
+ }
+
+ spec, err := Build(definition.Name, fill(definition.Params), fastTiming)
+ if err != nil {
+ t.Fatalf("build: %v", err)
+ }
+
+ if spec.DeliveredWhat != "" {
+ t.Fatalf(
+ "the rotation declares the delivery guard (%q), which aborts the run on the very observation this experiment exists to measure",
+ spec.DeliveredWhat,
+ )
+ }
+
+ if spec.RotatedWhat == "" {
+ t.Fatal(
+ "the rotation declares no observable of its own, so a run that measured it would say nothing",
+ )
+ }
+
+ // And the rest of the family still DOES declare it, or the exemption was
+ // applied to the whole family rather than to the one fault that earns it.
+ guarded := 0
+
+ for _, other := range Catalog() {
+ if other.Family != FamilyDNS || other.Name == NameDNSTTLRotate {
+ continue
+ }
+
+ built, err := Build(other.Name, fill(other.Params), fastTiming)
+ if err != nil {
+ t.Fatalf("build %s: %v", other.Name, err)
+ }
+
+ if built.DeliveredWhat != "" {
+ guarded++
+ }
+ }
+
+ if guarded == 0 {
+ t.Fatal(
+ "no other DNS fault carries the delivery guard, so the exemption was applied to the whole family",
+ )
+ }
+
+ if Judge(rotated(0), dnsPolarity).Epistemic != Passed {
+ t.Fatalf(
+ "a target that never asked again produced %s rather than a readable verdict",
+ Judge(rotated(0), dnsPolarity).Epistemic,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go
index 605423be..b6f64fb3 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go
@@ -288,6 +288,67 @@ func TestA503RefusalEndsTheRunOfAcceptances(t *testing.T) {
}
}
+// A target that DIED under the fault is not a target that failed closed, and
+// the two were one disposition. This experiment declares a 503 as its refusal,
+// so a crashed target was reported `deviated / DENIED` with "the valid
+// credential stopped being accepted" -- when it stopped ANSWERING, and this
+// fault takes nothing away from the target that could explain a crash.
+func TestACrashedTargetIsNotReadAsFailingClosed(t *testing.T) {
+ observation := shifted(probe.Rejected, probe.Rejected)
+ observation.ClosedWhen = []probe.Disposition{probe.Unavailable}
+
+ // Every probe dies on the connection, in both streams, which is what
+ // `unanswered` reads across.
+ observation.Fault = probe.Window{
+ Valid: stream(
+ probe.Valid,
+ repeat(probe.Unreachable, MinimumPaired*2)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Unreachable, MinimumPaired*2)...),
+ Revoked: stream(
+ probe.Revoked,
+ repeat(probe.Unreachable, MinimumPaired*2)...),
+ }
+
+ verdict := Judge(observation, authzPolarity)
+
+ if verdict.Direction != Crashed {
+ t.Fatalf(
+ "a target that answered nothing at all was judged %s: %v",
+ verdict.Direction,
+ verdict.Reasons,
+ )
+ }
+
+ // And the contrast, on the same declared set: a target ANSWERING 503 has
+ // refused, and that still has to read as this control failing closed.
+ answering := shifted(probe.Rejected, probe.Rejected)
+ answering.ClosedWhen = []probe.Disposition{probe.Unavailable}
+ answering.Fault = probe.Window{
+ Valid: stream(
+ probe.Valid,
+ repeat(probe.Unavailable, MinimumPaired*2)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Rejected, MinimumPaired*2)...),
+ Revoked: stream(
+ probe.Revoked,
+ repeat(probe.Unavailable, MinimumPaired*2)...),
+ }
+
+ if direction := Judge(
+ answering,
+ authzPolarity,
+ ).Direction; direction != Denied {
+ t.Fatalf(
+ "a target answering 503 to everything was judged %s, want %s: it answered, and this control declares that its refusal",
+ direction,
+ Denied,
+ )
+ }
+}
+
// The exposure figure has to describe the SAME set as the count beside it. The
// count is the trailing run; a first-to-last span over the whole window
// includes the period the target spent refusing, and quoting the two in one
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
index b1307862..ab738752 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
@@ -183,6 +183,16 @@ const (
// could not get, and the fault may well be in effect.
reasonShiftUnarmed = "khaos could not establish the second phase, so this run measured what the target does with a definitive deny and nothing about what it does when the dependency cannot answer; those are different branches and a correct caller needs both"
+ // Stated, and deliberately NOT a bypass. No credential got through that
+ // would not have before: the target is still talking to the endpoint it
+ // used during the baseline. Calling it PERMITTED would be khaos claiming a
+ // bypass it did not observe; saying nothing would hide the operational
+ // finding, which is that a DNS-based revocation on this name would not have
+ // taken effect either.
+ reasonRotationUnseen = "khaos %s and the target never asked again: %d queries during the baseline and ZERO during the fault window. The rotation did not reach it, so a DNS-based failover or endpoint revocation on that name would not have taken effect either. This is NOT a credential bypass and khaos reports none -- the target is still reaching the address it was reaching before the rotation"
+
+ reasonRotationFollowed = "khaos %s and the target asked again %d time(s) during the fault window, so the rotation reached it: a DNS-based failover or revocation on that name would take effect"
+
reasonShiftNotDelivered = "the second phase was armed and confirmed, and khaos did not observe the target meeting it: it did not %s once during that window, so it answered every probe from something it had already cached and nothing khaos changed there reached it. Window one treats that observation as a void run rather than a finding, and so does this one"
)
@@ -250,6 +260,13 @@ type Observation struct {
Delivered int64 `json:"delivered,omitempty"`
DeliveredWhat string `json:"delivered_what,omitempty"`
+ // RotatedWhat names a zone rotation khaos performed, and this is the one
+ // count read as a FINDING when it is zero: a target that never asked again
+ // never saw the rotation. Research 05 section 6.5 is the polarity, and the
+ // finding is deliberately NOT a bypass -- the target is still talking to
+ // the endpoint it used during the baseline.
+ RotatedWhat string `json:"rotated_what,omitempty"`
+
// DeliveredHint is what to check, in this fault's own terms. The guard is
// shared and the causes are not: a name the target never resolved again and
// a decision it never asked for again have nothing in common to advise on.
@@ -512,6 +529,10 @@ func finish(
verdict.Reasons = append(verdict.Reasons, storm)
}
+ if rotation := rotationReading(observation); rotation != "" {
+ verdict.Reasons = append(verdict.Reasons, rotation)
+ }
+
// EVERY direction is put to the declared polarity, including HELD. For a
// certificate fault a target that carries on unchanged has ACCEPTED the
// broken certificate, so "nothing looks wrong" is the finding.
@@ -587,6 +608,30 @@ func leaked(observation Observation) []string {
return reasons
}
+// rotationReading states whether a zone rotation reached the target at all,
+// which is the whole question for the one fault that MOVES the zone rather than
+// answering it wrongly. Zero queries during the fault window is the finding;
+// khaos says it in both directions so a clean run is not silence.
+func rotationReading(observation Observation) string {
+ if observation.RotatedWhat == "" {
+ return ""
+ }
+
+ if observation.Delivered == 0 {
+ return sprintf(
+ reasonRotationUnseen,
+ observation.RotatedWhat,
+ observation.Consulted,
+ )
+ }
+
+ return sprintf(
+ reasonRotationFollowed,
+ observation.RotatedWhat,
+ observation.Delivered,
+ )
+}
+
// retryStorm reports the target asking again about a credential the dependency
// has already definitively refused. It is a REASON rather than a direction: no
// credential got through, and what it tells an operator is that their caller
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/bind_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/bind_test.go
new file mode 100644
index 00000000..c6990023
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/bind_test.go
@@ -0,0 +1,44 @@
+// ©AngelaMos | 2026
+// bind_test.go
+
+package idp
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/bindguard"
+)
+
+// This provider mints credentials that are deliberately defective -- `alg:
+// none`, a signature over a zero-length key, a token carrying its own verifying
+// key -- and serves the key set a target resolves them against. It had no bind
+// guard at all while the fault proxy had carried one since M5, so the rule was
+// a property of one tier rather than of khaos. An audit found it.
+func TestAnExposedBindIsRefused(t *testing.T) {
+ for _, addr := range []string{"0.0.0.0:45999", ":45999", "[::]:45999"} {
+ provider, err := New(Config{})
+ if err != nil {
+ t.Fatalf("build a provider: %v", err)
+ }
+
+ err = provider.Listen(addr)
+ if err == nil {
+ bound, _ := provider.Addr()
+ _ = provider.Close()
+ t.Fatalf(
+ "the provider bound %s, which is reachable from outside this host",
+ bound,
+ )
+ }
+
+ if !errors.Is(err, bindguard.ErrUnsafeBind) {
+ t.Fatalf("%s: err = %v, want ErrUnsafeBind", addr, err)
+ }
+ }
+
+ // And loopback MUST still be accepted, or the refusal is a broken listener
+ // rather than a guard. `listening` binds loopback and fails the test if it
+ // is refused, which is the positive half.
+ _ = listening(t)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go
index 97dcde30..547cabc4 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go
@@ -12,6 +12,8 @@ import (
"sync"
"sync/atomic"
"time"
+
+ "github.com/CarterPerez-dev/khaos/internal/bindguard"
)
const (
@@ -43,6 +45,11 @@ const (
shutdownGrace = 2 * time.Second
+ // It mints the credentials khaos's own probes present and serves the key
+ // set a target resolves them against, so it does not belong on an
+ // interface that target cannot reach anyway.
+ SubjectProvider = "an identity provider that mints deliberately defective credentials"
+
errAlreadyListening = "%w: khaos's identity provider is already listening on %s"
errNotListening = "%w: khaos's identity provider is not listening, so it has no address"
errNoRevocable = "%w: this run never issued a revocable credential, so there is nothing to revoke"
@@ -202,6 +209,10 @@ func (p *Provider) clock() time.Time {
// Listen binds the address the target was built to address, not khaos's choice.
func (p *Provider) Listen(addr string) error {
+ if err := bindguard.Check(addr, SubjectProvider); err != nil {
+ return err
+ }
+
// Guard and bind under ONE acquisition, or two callers both bind and the
// second orphans the first listener and its goroutine.
p.mu.Lock()
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
index 0bbd264b..34210db8 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
@@ -28,7 +28,7 @@ const (
settleWindow = 250 * time.Millisecond
- subjectProxy = "a fault proxy in front of a dependency under test"
+ SubjectProxy = "a fault proxy in front of a dependency under test"
probePayload = "GET / HTTP/1.1\r\nHost: khaos.confirm\r\nConnection: close\r\n\r\n"
@@ -78,7 +78,7 @@ type Proxy struct {
// Open binds the proxy and refuses an address that would expose it.
func Open(listenAddr, upstream string) (*Proxy, error) {
- if err := bindguard.Check(listenAddr, subjectProxy); err != nil {
+ if err := bindguard.Check(listenAddr, SubjectProxy); err != nil {
return nil, err
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/bind_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/bind_test.go
new file mode 100644
index 00000000..2492d4a3
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/bind_test.go
@@ -0,0 +1,58 @@
+// ©AngelaMos | 2026
+// bind_test.go
+
+package pki
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/bindguard"
+)
+
+// BOTH listeners, because they are different exposures and each had none. The
+// TLS one presents certificates khaos deliberately broke -- expired, revoked,
+// wrongly named, signed with a weak key -- and the HTTP one serves the CRL and
+// OCSP responses that decide whether a certificate is still good. The fault
+// proxy had carried this guard since M5 and neither of these did, so the rule
+// was a property of one tier rather than of khaos. An audit found it.
+func TestBothListenersRefuseAnExposedBind(t *testing.T) {
+ for _, addr := range []string{"0.0.0.0:45999", ":45999", "[::]:45999"} {
+ t.Run(addr, func(t *testing.T) {
+ revocation := authority_t(t)
+
+ err := revocation.Listen(addr)
+ if err == nil {
+ _ = revocation.Close()
+ t.Fatalf(
+ "the revocation endpoint bound %s, which is reachable from outside this host",
+ addr,
+ )
+ }
+
+ if !errors.Is(err, bindguard.ErrUnsafeBind) {
+ t.Fatalf("Listen(%q) = %v, want ErrUnsafeBind", addr, err)
+ }
+
+ dependency := authority_t(t)
+
+ err = dependency.ListenTLS(addr)
+ if err == nil {
+ _ = dependency.Close()
+ t.Fatalf(
+ "the dependency endpoint bound %s, which is reachable from outside this host",
+ addr,
+ )
+ }
+
+ if !errors.Is(err, bindguard.ErrUnsafeBind) {
+ t.Fatalf("ListenTLS(%q) = %v, want ErrUnsafeBind", addr, err)
+ }
+ })
+ }
+
+ // And loopback MUST still be accepted on both, or the refusal is a broken
+ // tier rather than a guard. `listening` binds both and fails the test if
+ // either is refused.
+ _ = listening(t)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go
index af452e8d..eace378e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go
@@ -12,6 +12,8 @@ import (
"net/http"
"sync"
"time"
+
+ "github.com/CarterPerez-dev/khaos/internal/bindguard"
)
const (
@@ -24,6 +26,14 @@ const (
healthBody = "ok"
maxOCSPBody = 64 << 10
+ // Each listener names what it is, because one sentinel now serves every
+ // listener khaos opens and a refusal has to say which was about to be
+ // exposed. The CA signs for one hostname and the revocation endpoint
+ // decides whether a certificate is still good, so neither belongs on an
+ // interface the target under test cannot reach anyway.
+ SubjectRevocation = "a certificate authority's revocation endpoint"
+ SubjectDependency = "a dependency khaos presents deliberately broken certificates on"
+
errNoListener = "%w: the authority is not listening, so %s cannot be reached"
errBindFailed = "%w: binding %q: %v"
errAlreadyUp = "%w: the authority is already listening on %s"
@@ -65,6 +75,10 @@ func (a *Authority) SetBase(base string) {
}
func (a *Authority) Listen(addr string) error {
+ if err := bindguard.Check(addr, SubjectRevocation); err != nil {
+ return err
+ }
+
a.listener.mu.Lock()
defer a.listener.mu.Unlock()
@@ -109,6 +123,10 @@ func (a *Authority) Listen(addr string) error {
// certificate is resolved per handshake rather than fixed at bind time, so
// arming takes effect on the next connection.
func (a *Authority) ListenTLS(addr string) error {
+ if err := bindguard.Check(addr, SubjectDependency); err != nil {
+ return err
+ }
+
a.listener.mu.Lock()
defer a.listener.mu.Unlock()
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/answered_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/answered_test.go
new file mode 100644
index 00000000..f41f2739
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/answered_test.go
@@ -0,0 +1,111 @@
+// ©AngelaMos | 2026
+// answered_test.go
+
+package probe
+
+import (
+ "context"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+)
+
+// A target ANSWERING that it cannot serve and a target not answering at all are
+// different observations, and they were one disposition. Four experiments
+// declare the first as their fail-closed outcome, so the conflation made a
+// crashed target read as one that had correctly refused.
+func TestA503AndADeadTargetAreDifferentDispositions(t *testing.T) {
+ serving := httptest.NewServer(
+ http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ }),
+ )
+ defer serving.Close()
+
+ answered := NewHTTP(HTTPOptions{
+ Name: "answered",
+ URL: serving.URL,
+ Credential: Valid,
+ Timeout: time.Second,
+ }).Run(context.Background())
+
+ if answered.Disposition != Unavailable {
+ t.Fatalf(
+ "a target answering 503 produced %s, want %s",
+ answered.Disposition,
+ Unavailable,
+ )
+ }
+
+ if answered.Status != http.StatusServiceUnavailable {
+ t.Fatalf("the status was not recorded: %d", answered.Status)
+ }
+
+ // A listener that is closed: the connection is refused, and nothing
+ // answers. This is the case a declared fail-closed set must NOT cover.
+ dead := deadAddress(t)
+
+ refused := NewHTTP(HTTPOptions{
+ Name: "refused",
+ URL: "http://" + dead,
+ Credential: Valid,
+ Timeout: time.Second,
+ }).Run(context.Background())
+
+ if refused.Disposition != Unreachable {
+ t.Fatalf(
+ "a target that refused the connection produced %s, want %s",
+ refused.Disposition,
+ Unreachable,
+ )
+ }
+
+ if refused.Status != 0 {
+ t.Fatalf(
+ "a probe that was never answered carries status %d; zero is what says nothing answered",
+ refused.Status,
+ )
+ }
+}
+
+// Both are real evidence about the target and neither is a decision about the
+// credential, so they agree on the two properties every rate is built from.
+// Changing either would move the crash check for every experiment.
+func TestNeitherCountsAsAnAnswerAndBothAreCountable(t *testing.T) {
+ for _, disposition := range []Disposition{Unavailable, Unreachable} {
+ if disposition.Answered() {
+ t.Fatalf(
+ "%s reads as the target having answered, which would let a target that served nothing read as HELD for every experiment that declares no closed set",
+ disposition,
+ )
+ }
+
+ if !disposition.Countable() {
+ t.Fatalf(
+ "%s is excluded from every rate, so a target in that state would be invisible rather than measured",
+ disposition,
+ )
+ }
+ }
+}
+
+// deadAddress is a port nothing is listening on, obtained by binding and
+// releasing it, so the dial is REFUSED rather than timing out.
+func deadAddress(t *testing.T) string {
+ t.Helper()
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+
+ addr := listener.Addr().String()
+
+ if err := listener.Close(); err != nil {
+ t.Fatalf("close: %v", err)
+ }
+
+ return addr
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
index 59b88ca5..937aa019 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go
@@ -200,7 +200,13 @@ func transportDisposition(err error) Disposition {
if errors.Is(err, context.Canceled) {
return ProbeFailed
}
- return Unavailable
+
+ // Unreachable and not Unavailable: nothing answered. The target refusing
+ // the connection, resetting it or closing it without a response are all
+ // the target failing to ANSWER, which is a different observation from a
+ // 503 with a status line, and four experiments declare the second as their
+ // fail-closed outcome.
+ return Unreachable
}
// statusDisposition maps a status to what it says about the CREDENTIAL, not to
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go
index c1a0b4a3..896cd49b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go
@@ -15,12 +15,22 @@ const (
// wrong rather than as a target failure.
Unknown Disposition = "unknown"
- Accepted Disposition = "accepted"
- Rejected Disposition = "rejected"
- Redirected Disposition = "redirected"
- Throttled Disposition = "throttled"
+ Accepted Disposition = "accepted"
+ Rejected Disposition = "rejected"
+ Redirected Disposition = "redirected"
+ Throttled Disposition = "throttled"
+
+ // Unavailable is the target ANSWERING that it cannot serve: a 5xx with a
+ // status line. Unreachable is the target not answering at all. They were
+ // one value, and the conflation made a control's declared fail-closed
+ // outcome cover a target that had DIED: four experiments declare a 503 as
+ // their refusal, so a crashed target read as one that failed closed, with a
+ // reason saying the valid credential "stopped being accepted". Keeping them
+ // apart is what lets a declaration name one without the other.
Unavailable Disposition = "unavailable"
- TimedOut Disposition = "timed-out"
+ Unreachable Disposition = "unreachable"
+
+ TimedOut Disposition = "timed-out"
// ProbeFailed is khaos's own fault and is excluded from every rate.
ProbeFailed Disposition = "probe-failed"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/freeport_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/freeport_test.go
new file mode 100644
index 00000000..d23d47d7
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/freeport_test.go
@@ -0,0 +1,116 @@
+// ©AngelaMos | 2026
+// freeport_test.go
+
+package rangelab
+
+import (
+ "net"
+ "testing"
+)
+
+// Testing loopback alone was weaker than it read. The published container ports
+// go to 127.0.0.1, but four tiers bind the Docker bridge gateway, and a port
+// free on one is not evidence it is free on the other. An audit found it.
+func TestAPortTakenOnAnyRequiredAddressIsNotFree(t *testing.T) {
+ // A second loopback address stands in for the bridge gateway: it is a
+ // different address on the same host, which is the whole property under
+ // test, and it needs no daemon.
+ const other = "127.0.0.2"
+
+ probe, err := net.Listen("tcp", net.JoinHostPort(other, "0"))
+ if err != nil {
+ t.Skipf("this host has no second loopback address to bind: %v", err)
+ }
+
+ taken := probe.Addr().(*net.TCPAddr).Port
+
+ defer func() { _ = probe.Close() }()
+
+ // Free on 127.0.0.1 and taken on the other address, which is exactly the
+ // case the old check passed.
+ if !freeOn(taken, []string{loopback}) {
+ t.Skipf(
+ "port %d is also taken on loopback, so this case is not set up",
+ taken,
+ )
+ }
+
+ if freeOn(taken, []string{loopback, other}) {
+ t.Fatalf(
+ "port %d read as free while something is listening on it at %s; a tier binding that address would fail where khaos said it would not",
+ taken,
+ other,
+ )
+ }
+}
+
+// AND FreePort has to actually USE the extra addresses, which the case above
+// does not prove: it calls the predicate directly, so a FreePort that ignored
+// its argument passed it. Measured by gutting, which stayed green.
+//
+// This is deterministic rather than probabilistic. 192.0.2.1 is TEST-NET-1 and
+// is not assigned to this host, so NO port can be bound on it -- every
+// candidate fails, and the only way FreePort can succeed is by not testing the
+// address it was given.
+func TestFreePortActuallyTestsTheAddressesItIsGiven(t *testing.T) {
+ const unbindable = "192.0.2.1"
+
+ if _, err := net.Listen(
+ "tcp",
+ net.JoinHostPort(unbindable, "0"),
+ ); err == nil {
+ t.Skip(
+ "this host can bind TEST-NET-1, so it cannot stand in for an address no port is free on",
+ )
+ }
+
+ if port, err := FreePort(unbindable); err == nil {
+ t.Fatalf(
+ "FreePort returned %d while checking an address it cannot bind any port on, so it is not testing the addresses it is given",
+ port,
+ )
+ }
+
+ // And without it, the same call still has to answer, or the guard is a
+ // range that cannot come up.
+ if _, err := FreePort(); err != nil {
+ t.Fatalf("FreePort with no extra addresses failed: %v", err)
+ }
+}
+
+// And the allocator still has to return something usable, or the fix is a
+// range that cannot come up.
+func TestFreePortsStillAnswersWithEveryPortDistinct(t *testing.T) {
+ const other = "127.0.0.2"
+
+ chosen, err := FreePorts(other)
+ if err != nil {
+ t.Fatalf("FreePorts: %v", err)
+ }
+
+ seen := map[int]bool{}
+
+ for _, port := range []int{
+ chosen.Gateway, chosen.Proxied, chosen.Proxy, chosen.Verifier,
+ chosen.IdP, chosen.TLS, chosen.Authority, chosen.Dependency,
+ chosen.DNS, chosen.DBGateway, chosen.Secret, chosen.Env,
+ chosen.Authz, chosen.AuthzDep,
+ } {
+ if port < portFloor || port >= portCeil {
+ t.Fatalf("port %d is outside the high range khaos picks from", port)
+ }
+
+ if seen[port] {
+ t.Fatalf("port %d was handed out twice", port)
+ }
+
+ seen[port] = true
+
+ if !freeOn(port, []string{loopback, other}) {
+ t.Fatalf(
+ "port %d was handed out while taken on one of the addresses it was checked against",
+ port,
+ )
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
index 1458d90e..c2a09c0b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
@@ -123,9 +123,24 @@ const (
// a 200 is read as an absence of an answer, and allowed. AuthzRetries
// reaches the right verdict the wrong way, asking again every time it does
// not like an answer, which is that section's retry storm.
+ // AuthzCached is correct too, and it is the input khaos's own verdict
+ // machinery was wrong about: a one-second decision cache plus a refusal
+ // expressed as 503 rather than 403.
AuthzDistinguishes = "distinguishes"
AuthzCollapses = "collapses"
AuthzRetries = "retries"
+ AuthzCached = "cached"
+
+ // EnvReresolve is whether the DNS gateway resolves its dependency's name on
+ // every request. Both values are real target shapes: `reresolve` is a
+ // client that opens a fresh connection per call, and `pinned` is the
+ // DEFAULT behaviour of a Go service -- http.Transport resolves once when it
+ // opens a connection and never again while that connection lives, measured
+ // at two queries across forty-six requests. The rotation fault needs both.
+ EnvReresolve = "KHAOS_RANGE_RERESOLVE"
+
+ DNSReresolve = "reresolve"
+ DNSPinned = "pinned"
// SecretVariable is the variable the environment faults take away, which
// khaos names rather than discovers: the fixture is configured to read its
@@ -347,12 +362,15 @@ type Ports struct {
}
// FreePorts picks every host port the range needs in one go, all distinct.
-func FreePorts() (Ports, error) {
+// `also` is every further address a tier will bind one of them on, which the
+// caller discovers from the daemon: four tiers bind the bridge gateway rather
+// than loopback.
+func FreePorts(also ...string) (Ports, error) {
taken := map[int]bool{}
pick := func() (int, error) {
for range portTries {
- port, err := FreePort()
+ port, err := FreePort(also...)
if err != nil {
return 0, err
}
@@ -463,27 +481,47 @@ func orNone(declared string) string {
return declared
}
-func FreePort() (int, error) {
+// FreePort proves a candidate is free on every address a tier might bind it on,
+// and `also` is what makes that true. Testing loopback alone was weaker than it
+// read: the published container ports go to 127.0.0.1, but the proxy, the
+// identity provider, the certificate authority and the authorization dependency
+// all bind the DOCKER BRIDGE GATEWAY, and a port free on 127.0.0.1 is not
+// evidence it is free on 172.17.0.1.
+//
+// A wildcard bind would answer the same question in one call and is not used:
+// khaos refuses to bind a wildcard anywhere else, and a probe is not a good
+// enough reason to make an exception an operator would find in `ss` output.
+func FreePort(also ...string) (int, error) {
for range portTries {
//nolint:gosec // a bind candidate, immediately tested
port := portFloor + rand.IntN(portCeil-portFloor)
+ if freeOn(port, append([]string{loopback}, also...)) {
+ return port, nil
+ }
+ }
+
+ return 0, fmt.Errorf(errNoPort, portFloor, portCeil)
+}
+
+// freeOn requires EVERY address, because a port taken on any one of them is a
+// port a tier can fail to bind.
+func freeOn(port int, addresses []string) bool {
+ for _, address := range addresses {
listener, err := net.Listen(
"tcp",
- net.JoinHostPort(loopback, strconv.Itoa(port)),
+ net.JoinHostPort(address, strconv.Itoa(port)),
)
if err != nil {
- continue
+ return false
}
if err := listener.Close(); err != nil {
- continue
+ return false
}
-
- return port, nil
}
- return 0, fmt.Errorf(errNoPort, portFloor, portCeil)
+ return true
}
func (l Lab) GatewayURL() string {
@@ -509,6 +547,7 @@ func (l Lab) Up(
ctx context.Context,
out io.Writer,
failMode, cacheTTL, tlsMode, poolMode, secretMode, authzMode string,
+ reresolve string,
withDB bool,
) error {
environ := append(os.Environ(),
@@ -536,6 +575,7 @@ func (l Lab) Up(
EnvAuthzPort+"="+strconv.Itoa(l.AuthzPort),
EnvAuthzDep+"="+strconv.Itoa(l.AuthzDepPort),
EnvAuthzMode+"="+authzMode,
+ EnvReresolve+"="+reresolve,
)
args := []string{"up", "--build", "--detach", "--wait"}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
index 3b8e8855..b53e0db0 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
@@ -282,7 +282,7 @@ services:
# property of the fault class rather than a fixture defect, and khaos
# reports it when it measures it -- but a fixture that can never
# experience the fault cannot be what an exit gate asserts against.
- KHAOS_RANGE_RERESOLVE: "true"
+ KHAOS_RANGE_RERESOLVE: "${KHAOS_RANGE_RERESOLVE:-true}"
labels:
khaos.target: "true"
khaos.range: "true"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
index b1715aba..235e52c8 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh
@@ -867,15 +867,17 @@ declare -A WANT_AUTHZ=(
[collapses]="deviated / PERMITTED"
[distinguishes]="passed / HELD"
[retries]="passed / HELD"
+ [cached]="passed / HELD"
)
declare -A WANT_AUTHZ_EXIT=(
[collapses]=2
[distinguishes]=0
[retries]=0
+ [cached]=0
)
-for mode in collapses distinguishes retries; do
+for mode in collapses distinguishes retries cached; do
if ! start_authz_range "$mode"; then
continue
fi
@@ -965,6 +967,46 @@ if start_authz_range retries; then
"That is the third branch"
fi
+echo "== the regression control: a correct caller that refuses with 503 =="
+
+# THE INPUT THIS GATE MUST REJECT, and it could not until now. An audit found
+# khaos reporting a CORRECT fail-closed caller as `deviated / PERMITTED` with
+# "this target's fallback is to allow": the order-sensitive counts that stop
+# khaos calling a short decision cache a bypass reset on a REFUSAL, and this
+# experiment declares a 503 as its refusal, so a caller that served one request
+# from a one-second cache and then correctly refused with 503s read as a bypass.
+#
+# The three modes above cannot produce it. They answer 403 and hold no cache, so
+# the sequence is unreachable against them - which is exactly why the defect
+# shipped under 204 green assertions. This mode answers 503 and caches for a
+# second, in BOTH fault windows, so it exercises both halves of the fix:
+# `AcceptedDespiteLapse` on window one's revoked stream and
+# `AcceptedSinceLastRefusal` on window two's valid stream.
+#
+# VERIFIED by reverting the fix against this fixture on a live range: the same
+# run returns `deviated / PERMITTED` exit 2 with the predicate gutted, and
+# `passed / HELD` exit 0 with it in place.
+if start_authz_range cached; then
+ run_experiment credential.apikey_revoked
+
+ assert_verdict "a caller that refuses with 503" "passed / HELD"
+ assert_exit "a caller that refuses with 503" 0
+ assert_confirmed "a caller that refuses with 503"
+
+ assert_report_lacks \
+ "a correct caller is not told its fallback is to allow" \
+ "this target's fallback is to allow"
+ assert_report_lacks \
+ "and is not reported as having served in both phases" \
+ "served in BOTH phases"
+ assert_report_lacks \
+ "and no bypass is claimed against it" \
+ "was still accepted"
+ assert_report_has \
+ "it is credited with the third branch like any correct caller" \
+ "That is the third branch"
+fi
+
echo "== the authorization fixture is refused by name when it is not running =="
# The refusal has to arrive with a range UP and this one fixture stopped, which
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
index b327cf76..080876c6 100755
--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
@@ -63,10 +63,18 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }
# The gateway's fail mode is fixed at container creation, deliberately, so a
# gate spanning both directions has to restart the range. A gateway that could
# be reconfigured mid-run would let khaos change the target it is measuring.
+#
+# The second argument is the DNS gateway's resolution behaviour, and it is fixed
+# at creation for the same reason. `dns.ttl_rotate` needs both shapes: a client
+# that re-resolves follows a zone rotation, and one that resolved once and
+# reuses its connection never sees it.
start_range() {
+ local dns_mode="${2:-reresolve}"
+
"$BIN" range down --no-color >/dev/null 2>&1 || true
- if ! "$BIN" range up --fail-mode "$1" --no-color --quiet >/dev/null 2>&1; then
- fail "range up --fail-mode $1"
+ if ! "$BIN" range up --fail-mode "$1" --dns-mode "$dns_mode" \
+ --no-color --quiet >/dev/null 2>&1; then
+ fail "range up --fail-mode $1 --dns-mode $dns_mode"
return 1
fi
return 0
@@ -493,6 +501,67 @@ if start_range closed; then
fi
fi
+echo "== the rotation: a zone change that does not reach the target =="
+
+# `dns.ttl_rotate` is the one fault that MOVES the zone rather than answering it
+# wrongly, so the honest path serves the new address afterwards. Research 05
+# section 6.5 carries the polarity and it is NOT inverted: khaos rotates the
+# name to the address `wrong_address` uses, so a target that FOLLOWS the
+# rotation starts trusting a host on the strength of a name, which is the same
+# defect from the same evidence and stays PERMITTED.
+#
+# Both directions, because the finding is which one happened:
+#
+# reresolve the target asks again, follows the rotation, and accepts what the
+# new host approves -> deviated / PERMITTED exit 2
+# pinned the target resolved once and reuses its connection, never sees
+# the rotation -> passed / HELD exit 0, with the operational
+# finding STATED and explicitly not called a bypass
+#
+# The pinned row is the one that needed a target shape the range did not have.
+if start_range closed reresolve; then
+ run_experiment dns.ttl_rotate
+
+ assert_verdict "ttl_rotate against a re-resolving target" "deviated / PERMITTED"
+ assert_exit "ttl_rotate against a re-resolving target" 2
+ assert_confirmed "ttl_rotate against a re-resolving target"
+
+ # The confirmation names BOTH addresses and says the answer came from the
+ # honest path, which is what distinguishes a rotation from a redirect.
+ assert_report_has \
+ "the confirmation names the address the rotation replaced" \
+ "before khaos moved it"
+ assert_report_has \
+ "and says the honest path serves it" \
+ "from the resolver's own honest path rather than from a fault"
+ assert_report_has \
+ "the rotation is reported as having reached the target" \
+ "so the rotation reached it"
+fi
+
+if start_range closed pinned; then
+ run_experiment dns.ttl_rotate
+
+ assert_verdict "ttl_rotate against a pinning target" "passed / HELD"
+ assert_exit "ttl_rotate against a pinning target" 0
+ assert_confirmed "ttl_rotate against a pinning target"
+
+ # The finding has to be STATED, with both counts, and must NOT be called a
+ # bypass: no credential got through that would not have before.
+ assert_report_has \
+ "the rotation is reported as not having reached the target" \
+ "ZERO during the fault window"
+ assert_report_has \
+ "and the operational consequence is named" \
+ "would not have taken effect either"
+ assert_report_has \
+ "and khaos explicitly reports no bypass" \
+ "This is NOT a credential bypass"
+ assert_report_lacks \
+ "no credential is claimed to have been accepted" \
+ "was accepted"
+fi
+
echo "== the census: every catalogued fault was actually run =="
# A catalogued experiment this gate never runs is a failure, for the same reason