Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
fix(khaos): the six audit findings, closed rather than carried, and the rotation they cleared the way for
CarterPerez-dev3f49be219f24
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.goindex aa6fd6df..edcc0c92 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway.go@@ -34,6 +34,25 @@ const (// step 4 names as the retry storm.AuthzRetries = "retries"+ // AuthzCached is also CORRECT, and it exists because khaos's own verdict+ // machinery was wrong about it. It differs from `distinguishes` in two ways+ // that are ordinary in real services and that khaos had never been shown+ // together: it caches an authorization decision for a second, and it+ // expresses a refusal as 503 rather than 403, the way a service that maps+ // "my dependency would not authorize this" onto an upstream error does.+ //+ // That produces one acceptance from the cache followed by refusals khaos+ // reads as `Unavailable`, in both fault windows. khaos reported it+ // `deviated / PERMITTED` with "this target's fallback is to allow",+ // because its order-sensitive counts reset on `Rejected` alone while this+ // experiment DECLARES a 503 as its fail-closed outcome. This mode is the+ // input the gate needs to be able to reject.+ AuthzCached = "cached"++ // Short enough to lapse several times inside a fault window, which is what+ // makes the cached acceptance the FIRST sample rather than the whole run.+ authzCacheFor = time.Second+headerAuthzReason = "X-Khaos-Authz"reasonNotAuthenticated = "this service never issued that credential, which is its own check and not the dependency's"@@ -48,6 +67,10 @@ const (reasonCollapsed = "the authorization dependency DENIED it and this mode read that as no answer, so it allowed it"+ reasonCachedAllow = "this caller's own authorization-decision cache still holds an allow for that credential"++ reasonCachedRefusal = "this caller's own authorization-decision cache still holds a refusal for that credential"+// Three attempts, because the finding is the ask COUNT and one retry would// make it indistinguishable from a second probe arriving.authzAttempts = 3@@ -71,6 +94,18 @@ type authzGateway struct {mu sync.Mutexissued map[string]bool++ // decided is this caller's own authorization-decision cache, and only the+ // cached mode writes to it.+ decided map[string]verdictAt+}++// verdictAt is one cached authorization decision. A caller that caches is+// ordinary; what khaos was wrong about is how the cache's first sample reads+// once the refusals that follow it are 503s rather than 403s.+type verdictAt struct {+ allow bool+ at time.Time}func newAuthzGateway(dependency, mode string) *authzGateway {@@ -79,7 +114,47 @@ func newAuthzGateway(dependency, mode string) *authzGateway {mode: mode,client: &http.Client{Timeout: upstreamTimeout},issued: map[string]bool{},+ decided: map[string]verdictAt{},+ }+}++// refusalStatus is how THIS caller says no. A 503 is as common as a 403 in+// service-to-service code -- a service that maps a dependency's refusal onto an+// upstream error -- and khaos has to read both as the same refusal.+func (g *authzGateway) refusalStatus() int {+ if g.mode == AuthzCached {+ return http.StatusServiceUnavailable+ }++ return http.StatusForbidden+}++// cachedVerdict answers from this caller's own cache while the entry is fresh.+func (g *authzGateway) cachedVerdict(credential string) (bool, bool) {+ if g.mode != AuthzCached {+ return false, false+ }++ g.mu.Lock()+ defer g.mu.Unlock()++ held, found := g.decided[credential]+ if !found || time.Since(held.at) > authzCacheFor {+ return false, false}++ return held.allow, true+}++func (g *authzGateway) remember(credential string, allow bool) {+ if g.mode != AuthzCached {+ return+ }++ g.mu.Lock()+ defer g.mu.Unlock()++ g.decided[credential] = verdictAt{allow: allow, at: time.Now()}}func (g *authzGateway) routes() *http.ServeMux {@@ -144,13 +219,31 @@ func (g *authzGateway) protected(w http.ResponseWriter, r *http.Request) {return}- w.WriteHeader(http.StatusForbidden)+ w.WriteHeader(g.refusalStatus())}// decide is the whole fixture. One branch reads the dependency's three answers// as three; the second reads "not 200" as "no answer"; the third reads them// correctly and asks again every time it does not like one.func (g *authzGateway) decide(credential string) (bool, string) {+ if allow, fresh := g.cachedVerdict(credential); fresh {+ if allow {+ return true, reasonCachedAllow+ }++ return false, reasonCachedRefusal+ }++ allow, reason := g.readDependency(credential)++ g.remember(credential, allow)++ return allow, reason+}++// readDependency asks and applies this mode's READING of the answer, which is+// the one line the whole experiment is about.+func (g *authzGateway) readDependency(credential string) (bool, string) {status, reached := g.ask(credential)switch g.mode {diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.goindex c0d4e7ea..5317153a 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/authzgateway_test.go@@ -10,6 +10,7 @@ import ("strings""sync/atomic""testing"+ "time")// stubAuthz is khaos's side of the experiment, reduced to what the fixture can@@ -96,6 +97,7 @@ func TestEveryModeAuthenticatesLocally(t *testing.T) {AuthzDistinguishes,AuthzCollapses,AuthzRetries,+ AuthzCached,} {t.Run(mode, func(t *testing.T) {gateway, dependency := authzFixture(t, mode)@@ -239,6 +241,95 @@ func TestTheCollapseNamesWhichAnswerItIgnored(t *testing.T) {}}+// The cached mode is the input khaos's own verdict machinery was wrong about,+// so the fixture has to actually produce that sequence: one acceptance from the+// cache, then refusals expressed as 503. Both halves matter -- the cache, and+// the status -- because khaos's order-sensitive counts reset on a REFUSAL and+// a 503 is only a refusal for a control that declared it.+func TestTheCachedModeServesOnceThenRefusesWith503(t *testing.T) {+ gateway, dependency := authzFixture(t, AuthzCached)+ credential := issued(t, gateway)++ if got := askAuthz(t, gateway, credential); got != http.StatusOK {+ t.Fatalf("an allowed credential returned %d, want 200", got)+ }++ dependency.status.Store(http.StatusForbidden)++ // Inside the cache window the caller answers from its own cache and never+ // asks, which is what puts the acceptance FIRST.+ dependency.asked.Store(0)++ if got := askAuthz(t, gateway, credential); got != http.StatusOK {+ t.Fatalf(+ "within the cache window the caller answered %d; a cached allow is what makes the first sample an acceptance",+ got,+ )+ }++ if asked := dependency.asked.Load(); asked != 0 {+ t.Fatalf(+ "the caller asked %d time(s) inside its own cache window, so it holds no cache and this mode cannot produce the sequence it exists for",+ asked,+ )+ }++ time.Sleep(authzCacheFor + 100*time.Millisecond)++ if got := askAuthz(+ t,+ gateway,+ credential,+ ); got != http.StatusServiceUnavailable {+ t.Fatalf(+ "once the cache lapsed the caller answered %d, want %d: this mode expresses its refusal as an upstream error, which is the half khaos read wrong",+ got,+ http.StatusServiceUnavailable,+ )+ }++ if asked := dependency.asked.Load(); asked == 0 {+ t.Fatal("the caller never asked after its cache lapsed")+ }++ // And it is still CORRECT: it refuses, it does not serve a credential the+ // dependency denied.+ if got := askAuthz(+ t,+ gateway,+ credential,+ ); got == http.StatusOK {+ t.Fatal(+ "the cached mode served a credential its dependency had denied, which would make it a second collapse rather than a correct caller",+ )+ }+}++// Only the cached mode caches, or every other mode's measured behaviour moves.+func TestNoOtherModeHoldsADecisionCache(t *testing.T) {+ for _, mode := range []string{+ AuthzDistinguishes,+ AuthzCollapses,+ AuthzRetries,+ } {+ t.Run(mode, func(t *testing.T) {+ gateway, dependency := authzFixture(t, mode)+ credential := issued(t, gateway)++ askAuthz(t, gateway, credential)+ dependency.asked.Store(0)+ askAuthz(t, gateway, credential)++ if asked := dependency.asked.Load(); asked == 0 {+ t.Fatalf(+ "%s answered a second request without asking its dependency, so it is caching and its measured ask-rate is not what khaos thinks it is",+ mode,+ )+ }+ })+ }+}+// A caller that cannot reach its dependency at all is the branch every network// fault already reaches, and the correct modes must refuse there too.func TestAnUnreachableDependencyIsRefusedByTheCorrectModes(t *testing.T) {diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.goindex ebf429bd..6e7fd6e1 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/authz/authz.go@@ -35,7 +35,7 @@ const (reasonDenied = "khaos's authorization dependency denies this credential: a definitive verdict, with full reachability"reasonOutage = "khaos's authorization dependency cannot answer; this is the absence of a decision, not a decision"- subjectAuthz = "an authorization service that allows every credential it was not told to deny"+ SubjectAuthz = "an authorization service that allows every credential it was not told to deny"askTimeout = 2 * time.SecondshutdownGrace = 2 * time.Second@@ -101,7 +101,7 @@ func New() *Dependency {// wildcard bind would put an allow-everything authorization service on every// interface the host has.func (d *Dependency) Listen(addr string) error {- if err := bindguard.Check(addr, subjectAuthz); err != nil {+ if err := bindguard.Check(addr, SubjectAuthz); err != nil {return err}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.goindex 1d3be2fc..8ae280c2 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard.go@@ -31,6 +31,8 @@ const (errBadAddr = "%w: %q is not a host and port (%v)"errOffBox = "%w: %q is not on loopback or on a Docker bridge, so it is reachable from outside this host and a container cannot reach it anyway; bind the bridge gateway the daemon reports"++ errUnmanaged = "%w: %q is not a gateway of any network the Docker daemon manages, and %s must not be reachable from anything but the target under test. khaos checked %d network gateway(s). A kernel bridge is not enough: a CNI pod bridge, a swarm gateway bridge and a host interface enslaved to a bridge all look like one, and the last of those is the host's own routable address")// ErrUnsafeBind is one sentinel for every listener khaos opens. Each tier@@ -86,6 +88,50 @@ func Check(listenAddr, subject string) error {return nil}+// Managed is the narrow check, and Check is the structural one. Check asks+// "could a container reach this and nothing off the host"; this asks "is this+// an address DOCKER manages", which is the question `/sys/class/net/<if>/bridge`+// cannot answer: that file exists for every kernel bridge device, including a+// CNI pod bridge, a swarm gateway bridge, every user-defined Docker network's+// bridge, and -- on a host whose LAN interface is enslaved to one -- the+// routable LAN address itself. Check accepted all of those while its own+// comment claimed nothing but the target could reach the listener.+//+// gateways comes from the daemon, so this lives wherever a Docker client does+// and the structural check stays in the packages that bind. An empty set is a+// refusal rather than a pass: khaos could not establish the allowed addresses,+// and binding anyway is the behaviour this exists to stop.+func Managed(listenAddr, subject string, gateways []string) error {+ host, _, err := net.SplitHostPort(listenAddr)+ if err != nil {+ return fmt.Errorf(errBadAddr, ErrUnsafeBind, listenAddr, err)+ }++ ip := net.ParseIP(host)+ if ip == nil {+ return fmt.Errorf(errWildcard, ErrUnsafeBind, listenAddr, subject)+ }++ if ip.IsLoopback() {+ return nil+ }++ for _, gateway := range gateways {+ if managed := net.ParseIP(gateway); managed != nil &&+ managed.Equal(ip) {+ return nil+ }+ }++ return fmt.Errorf(+ errUnmanaged,+ ErrUnsafeBind,+ listenAddr,+ subject,+ len(gateways),+ )+}+func bridgeHosts(ip net.IP) (bool, error) {interfaces, err := net.Interfaces()if err != nil {diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.goindex 315b6f5f..8ffdbf67 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/bindguard/bindguard_test.go@@ -81,3 +81,83 @@ func TestAWildcardRefusalNamesTheSubject(t *testing.T) {)}}++// Managed is the check a kernel bridge cannot satisfy. Measured on the+// development host when this was written: a CNI pod bridge at 10.42.0.1, a+// swarm gateway bridge at 172.23.0.1 and forty-nine user-defined bridges, one+// of them carrying nine containers of an unrelated project -- every one of them+// carrying the sysfs marker Check reads, and none of them an address khaos+// should serve an allow-everything listener on.+func TestManagedTakesTheAllowedSetFromTheDaemon(t *testing.T) {+ gateways := []string{"172.17.0.1", "10.244.244.1"}++ for _, row := range []struct {+ name string+ addr string+ set []string+ want bool+ }{+ {+ name: "a gateway the daemon manages",+ addr: "172.17.0.1:45999",+ set: gateways,+ want: true,+ },+ {+ name: "another one it manages",+ addr: "10.244.244.1:45999",+ set: gateways,+ want: true,+ },+ {+ name: "loopback, which needs no daemon",+ addr: "127.0.0.1:45999",+ set: nil,+ want: true,+ },+ {+ name: "a CNI pod bridge, which carries the sysfs marker",+ addr: "10.42.0.1:45999",+ set: gateways,+ want: false,+ },+ {+ name: "a host interface enslaved to a bridge",+ addr: "192.168.1.183:45999",+ set: gateways,+ want: false,+ },+ {+ name: "an empty set is a refusal, not a pass",+ addr: "172.17.0.1:45999",+ set: nil,+ want: false,+ },+ {+ name: "a wildcard, which names no address at all",+ addr: "0.0.0.0:45999",+ set: gateways,+ want: false,+ },+ } {+ t.Run(row.name, func(t *testing.T) {+ err := Managed(row.addr, subject, row.set)++ if row.want && err != nil {+ t.Fatalf("%s was refused: %v", row.addr, err)+ }++ if row.want {+ return+ }++ if err == nil {+ t.Fatalf("%s was accepted", row.addr)+ }++ if !errors.Is(err, ErrUnsafeBind) {+ t.Fatalf("%s: err = %v, want ErrUnsafeBind", row.addr, err)+ }+ })+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.goindex e3d89238..f21ed690 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go@@ -11,20 +11,24 @@ import ("maps""net""net/url"+ "os""slices""strconv""strings""time"+ "github.com/CarterPerez-dev/khaos/internal/authz""github.com/CarterPerez-dev/khaos/internal/bindguard""github.com/CarterPerez-dev/khaos/internal/dbx""github.com/CarterPerez-dev/khaos/internal/dnsrv""github.com/CarterPerez-dev/khaos/internal/dockerx""github.com/CarterPerez-dev/khaos/internal/experiment"+ "github.com/CarterPerez-dev/khaos/internal/idp""github.com/CarterPerez-dev/khaos/internal/inject""github.com/CarterPerez-dev/khaos/internal/inject/proxy""github.com/CarterPerez-dev/khaos/internal/journal""github.com/CarterPerez-dev/khaos/internal/lease"+ "github.com/CarterPerez-dev/khaos/internal/pki""github.com/CarterPerez-dev/khaos/internal/rangelab""github.com/CarterPerez-dev/khaos/internal/render""github.com/CarterPerez-dev/khaos/internal/run"@@ -128,6 +132,15 @@ enforcing in the first place.`msgAuthzUp = "khaos is answering authorization decisions on %s, allowing every credential until it is told otherwise"+ // Every parameter can also arrive in the environment, which is the only+ // channel that does not put a credential in the world-readable process+ // table. Same convention as the resolver nonce.+ paramEnv = "%s [or %s]"++ envParamPrefix = "KHAOS_"+ flagWordSep = "-"+ envWordSep = "_"+errNoSubjectSecret = "%s does not carry %s in its environment, so khaos cannot read the credential its own connection pool authenticated with, and a rotation it cannot undo is not one khaos will perform"errTargetOffNetwork = "%s is not attached to %s, so it cannot be using the chaos resolver that serves it and khaos could not count its queries: %w"@@ -297,11 +310,16 @@ func experimentFlags(cmd *cobra.Command, definition experiment.Definition) {Duration(flagSampleEvery, experiment.DefaultSampleInterval, sampleUsage)}+// paramUsage names the environment variable too, because a channel an operator+// cannot discover is a channel nobody uses -- and for a credential it is the+// only one that keeps the value out of the world-readable process table.func paramUsage(param experiment.Param) string {- if param.Default == "" {- return param.Usage+ usage := param.Usage+ if param.Default != "" {+ usage = fmt.Sprintf(paramDefault, usage, param.Default)}- return fmt.Sprintf(paramDefault, param.Usage, param.Default)++ return fmt.Sprintf(paramEnv, usage, envFor(param.Name))}type inputs struct {@@ -324,6 +342,17 @@ func experimentInputs(supplied := map[string]string{}for _, param := range definition.Params {if !cmd.Flags().Changed(param.Name) {+ // A FLAG is not the only way in, and for a credential it is the+ // worst one: /proc/<pid>/cmdline is world-readable (mode 444,+ // measured) while /proc/<pid>/environ is 400, which is why the+ // chaos resolver's nonce has always been an environment variable+ // rather than an argument. An experiment can require two live+ // credentials, so argv-only put both of an operator's secrets in+ // the process table and in shell history.+ if value, given := os.LookupEnv(envFor(param.Name)); given {+ supplied[param.Name] = value+ }+continue}@@ -343,6 +372,20 @@ func experimentInputs(return inputs{Project: project, Supplied: supplied, Timing: timing}, nil}+// envFor is the environment variable that supplies a parameter when its flag is+// absent. Derived from the parameter's own name rather than listed, so a new+// parameter gets the channel without anyone remembering to add it:+// `--revocable-token` reads `KHAOS_REVOCABLE_TOKEN`.+//+// Precedence is flag, then environment, then the range, then the declaration's+// default. The environment beats the range because both are the operator+// speaking and the range is only a convenience.+func envFor(param string) string {+ return envParamPrefix + strings.ToUpper(+ strings.ReplaceAll(param, flagWordSep, envWordSep),+ )+}+func experimentTiming(cmd *cobra.Command) (experiment.Timing, error) {durations := map[string]time.Duration{}@@ -1212,6 +1255,26 @@ func startProxy(// startProvider binds khaos's own identity provider on the Docker bridge gateway// at the port the issuer names, never a wildcard: the target addresses the issuer// as `host.docker.internal`, which does not resolve here.+// providerListen is where the provider will bind, computed before the bind so+// the guard and the listener are given the same string.+func providerListen(+ ctx context.Context,+ reader rangelab.PortReader,+ spec experiment.Experiment,+) (string, error) {+ port, err := portOf(spec.IdP.Issuer())+ if err != nil {+ return "", err+ }++ gateway, err := reader.BridgeGateway(ctx)+ if err != nil {+ return "", err+ }++ return net.JoinHostPort(gateway, strconv.Itoa(port)), nil+}+func startProvider(ctx context.Context,out io.Writer,@@ -1220,18 +1283,11 @@ func startProvider() error {issuer := spec.IdP.Issuer()- port, err := portOf(issuer)- if err != nil {- return err- }-- gateway, err := reader.BridgeGateway(ctx)+ listen, err := providerListen(ctx, reader, spec)if err != nil {return err}- listen := net.JoinHostPort(gateway, strconv.Itoa(port))-if err := spec.IdP.Listen(listen); err != nil {return fmt.Errorf(errIdPBind, listen, err)}@@ -1274,10 +1330,10 @@ func standUpTiers(return standUpIdPTier(ctx, out, work.Docker, spec, &env)},func() (func(), error) {- return standUpPKITier(out, spec, &env)+ return standUpPKITier(ctx, out, work.Docker, spec, &env)},func() (func(), error) {- return standUpAuthzTier(out, spec, &env)+ return standUpAuthzTier(ctx, out, work.Docker, spec, &env)},func() (func(), error) {return standUpDNSTier(ctx, out, work.Docker, spec, target, &env)@@ -1316,6 +1372,26 @@ func standUpTiers(return env, closeAll, nil}+// managedBind is the NARROW half of the bind guard and it lives here because+// the allowed set comes from the daemon. Each binding package carries the+// structural half -- no wildcard, no privileged port, loopback or a bridge --+// and a kernel bridge is not enough: measured on a development host, that+// includes a CNI pod bridge, a swarm gateway bridge, forty-nine user-defined+// bridges carrying other projects' containers, and on a host whose LAN+// interface is enslaved to a bridge, the routable LAN address itself.+func managedBind(+ ctx context.Context,+ docker *dockerx.Client,+ addr, subject string,+) error {+ gateways, err := docker.Gateways(ctx)+ if err != nil {+ return err+ }++ return bindguard.Managed(addr, subject, gateways)+}+func standUpProxyTier(ctx context.Context,out io.Writer,@@ -1327,6 +1403,15 @@ func standUpProxyTier(return nil, nil}+ if err := managedBind(+ ctx,+ docker,+ spec.ProxyListen,+ proxy.SubjectProxy,+ ); err != nil {+ return nil, err+ }+fault, err := startProxy(ctx, out, docker, *spec)if err != nil {return nil, err@@ -1349,6 +1434,20 @@ func standUpIdPTier(return nil, nil}+ listen, err := providerListen(ctx, docker, *spec)+ if err != nil {+ return nil, err+ }++ if err := managedBind(+ ctx,+ docker,+ listen,+ idp.SubjectProvider,+ ); err != nil {+ return nil, err+ }+if err := startProvider(ctx, out, docker, *spec); err != nil {return nil, err}@@ -1360,7 +1459,9 @@ func standUpIdPTier(}func standUpPKITier(+ ctx context.Context,out io.Writer,+ docker *dockerx.Client,spec *experiment.Experiment,env *inject.Env,) (func(), error) {@@ -1368,6 +1469,15 @@ func standUpPKITier(return nil, nil}+ for addr, subject := range map[string]string{+ spec.AuthorityListen: pki.SubjectRevocation,+ spec.AuthorityTLS: pki.SubjectDependency,+ } {+ if err := managedBind(ctx, docker, addr, subject); err != nil {+ return nil, err+ }+ }+if err := startAuthority(out, *spec); err != nil {return nil, err}@@ -1383,7 +1493,9 @@ func standUpPKITier(// through the whole run, or the baseline measures a dependency that is down and// the fault window is the only one where it is up.func standUpAuthzTier(+ ctx context.Context,out io.Writer,+ docker *dockerx.Client,spec *experiment.Experiment,env *inject.Env,) (func(), error) {@@ -1391,6 +1503,15 @@ func standUpAuthzTier(return nil, nil}+ if err := managedBind(+ ctx,+ docker,+ spec.AuthzListen,+ authz.SubjectAuthz,+ ); err != nil {+ return nil, err+ }+if err := spec.Authz.Listen(spec.AuthzListen); err != nil {// The guard refuses BEFORE the bind is attempted, so the port-collision// advice below is wrong for the one refusal an operator will actuallydiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.goindex 39250fe4..f06bb715 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go@@ -918,3 +918,96 @@ func answerableFor(definition experiment.Definition) []string {return out}++// A credential supplied as a FLAG is in /proc/<pid>/cmdline, which is+// world-readable (mode 444, measured), while /proc/<pid>/environ is 400. That+// is why the chaos resolver's nonce has always been an environment variable+// rather than an argument, and `credential.apikey_revoked` is the first+// experiment to require TWO live credentials, so argv-only put both of an+// operator's secrets in the process table and in shell history.+func TestAParameterCanArriveInTheEnvironmentInsteadOfArgv(t *testing.T) {+ definition := outage(t)++ t.Setenv("KHAOS_TOKEN", "a-credential-from-the-environment")++ cmd := newRunCmd()+ cmd.SetArgs([]string{definition.Name})++ in, err := experimentInputs(+ experimentCmdFor(t, cmd, definition),+ definition,+ )+ if err != nil {+ t.Fatalf("inputs: %v", err)+ }++ if in.Supplied[experiment.ParamToken] != "a-credential-from-the-environment" {+ t.Fatalf(+ "the environment channel was ignored: %q; a credential has no other way in that keeps it out of the process table",+ in.Supplied[experiment.ParamToken],+ )+ }+}++// The FLAG still wins, because an operator who typed one meant it, and the+// environment must not silently override what they just said.+func TestAFlagBeatsTheEnvironment(t *testing.T) {+ definition := outage(t)++ t.Setenv("KHAOS_TOKEN", "from-the-environment")++ cmd := newRunCmd()+ sub := experimentCmdFor(t, cmd, definition)++ if err := sub.Flags().+ Set(experiment.ParamToken, "from-the-flag"); err != nil {+ t.Fatalf("set the flag: %v", err)+ }++ in, err := experimentInputs(sub, definition)+ if err != nil {+ t.Fatalf("inputs: %v", err)+ }++ if in.Supplied[experiment.ParamToken] != "from-the-flag" {+ t.Fatalf(+ "the environment overrode a flag the operator typed: %q",+ in.Supplied[experiment.ParamToken],+ )+ }+}++// And the mapping is DERIVED from the parameter's name, so a new parameter gets+// the channel without anyone remembering to add it.+func TestTheEnvironmentNameIsDerivedFromTheParameter(t *testing.T) {+ for param, want := range map[string]string{+ experiment.ParamToken: "KHAOS_TOKEN",+ experiment.ParamRevocable: "KHAOS_REVOCABLE_TOKEN",+ experiment.ParamAuthzListen: "KHAOS_AUTHZ_LISTEN",+ experiment.ParamInvalidToken: "KHAOS_INVALID_TOKEN",+ } {+ if got := envFor(param); got != want {+ t.Fatalf("envFor(%q) = %q, want %q", param, got, want)+ }+ }+}++// experimentCmdFor finds the per-experiment subcommand, which is where the+// parameters are declared as flags.+func experimentCmdFor(+ t *testing.T,+ root *cobra.Command,+ definition experiment.Definition,+) *cobra.Command {+ t.Helper()++ for _, sub := range root.Commands() {+ if sub.Name() == definition.Name {+ return sub+ }+ }++ t.Fatalf("no subcommand for %s", definition.Name)++ return nil+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.goindex 577aa492..ab70925c 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go@@ -30,6 +30,10 @@ var ErrUnknownPoolMode = errors.New("that is not a pool configuration the range fixture understands",)+var ErrUnknownDNSMode = errors.New(+ "that is not a resolution behaviour the range gateway understands",+)+var ErrUnknownAuthzMode = errors.New("that is not an authorization-error handling the range fixture understands",)@@ -46,6 +50,7 @@ const (flagPoolMode = "pool-mode"flagSecretMode = "secret-mode"flagAuthzMode = "authz-mode"+ flagDNSMode = "dns-mode"mintedSecrets = 4@@ -74,7 +79,9 @@ running.`quietUsage = "suppress compose build and start output"secretModeUsage = "how the range's secret fixture handles the secret it reads. validated checks the length after every read, which is the correct handling and the fix for CVE-2019-20933; unchecked lets an empty value flow into the crypto, which is that CVE's own shape. An unrecognised value is refused rather than quietly leaving the fixture correct"- authzModeUsage = "how the range's service-to-service caller reads its authorization dependency's answers. distinguishes reads allow, deny and unknown as three outcomes, which is correct; collapses reads anything that is not a 200 as an absence of an answer and allows it, which is research 07's F20 and the single most common fail-open in service-to-service code; retries reaches the right verdict the wrong way, asking again every time it does not like an answer, which hammers the dependency with a credential it has already been told is dead. An unrecognised value is refused rather than quietly leaving the caller correct"+ dnsModeUsage = "how the range's DNS gateway resolves its dependency's name. reresolve resolves on every request, which is a client that opens a fresh connection per call; pinned resolves once and reuses the connection, which is the DEFAULT behaviour of a Go service and the shape that never sees a zone rotation. Both are real, and `dns.ttl_rotate` needs both to assert its verdict in two directions. An unrecognised value is refused rather than quietly leaving the gateway re-resolving"++ authzModeUsage = "how the range's service-to-service caller reads its authorization dependency's answers. distinguishes reads allow, deny and unknown as three outcomes, which is correct; collapses reads anything that is not a 200 as an absence of an answer and allows it, which is research 07's F20 and the single most common fail-open in service-to-service code; retries reaches the right verdict the wrong way, asking again every time it does not like an answer, which hammers the dependency with a credential it has already been told is dead; cached is correct as well and is the shape khaos's own verdict machinery was wrong about, holding a one-second decision cache and expressing its refusals as 503 rather than 403. An unrecognised value is refused rather than quietly leaving the caller correct"poolModeUsage = "how the stateful tier's pooled fixture configures its connection pool. unbounded is the DEFAULT configuration of a Go service rather than a broken one, because database/sql closes a connection for age only when SetConnMaxLifetime is positive; bounded sets a lifetime, which is the knob that turns a rotation into a revocation; held checks one connection out and never returns it, which no lifetime bounds at all"@@ -92,6 +99,7 @@ running.`errUnknownPoolMode = "%w: %q (known pool modes: %s)"errUnknownSecretMode = "%w: %q (known secret modes: %s)"errUnknownAuthzMode = "%w: %q (known authorization modes: %s)"+ errUnknownDNSMode = "%w: %q (known DNS modes: %s)"msgStatefulUp = "the stateful tier is up: PostgreSQL, MySQL, MariaDB and Redis, none of them published to a host port"msgPortHint = "host port %d was chosen at random; pass --port to pin it"@@ -163,6 +171,10 @@ func newRangeUpCmd() *cobra.Command {if err != nil {return err}+ dnsMode, err := cmd.Flags().GetString(flagDNSMode)+ if err != nil {+ return err+ }return runRangeUp(cmd.Context(),cmd.OutOrStdout(),@@ -176,6 +188,7 @@ func newRangeUpCmd() *cobra.Command {pool: poolMode,secret: secretMode,authz: authzMode,+ dns: dnsMode,},)},@@ -198,6 +211,7 @@ func newRangeUpCmd() *cobra.Command {rangelab.AuthzCollapses,authzModeUsage,)+ cmd.Flags().String(flagDNSMode, rangelab.DNSReresolve, dnsModeUsage)return cmd}@@ -209,6 +223,7 @@ type rangeModes struct {pool stringsecret stringauthz string+ dns string}// mintRangeSecrets gives the range a fresh nonce and a fresh database password@@ -263,11 +278,30 @@ func knownPoolModes() []string {// knownAuthzModes is validated against the whole set for the same reason as// the other three: `collapses` is the DEFAULT here, so a typo would hand the// experiment a correct caller and report the defect as absent.+// knownDNSModes is validated by name for the same reason as the rest: the+// default leaves the gateway re-resolving, so a typo would hand `dns.ttl_rotate`+// the one target shape that always follows a rotation and the pinning direction+// would never be measured.+// reresolveFor turns the mode into the value the gateway's own environment+// carries, which is a bare boolean string the fixture reads.+func reresolveFor(mode string) string {+ if mode == rangelab.DNSPinned {+ return "false"+ }++ return "true"+}++func knownDNSModes() []string {+ return []string{rangelab.DNSReresolve, rangelab.DNSPinned}+}+func knownAuthzModes() []string {return []string{rangelab.AuthzDistinguishes,rangelab.AuthzCollapses,rangelab.AuthzRetries,+ rangelab.AuthzCached,}}@@ -357,9 +391,43 @@ func validateModes(failMode string, modes rangeModes) error {modes.authz, strings.Join(knownAuthzModes(), listSep))}+ if !slices.Contains(knownDNSModes(), modes.dns) {+ return fmt.Errorf(errUnknownDNSMode, ErrUnknownDNSMode,+ modes.dns, strings.Join(knownDNSModes(), listSep))+ }+return nil}+// alsoBoundOn is every address BESIDES loopback that a port has to be free on.+// Four of the tiers bind the Docker bridge gateway rather than loopback, so a+// port khaos proved free on 127.0.0.1 can still be taken where they will bind+// it. A daemon that will not answer is not fatal: the free check degrades to+// loopback and the tier's own bind refusal names the port.+func alsoBoundOn(ctx context.Context, work *workspace.Workspace) []string {+ gateway, err := work.Docker.BridgeGateway(ctx)+ if err != nil {+ return nil+ }++ return []string{gateway}+}++// sayRangeIsUp is the operator-facing summary, split out so the function that+// builds the range is about building it.+func sayRangeIsUp(out io.Writer, failMode string, port int, withDB bool) {+ event(out, markOK, theme.StatusOK, msgRangeUp)+ note(out, fmt.Sprintf(msgFailMode, failMode))+ note(out, fmt.Sprintf(msgPortHint, port))++ if withDB {+ note(out, msgStatefulUp)+ }++ note(out, msgTokenHint)+ fmt.Fprintln(out)+}+func runRangeUp(ctx context.Context,out io.Writer,@@ -394,9 +462,11 @@ func runRangeUp(pinned := port != 0var lab rangelab.Lab+ also := alsoBoundOn(ctx, work)+for attempt := range portAttempts {// Only the direct gateway's port is pinnable; the others are plumbing.- ports, err := rangelab.FreePorts()+ ports, err := rangelab.FreePorts(also...)if err != nil {return err}@@ -424,6 +494,7 @@ func runRangeUp(modes.pool,modes.secret,modes.authz,+ reresolveFor(modes.dns),withDB,)if err == nil {@@ -437,15 +508,7 @@ func runRangeUp(note(out, fmt.Sprintf(msgPortRetry, port))}- event(out, markOK, theme.StatusOK, msgRangeUp)- note(out, fmt.Sprintf(msgFailMode, failMode))- note(out, fmt.Sprintf(msgPortHint, lab.Port))- if withDB {- note(out, msgStatefulUp)- }-- note(out, msgTokenHint)- fmt.Fprintln(out)+ sayRangeIsUp(out, failMode, lab.Port, withDB)// Read back from the containers, which are the only authority on where// they will actually dial and which issuer they will trust.diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.goindex ab6147dc..3a74753f 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go@@ -74,6 +74,21 @@ func (c *Client) Arm(ctx context.Context, trap Trap) (Stats, error) {return c.call(ctx, http.MethodPost, PathArm, body)}+// Rotate moves a name's address in the zone. A non-2xx is an error rather than+// a silent no-op, which is what lets a failed rotation VOID the run instead of+// leaving khaos measuring a name nobody moved.+func (c *Client) Rotate(+ ctx context.Context,+ name, address string,+) (Stats, error) {+ body, err := json.Marshal(Rotation{Name: name, Address: address})+ if err != nil {+ return Stats{}, err+ }++ return c.call(ctx, http.MethodPost, PathRotate, body)+}+func (c *Client) Disarm(ctx context.Context) (Stats, error) {return c.call(ctx, http.MethodPost, PathDisarm, nil)}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.goindex af79114a..2edc8d19 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go@@ -26,6 +26,7 @@ const (PathArm = "/arm"PathDisarm = "/disarm"+ PathRotate = "/rotate"PathState = "/state"PathHealthz = "/healthz"@@ -65,6 +66,7 @@ func (r *Resolver) ServeControl(listen string) error {mux.HandleFunc(PathState, r.guard(r.handleState))mux.HandleFunc(PathArm, r.guard(r.handleArm))mux.HandleFunc(PathDisarm, r.guard(r.handleDisarm))+ mux.HandleFunc(PathRotate, r.guard(r.handleRotate))server := &http.Server{Handler: mux,@@ -152,6 +154,39 @@ func (r *Resolver) handleState(w http.ResponseWriter, _ *http.Request) {r.writeJSON(w, r.Stats())}+// handleRotate moves a name's address in the zone and echoes the state, whose+// Zone map is read back rather than remembered -- so khaos's observable can name+// the address the rotation replaced rather than the one it asked for.+func (r *Resolver) handleRotate(w http.ResponseWriter, req *http.Request) {+ var rotation Rotation++ decoder := json.NewDecoder(+ http.MaxBytesReader(w, req.Body, controlBodyLimit),+ )+ if err := decoder.Decode(&rotation); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }++ previous, err := r.Rotate(rotation.Name, rotation.Address)+ if err != nil {+ http.Error(w, err.Error(), http.StatusUnprocessableEntity)+ return+ }++ stats := r.Stats()+ stats.Rotated = previous++ r.writeJSON(w, stats)+}++// Rotation is one zone change, and it is a different request shape from a Trap+// on purpose: arming a fault and moving the zone are different acts.+type Rotation struct {+ Name string `json:"name"`+ Address string `json:"address"`+}+// handleArm installs a fault and echoes the resulting state. The echo says// which branch was chosen, not what a target would receive, so it is never// what a fault is confirmed by.diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.goindex 17c6b178..d8e9d93a 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go@@ -18,6 +18,10 @@ const (testUpstream = "upstream.elsewhere.test."testUpAddr = "10.88.88.8"loopbackAny = "127.0.0.1:0"++ // Distinct from DefaultWrongAddress so the oracle census can tell an armed+ // rotation from an armed redirect.+ testRotatedAddress = "10.99.99.9")// testUpstreamServer is a resolver khaos is NOT authoritative for, so the@@ -116,6 +120,13 @@ func startResolver(t *testing.T) (*Resolver, *Client) {testControl,)+ // The zone SNAPSHOT, taken before any fault, exactly as the CLI takes it+ // when it attaches to a running resolver. Without it the oracle has no+ // record of what a name answered before khaos touched it, which the+ // rotation's confirmation needs and which wrong-address's observable has+ // been printing as "unknown" in every test until now.+ client.Zone = resolver.Zone()+return resolver, client}@@ -152,6 +163,13 @@ func deliverableKinds() []Kind {func trapFor(kind Kind) Trap {trap := Trap{Kind: kind, Host: testHost}+ // A rotation needs a destination, and a DIFFERENT one from wrong-address's+ // default: asking the wrong-address oracle about an armed rotation has to+ // REFUTE, and it cannot if both move the name to the same place.+ if kind == AddressRotated {+ trap.Address = testRotatedAddress+ }+if kind == SlowResponse {// Well above the oracle's differential margin and well under the// embedded resolver's ceiling, so the control is not deciding on noise.diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.goindex 7bfb475f..1028ec37 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go@@ -29,11 +29,13 @@ const (observeServFail = "answering SERVFAIL for %s while still resolving %s"observeSlow = "answering %s correctly after %s, against %s for the control on the same resolver"observeWrong = "answering %s with %v instead of %s"- observeChain = "answering %s with a %d-hop CNAME ending at %s"- observeLoop = "answering %s with a CNAME cycle through %v"- observeTrunc = "setting the truncation bit for %s over UDP with an empty answer, and serving %v completely over TCP"- observeTruncNo = "setting the truncation bit for %s with an empty answer over UDP and over TCP, so no transport ever completes"- observeTTL = "answering %s with a TTL of %d, against %d for %s on the same resolver"++ observeRotated = "%s now answers %v where the zone served %s before khaos moved it, from the resolver's own honest path rather than from a fault, so a target that asks again follows the rotation"+ observeChain = "answering %s with a %d-hop CNAME ending at %s"+ observeLoop = "answering %s with a CNAME cycle through %v"+ observeTrunc = "setting the truncation bit for %s over UDP with an empty answer, and serving %v completely over TCP"+ observeTruncNo = "setting the truncation bit for %s with an empty answer over UDP and over TCP, so no transport ever completes"+ observeTTL = "answering %s with a TTL of %d, against %d for %s on the same resolver"refuteControl = "the control name %s did not resolve through this resolver (%v), so the resolver is broken rather than lying"refuteNoFault = "%s still resolves to %v through this resolver, so nothing is armed against it"@@ -44,6 +46,9 @@ const (refuteSlowWrong = "%s answered %v late instead of %s, so the answer was changed as well as delayed and this fault changes only the timing"refuteWrong = "%s answered %v, which does not contain %s"refuteWrongIndirect = "%s answered the right address through CNAMEs %v: that is the chain fault, not a direct redirect"+ refuteRotated = "%s answered %v, which does not contain the address khaos rotated it to, %s"+ refuteRotatedUnknown = "%s answered the rotated address and khaos holds no record of what the zone served before the move, so it cannot show a rotation happened rather than a name that always pointed there"+ refuteRotatedSame = "%s answered %s both before and after the rotation, so nothing moved"refuteChain = "%s answered %d CNAME hops, and a chain needs %d"refuteChainCycle = "%s answered CNAMEs %v, which cycle: that is the loop fault, not a chain"refuteTruncReach = "%s could not be reached over TCP at all (%v), which is an unreachable resolver rather than one that truncates on both transports"@@ -116,6 +121,9 @@ func (c *Client) Observe(trap Trap) (string, bool, error) {case TTLZero, TTLLong:return ttlObservation(trap, faulted, control, c.ControlName)+ case AddressRotated:+ return rotatedObservation(trap, faulted, c.honest(trap.Host))+default:return refutation(refuteUnknown, string(trap.Kind))}@@ -178,6 +186,48 @@ func wrongObservation(return fmt.Sprintf(observeWrong, trap.Host, addresses, honest), true, nil}+// rotatedObservation is the rotation's own observable, and it is DISTINCT from+// wrong-address's on purpose: that one answers wrongly while the zone stays+// put, and this one moved the zone, so the honest path serves the new address+// too. Naming both addresses is what tells an operator which act khaos+// performed.+//+// The snapshot the client holds was taken before the fault, so it is the+// address the rotation replaced. Requiring the two to DIFFER is the read-back+// half of the confirmation: Rotate already refuses a move to the address a name+// already answers with, and this proves the resolver served the move.+func rotatedObservation(+ trap Trap,+ faulted answer,+ before string,+) (string, bool, error) {+ addresses := faulted.addresses()+ want := trap.address()++ if !slices.Contains(addresses, want) {+ return refutation(refuteRotated, trap.Host, addresses, want)+ }++ if hops := faulted.cnames(); len(hops) > 0 {+ return refutation(refuteWrongIndirect, trap.Host, hops)+ }++ if before == "" {+ return refutation(refuteRotatedUnknown, trap.Host)+ }++ if before == want {+ return refutation(refuteRotatedSame, trap.Host, want)+ }++ return fmt.Sprintf(+ observeRotated,+ trap.Host,+ addresses,+ before,+ ), true, nil+}+func (c *Client) honest(host string) string {if c.Zone == nil {return ""diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.goindex 691eb29b..8a8d86d2 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go@@ -142,10 +142,25 @@ func rettl(msg *dns.Msg, ttl uint32) *dns.Msg {// validAddress refuses a malformed WrongAddress magnitude before the trap arms,// where it would answer an empty NOERROR and read as a target that resolved// nothing.+// validAddress requires IPv4, and that is narrower than "parses as an IP" for a+// measured reason. Every khaos probe asks for an A record, and addressAnswer+// answers an empty NOERROR when the family asked for is not the family of the+// address it was given -- correctly, since inventing an A record from an IPv6+// address builds nothing (measured: `dns: bad A A`). So an IPv6 address passes+// a parse check, arms cleanly, and then makes khaos serve an EMPTY ANSWER while+// reporting that it armed a redirect. Refusing it here means the operator finds+// out when they ask for it rather than reading a verdict about a fault khaos+// never served.func validAddress(address string) error {- if net.ParseIP(address) == nil {+ parsed := net.ParseIP(address)+ if parsed == nil {return fmt.Errorf(errBadAddress, ErrBadAddress, address)}++ if parsed.To4() == nil {+ return fmt.Errorf(errNotV4, ErrBadAddress, address)+ }+return nil}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.goindex 324ba438..e6a472e7 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go@@ -47,6 +47,7 @@ const (errControlName = "%w: %q is not in the zone, so a confirmation could never use it as a control"errFaultedHost = "%w: %s is the confirmation control and cannot also be the faulted name"errOutsideZone = "%w: %s is not a name this resolver is authoritative for (zone: %s)"+ errSameAddress = "%w: %s already answers with %s, so rotating it there changes nothing and khaos would measure a rotation that never happened"errNoZoneTarget = "%w: this resolver was given no confirmation control name")@@ -70,6 +71,10 @@ var ErrControlIsHost = errors.New("the confirmation control cannot be the name under fault",)+var ErrSameAddress = errors.New(+ "a rotation to the address a name already answers with is not a rotation",+)+var ErrOutsideZone = errors.New("this resolver is not authoritative for that name",)@@ -102,6 +107,7 @@ type Resolver struct {mu sync.Mutextrap Trap+ rotated rotationservers []*dns.Serverarrival map[string]int64queries map[string]int64@@ -311,6 +317,26 @@ func (r *Resolver) Arm(trap Trap) error {}}+ // The one trap whose arming moves the ZONE rather than installing a wrong+ // answer, so the honest path serves the new address afterwards and a target+ // that asks again follows it. The previous address is recorded here, which+ // is what makes Disarm a restore rather than a guess.+ // INERT skips the rotation and keeps everything else, which is the whole+ // point of the control: an inert arm that still moved the zone would be+ // confirmed by an oracle reading the served answer, and the one negative+ // control this package relies on would pass against a mutation it was+ // built to catch. Measured -- it did.+ if trap.Kind == AddressRotated && !trap.Inert {+ previous, err := r.Rotate(trap.Host, trap.Address)+ if err != nil {+ return err+ }++ r.mu.Lock()+ r.rotated = rotation{name: trap.Host, from: previous}+ r.mu.Unlock()+ }+r.mu.Lock()r.trap = trapr.mu.Unlock()@@ -338,10 +364,74 @@ func (r *Resolver) inZone(host string) error {)}+// Rotate changes the ADDRESS a name answers with, which is a different act from+// arming a fault: a fault is a wrong answer the resolver gives while the zone+// stays put, and this moves the zone itself. The honest answer is served from+// the same table afterwards, so a target that asks again gets the new address+// and nothing else about the resolver changes.+//+// It is the operational act an operator performs for DNS-based failover or+// endpoint revocation, and the finding khaos measures is whether it reached the+// target at all.+func (r *Resolver) Rotate(name, address string) (string, error) {+ if err := validAddress(address); err != nil {+ return "", err+ }++ canonical := dns.CanonicalName(name)++ r.mu.Lock()+ defer r.mu.Unlock()++ previous, ok := r.zone[canonical]+ if !ok {+ names := make([]string, 0, len(r.zone))+ for zoneName := range r.zone {+ names = append(names, zoneName)+ }++ return "", fmt.Errorf(+ errOutsideZone,+ ErrOutsideZone,+ name,+ joinWith(sorted(names), ", "),+ )+ }++ if previous == address {+ return "", fmt.Errorf(errSameAddress, ErrSameAddress, name, address)+ }++ r.zone[canonical] = address++ return previous, nil+}++// Disarm removes the fault and RESTORES a rotation, because khaos has to leave+// the zone as it found it: a rotation that outlived the run would be khaos+// changing an operator's resolution permanently.func (r *Resolver) Disarm() {r.mu.Lock()+ moved := r.rotated+ r.rotated = rotation{}r.trap = Trap{}r.mu.Unlock()++ if moved.name == "" {+ return+ }++ // Best effort and deliberately not reported: Disarm has no error channel,+ // and the restore can only fail for a name that is no longer in the zone,+ // which this resolver has no path to.+ _, _ = r.Rotate(moved.name, moved.from)+}++// rotation is what a restore needs: the name and the address it answered with+// before khaos moved it.+type rotation struct {+ name string+ from string}func (r *Resolver) armed() Trap {@@ -377,6 +467,10 @@ type Stats struct {// address the fault replaced.Zone map[string]string `json:"zone"`+ // Rotated is the address a rotation REPLACED, echoed by that one route so+ // khaos's observable can name what the zone used to say.+ Rotated string `json:"rotated,omitempty"`+// Only the queries khaos was AUTHORITATIVE for, keyed by source and name.// An arrival count alone goes non-zero for a target resolving anything else.Queries map[string]int64 `json:"queries"`@@ -502,6 +596,13 @@ func (r *Resolver) serveFault(Address: trap.address(),}, chainAnswer)+ case AddressRotated:+ // The HONEST answer, deliberately: this trap moved the zone, so the+ // honest path already serves the new address. Synthesizing from the+ // trap instead would make it a wrong-answer fault and khaos would be+ // serving a redirect while reporting a rotation.+ return r.synthesize(req, question, honest, addressAnswer)+case CNAMELoop:return r.synthesize(req, question, honest, loopAnswer)diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/rotate_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/rotate_test.gonew file mode 100644index 00000000..d20b1f0e--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/rotate_test.go@@ -0,0 +1,107 @@+// ©AngelaMos | 2026+// rotate_test.go++package dnsrv++import (+ "context"+ "errors"+ "testing"++ "github.com/miekg/dns"+)++// A rotation moves the ZONE, which is a different act from arming a fault: a+// fault is a wrong answer given while the zone stays put. Research 05 section+// 6.5 is the polarity this serves.+func TestARotationChangesWhatTheNameAnswersWith(t *testing.T) {+ resolver, _ := startResolver(t)++ const rotated = "10.244.244.77"++ previous, err := resolver.Rotate(testHost, rotated)+ if err != nil {+ t.Fatalf("rotate: %v", err)+ }++ if previous != testAddress {+ t.Fatalf(+ "the rotation reported %q as the address it replaced, want %q: khaos's observable names what the zone USED to say",+ previous,+ testAddress,+ )+ }++ if got := resolver.Stats().Zone[dns.CanonicalName(testHost)]; got != rotated {+ t.Fatalf("after the rotation the zone says %q, want %q", got, rotated)+ }+}++// A rotation to the address a name already answers with is not a rotation, and+// accepting it would let khaos measure a change that never happened -- the same+// reason the credential tier's inert control must be REJECTED by confirmation.+func TestARotationThatChangesNothingIsRefused(t *testing.T) {+ resolver, _ := startResolver(t)++ if _, err := resolver.Rotate(+ testHost,+ testAddress,+ ); !errors.Is(err, ErrSameAddress) {+ t.Fatalf("rotating to the current address returned %v", err)+ }+}++// And a name outside the zone, for the same reason arming one is refused: it+// would succeed, answer nothing, and report a survival against nothing.+func TestARotationOutsideTheZoneIsRefused(t *testing.T) {+ resolver, _ := startResolver(t)++ if _, err := resolver.Rotate(+ "not-in-this-zone.invalid.",+ "10.244.244.77",+ ); !errors.Is(err, ErrOutsideZone) {+ t.Fatalf("rotating a name outside the zone returned %v", err)+ }+}++func TestARotationToAnUnusableAddressIsRefused(t *testing.T) {+ resolver, _ := startResolver(t)++ for _, address := range []string{"", "not-an-address", "::1"} {+ if _, err := resolver.Rotate(testHost, address); err == nil {+ t.Fatalf("rotating to %q was accepted", address)+ }+ }+}++// The control surface carries it, nonce-guarded like every other route, and a+// failure is an ERROR rather than a silent no-op: that is what lets a failed+// rotation void a run instead of leaving khaos measuring a name nobody moved.+func TestTheControlSurfaceRotatesAndRefusesWithoutTheNonce(t *testing.T) {+ _, client := startResolver(t)++ stats, err := client.Rotate(context.Background(), testHost, "10.244.244.78")+ if err != nil {+ t.Fatalf("rotate through the control surface: %v", err)+ }++ if stats.Rotated != testAddress {+ t.Fatalf(+ "the echo reports %q as the replaced address, want %q",+ stats.Rotated,+ testAddress,+ )+ }++ wrong := NewClient(client.DNS, client.Control, "not-the-nonce", testControl)++ if _, err := wrong.Rotate(+ context.Background(),+ testHost,+ "10.244.244.79",+ ); err == nil {+ t.Fatal(+ "a rotation was accepted without the nonce, so anything that can reach the control surface can move the zone mid-run",+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.goindex 8d624b1b..413ea659 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go@@ -57,6 +57,15 @@ const (TTLZero Kind = "ttl-zero"TTLLong Kind = "ttl-long"+ // AddressRotated MOVES THE ZONE rather than answering it wrongly, which is+ // what makes it the only trap whose arming changes what the HONEST path+ // serves. It is the operational act behind DNS-based failover and endpoint+ // revocation, and the finding is whether it reached the target at all --+ // research 05 section 6.5 carries the polarity, which is NOT inverted: the+ // rotation points at the same endpoint `wrong-address` uses, so a target+ // that follows it starts trusting a host on the strength of a name.+ AddressRotated Kind = "address-rotated"+// REFUSED BY KHAOS: the embedded resolver rewrites both to SERVFAIL, and a// real library branches on the rcode, so delivering one under its own name// would be a wrong measurement rather than a fault.@@ -93,6 +102,8 @@ const (describeTTLZero = "answer %s correctly with a TTL of zero, so nothing may cache it"describeTTLLong = "answer %s correctly with a TTL of %d seconds, so a cache pins it for a week"+ describeRotated = "MOVE %s's dependency to %s in the zone itself, the way an operator performs a DNS failover or an endpoint revocation, and measure whether the change reaches it at all"+describeRefused = "answer %s REFUSED, which khaos cannot deliver"describeTimeout = "drop %s's query entirely, which khaos cannot deliver"@@ -105,6 +116,8 @@ const (errDelayRange = "%w: %s was asked for a %s delay, and the ceiling is %s"errNoHost = "%w: %s needs the dependency's name, and none was given"errBadAddress = "%w: %q is not an IP address"+ errNoRotation = "%w: %s needs the address to move the name TO; a rotation with no destination is not a rotation"+ errNotV4 = "%w: %q is an IPv6 address, and this resolver answers the A queries khaos's probes send; an IPv6 answer to an A query is an empty NOERROR, so khaos would serve nothing while reporting that it armed a redirect"// Each names the mechanism, what the target would really have received, and// what to ask for instead.@@ -128,6 +141,10 @@ var ErrDelayRange = errors.New(var ErrNoHost = errors.New("this DNS fault needs a name to act on")+var ErrNoRotation = errors.New(+ "a rotation needs the address it moves the name to",+)+var ErrBadAddress = errors.New("that is not an IP address")// Trap travels through the journal, so every field has to be enough to@@ -159,6 +176,7 @@ func Kinds() []string {string(TruncationNoTCP),string(TTLZero),string(TTLLong),+ string(AddressRotated),string(Refused),string(Timeout),}@@ -211,6 +229,8 @@ func (t Trap) Describe(target string) string {return fmt.Sprintf(describeTTLZero, target)case TTLLong:return fmt.Sprintf(describeTTLLong, target, LongTTL)+ case AddressRotated:+ return fmt.Sprintf(describeRotated, target, t.address())case Refused:return fmt.Sprintf(describeRefused, target)case Timeout:@@ -271,6 +291,19 @@ func (t Trap) ValidateShape() error {}}+ // A rotation needs its destination at SHAPE time rather than defaulting to+ // one, because moving a name to an address khaos chose for it is not an+ // operational act anybody performs.+ if t.Kind == AddressRotated {+ if t.Address == "" {+ return fmt.Errorf(errNoRotation, ErrNoRotation, string(t.Kind))+ }++ if err := validAddress(t.Address); err != nil {+ return err+ }+ }+return nil}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.goindex 9a4ac000..9dbe5a94 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/actions.go@@ -31,6 +31,7 @@ const (errNoNetwork = "%s has no network address at all, so khaos has nothing to dial it on"errBridge = "read the gateway of the default bridge network: %w"errNoBridge = "the default bridge network reports no gateway address, so khaos cannot find an address a container can reach it on"+ errGateways = "list the networks the daemon manages, to find the addresses khaos may bind a listener on: %w"bridgeNetwork = "bridge"envPairParts = 2@@ -256,6 +257,33 @@ func (c *Client) BridgeGateway(ctx context.Context) (string, error) {return "", errors.New(errNoBridge)}+// Gateways is every address DOCKER manages as a network gateway, which is a+// narrower question than "is this a bridge". `/sys/class/net/<if>/bridge`+// exists for every kernel bridge device: measured on this host, that includes+// a CNI pod bridge, a swarm overlay's gateway bridge and forty-nine+// user-defined ones carrying unrelated projects' containers -- and on a host+// whose LAN interface is enslaved to a bridge, the routable LAN address itself.+// khaos binds listeners that answer whatever asks them, so the allowed set has+// to come from the daemon rather than from sysfs.+func (c *Client) Gateways(ctx context.Context) ([]string, error) {+ networks, err := c.api.NetworkList(ctx, client.NetworkListOptions{})+ if err != nil {+ return nil, fmt.Errorf(errGateways, err)+ }++ addresses := []string{}++ for _, network := range networks.Items {+ for _, config := range network.IPAM.Config {+ if config.Gateway.IsValid() {+ addresses = append(addresses, config.Gateway.String())+ }+ }+ }++ return addresses, nil+}+// classify turns a daemon NotFound into ErrContainerGone, so it is only valid// where the MISSING THING IS THE CONTAINER; NetworkConnect returns NotFound for// a missing NETWORK, so see classifyNetwork.diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.goindex 0cf583c1..66000b1d 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go@@ -99,6 +99,11 @@ type Shape struct {ClosedWhen []probe.Disposition+ // RotatedWhat names a zone rotation, and it is the one observable read as a+ // finding when it is ZERO: a target that never asked again never saw the+ // rotation, which is a revocation that did not take effect.+ RotatedWhat string+// The liveness guard for a fault whose landing the probe streams cannot see.Consulted func() (int64, error)ConsultedWhat string@@ -312,6 +317,7 @@ func Build(CacheRequired: shape.CacheRequired,DeliveredWhat: shape.DeliveredWhat,DeliveredHint: shape.DeliveredHint,+ RotatedWhat: shape.RotatedWhat,DNSTrap: shape.DNSTrap,ResolverNetwork: shape.ResolverNetwork,DNSHost: shape.DNSHost,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.goindex 31052bb7..0ff8a9f2 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/credtier.go@@ -241,7 +241,10 @@ func dispositionOf(result dbx.Result) (probe.Disposition, string) {case dbx.Refused, dbx.Locked, dbx.Expired, dbx.MustReset, dbx.Terminated:return probe.Rejected, result.Detailcase dbx.Unreachable:- return probe.Unavailable, result.Detail+ // An engine khaos could not reach ANSWERED nothing, which is not the+ // same observation as a service answering 503 - and the pooled+ // experiments declare the second as their fail-closed outcome.+ return probe.Unreachable, result.Detaildefault:return probe.ProbeFailed, result.Detail}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.goindex 5122396d..03f5b7ee 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go@@ -19,6 +19,7 @@ const (NameDNSTruncationNoTCP = "dns.truncation_no_tcp"NameDNSTTLZero = "dns.ttl_zero"NameDNSTTLLong = "dns.ttl_long"+ NameDNSTTLRotate = "dns.ttl_rotate"NameDNSRefused = "dns.refused"NameDNSTimeout = "dns.timeout")@@ -44,6 +45,8 @@ const (summaryTTLLong = "answer the dependency's name correctly with a week-long TTL and measure whether the target honours it, which is how long a stale answer would keep being trusted after the dependency moved"+ summaryTTLRotate = "MOVE the dependency's address in the zone, the way an operator performs a DNS failover or an endpoint revocation, and measure whether the change reaches the target at all -- a client still talking to the address that was rotated away is a revocation that never took effect"+summaryRefused = "REFUSED through Docker's embedded resolver, which khaos refuses to approximate; the resolver rewrites it to SERVFAIL and a real library branches on the rcode"summaryTimeout = "a dropped query through Docker's embedded resolver, which khaos refuses to approximate; the resolver answers SERVFAIL itself at a measured 4.001s and a client never sees a socket timeout"@@ -105,6 +108,13 @@ var dnsCatalogue = []dnsSpec{fault: dnsrv.TTLLong,needsCache: true,},+ {+ name: NameDNSTTLRotate,+ summary: summaryTTLRotate,+ fault: dnsrv.AddressRotated,+ extra: []Param{dnsAddressParam},+ rotates: true,+ },{name: NameDNSRefused,summary: summaryRefused,diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.goindex b72f7d2f..189c8fcd 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go@@ -42,6 +42,8 @@ const (deliveredResolve = "re-resolve its dependency's name"+ rotatedWhat = "moved %s to %s in the zone"+// What to check when a DNS fault was armed, confirmed and never met. It// lives here rather than in the verdict's shared sentence because the only// other tier carrying that guard reaches it for an unrelated reason.@@ -96,6 +98,12 @@ type dnsSpec struct {// than a clean survival it never earned.needsCache bool+ // rotates MOVES the zone rather than answering it wrongly, and it is exempt+ // from the delivery guard for the same reason the TTL pair is: a target+ // that never asks again IS the measurement here, not a void run. Research+ // 05 section 6.5 carries the polarity.+ rotates bool+extra []Param}@@ -160,6 +168,19 @@ func buildDNS(spec dnsSpec) func(*Params) (Shape, error) {shape.CacheRequired = cacheRequiredTTL}+ // The rotation's own observable. A target that never asks again did not+ // see the rotation, which is a finding khaos must state and must not+ // call a bypass: it is still talking to the honest endpoint it used+ // during the baseline, so no credential got through that would not+ // have.+ if spec.rotates {+ shape.RotatedWhat = fmt.Sprintf(+ rotatedWhat,+ host,+ trap.Address,+ )+ }+return shape, nil}}@@ -206,7 +227,7 @@ func heldCaveatFor(spec dnsSpec) string {// deliveredFor is derived from the same needsCache declaration that turns on the// cache reading, so the two cannot disagree about which faults are about caching.func deliveredFor(spec dnsSpec) string {- if spec.needsCache {+ if spec.needsCache || spec.rotates {return ""}@@ -214,7 +235,7 @@ func deliveredFor(spec dnsSpec) string {}func hintFor(spec dnsSpec) string {- if spec.needsCache {+ if spec.needsCache || spec.rotates {return ""}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.goindex 24d4c539..8caed1aa 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go@@ -314,6 +314,7 @@ func (e Engine) measureFault(observation.Delivered = after - beforeobservation.DeliveredWhat = spec.DeliveredWhatobservation.DeliveredHint = spec.DeliveredHint+ observation.RotatedWhat = spec.RotatedWhatobservation.Refusals = refusedAfter - refusedBefore// NOT folded into CounterUnreadable, which voids the run: this count feedsdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.goindex ff0e5d36..21bb1c5b 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go@@ -157,6 +157,10 @@ type Experiment struct {Consulted func() (int64, error)ConsultedWhat string+ // RotatedWhat names a zone rotation whose arrival khaos measures by whether+ // the target asked again at all.+ RotatedWhat string+// Refusals is the count of definitive denials the dependency issued, which// only the credential under test is ever answered with.Refusals func() (int64, error)diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.goindex 8450e39c..b5adbaed 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/integration_test.go@@ -231,11 +231,16 @@ func TestAnUnreachableEndpointIsNotMistakenForARejection(t *testing.T) {outcome := spec.Probes.Invalid.Run(context.Background())- if outcome.Disposition != probe.Unavailable {+ // Unreachable rather than Unavailable since the dispositions were split:+ // nothing ANSWERED here. Reading it as a rejection would make an outage+ // look like a control doing its job, and reading it as an answered 503+ // would let a control that declares a 503 its fail-closed outcome cover a+ // target that had died.+ if outcome.Disposition != probe.Unreachable {t.Fatalf(- "a connection failure was classified as %s; reading it as a rejection would "+- "make an outage look like a control doing its job",+ "a connection failure was classified as %s, want %s",outcome.Disposition,+ probe.Unreachable,)}}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/rotation_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/rotation_test.gonew file mode 100644index 00000000..919e738b--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/rotation_test.go@@ -0,0 +1,169 @@+// ©AngelaMos | 2026+// rotation_test.go++package experiment++import (+ "strings"+ "testing"+)++const movedWhat = "moved dependency.khaos.test. to 10.244.244.20 in the zone"++func rotated(queriesDuringFault int64) Observation {+ return Observation{+ Baseline: healthyBaseline(),+ Fault: heldFaultWindow(),+ Recovery: healthyRecovery(),+ FaultConfirmed: true,+ RotatedWhat: movedWhat,+ ConsultedWhat: ConsultsResolver,+ Consulted: 2,+ Delivered: queriesDuringFault,+ }+}++// The whole question for the one fault that MOVES the zone: did the change+// reach the target at all. khaos states it in BOTH directions, so a clean run+// is an answer rather than silence.+func TestTheRotationIsReportedInBothDirections(t *testing.T) {+ followed := Judge(rotated(74), dnsPolarity)++ if !hasReason(followed, "so the rotation reached it") {+ t.Fatalf(+ "a target that asked again was not told the rotation reached it: %v",+ followed.Reasons,+ )+ }++ unseen := Judge(rotated(0), dnsPolarity)++ if !hasReason(unseen, "ZERO during the fault window") {+ t.Fatalf(+ "a target that never asked again was not told the rotation missed it: %v",+ unseen.Reasons,+ )+ }++ // Both carry the counts and the act, or an operator cannot audit either.+ for _, verdict := range []Verdict{followed, unseen} {+ if !hasReason(verdict, movedWhat) {+ t.Fatalf(+ "the reason does not name what khaos moved: %v",+ verdict.Reasons,+ )+ }+ }+}++// AND IT IS NOT A BYPASS. No credential got through that would not have before:+// the target is still reaching the endpoint it reached during the baseline.+// Reporting PERMITTED here would be khaos claiming a bypass it did not observe,+// which is the overstatement this family's polarity exists to prevent.+func TestARotationThatMissedTheTargetIsNotCalledABypass(t *testing.T) {+ verdict := Judge(rotated(0), dnsPolarity)++ if verdict.Direction != Held {+ t.Fatalf(+ "a target that never saw the rotation was judged %s: %v",+ verdict.Direction,+ verdict.Reasons,+ )+ }++ if verdict.Defect {+ t.Fatalf("it was reported as a defect: %v", verdict.Reasons)+ }++ if !hasReason(verdict, "NOT a credential bypass") {+ t.Fatalf(+ "khaos did not say it was reporting no bypass, so a reader has to infer it: %v",+ verdict.Reasons,+ )+ }+}++// And an experiment that performed no rotation must be told nothing about one,+// or every other fault in every family gains a sentence about a zone it never+// touched.+func TestAnExperimentThatRotatedNothingSaysNothingAboutIt(t *testing.T) {+ observation := Observation{+ Baseline: healthyBaseline(),+ Fault: heldFaultWindow(),+ Recovery: healthyRecovery(),+ FaultConfirmed: true,+ }++ for _, reason := range Judge(observation, dnsPolarity).Reasons {+ if strings.Contains(reason, "in the zone") ||+ strings.Contains(reason, "rotation") {+ t.Fatalf(+ "an experiment that rotated nothing was told about one: %q",+ reason,+ )+ }+ }+}++// The rotation is EXEMPT from the delivery guard, and that exemption is the+// whole design: window one's guard reads "the target never met the fault" as a+// void run, and here not asking again IS the measurement.+func TestTheRotationIsExemptFromTheDeliveryGuard(t *testing.T) {+ // The BUILT experiment, not a hand-made observation: the exemption lives in+ // the builder, so an observation written by hand proves nothing about it.+ // Measured by gutting, which stayed green against the hand-made version.+ definition, err := Lookup(NameDNSTTLRotate)+ if err != nil {+ t.Fatalf("lookup: %v", err)+ }++ spec, err := Build(definition.Name, fill(definition.Params), fastTiming)+ if err != nil {+ t.Fatalf("build: %v", err)+ }++ if spec.DeliveredWhat != "" {+ t.Fatalf(+ "the rotation declares the delivery guard (%q), which aborts the run on the very observation this experiment exists to measure",+ spec.DeliveredWhat,+ )+ }++ if spec.RotatedWhat == "" {+ t.Fatal(+ "the rotation declares no observable of its own, so a run that measured it would say nothing",+ )+ }++ // And the rest of the family still DOES declare it, or the exemption was+ // applied to the whole family rather than to the one fault that earns it.+ guarded := 0++ for _, other := range Catalog() {+ if other.Family != FamilyDNS || other.Name == NameDNSTTLRotate {+ continue+ }++ built, err := Build(other.Name, fill(other.Params), fastTiming)+ if err != nil {+ t.Fatalf("build %s: %v", other.Name, err)+ }++ if built.DeliveredWhat != "" {+ guarded+++ }+ }++ if guarded == 0 {+ t.Fatal(+ "no other DNS fault carries the delivery guard, so the exemption was applied to the whole family",+ )+ }++ if Judge(rotated(0), dnsPolarity).Epistemic != Passed {+ t.Fatalf(+ "a target that never asked again produced %s rather than a readable verdict",+ Judge(rotated(0), dnsPolarity).Epistemic,+ )+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.goindex 605423be..b6f64fb3 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/shift_test.go@@ -288,6 +288,67 @@ func TestA503RefusalEndsTheRunOfAcceptances(t *testing.T) {}}+// A target that DIED under the fault is not a target that failed closed, and+// the two were one disposition. This experiment declares a 503 as its refusal,+// so a crashed target was reported `deviated / DENIED` with "the valid+// credential stopped being accepted" -- when it stopped ANSWERING, and this+// fault takes nothing away from the target that could explain a crash.+func TestACrashedTargetIsNotReadAsFailingClosed(t *testing.T) {+ observation := shifted(probe.Rejected, probe.Rejected)+ observation.ClosedWhen = []probe.Disposition{probe.Unavailable}++ // Every probe dies on the connection, in both streams, which is what+ // `unanswered` reads across.+ observation.Fault = probe.Window{+ Valid: stream(+ probe.Valid,+ repeat(probe.Unreachable, MinimumPaired*2)...),+ Invalid: stream(+ probe.Invalid,+ repeat(probe.Unreachable, MinimumPaired*2)...),+ Revoked: stream(+ probe.Revoked,+ repeat(probe.Unreachable, MinimumPaired*2)...),+ }++ verdict := Judge(observation, authzPolarity)++ if verdict.Direction != Crashed {+ t.Fatalf(+ "a target that answered nothing at all was judged %s: %v",+ verdict.Direction,+ verdict.Reasons,+ )+ }++ // And the contrast, on the same declared set: a target ANSWERING 503 has+ // refused, and that still has to read as this control failing closed.+ answering := shifted(probe.Rejected, probe.Rejected)+ answering.ClosedWhen = []probe.Disposition{probe.Unavailable}+ answering.Fault = probe.Window{+ Valid: stream(+ probe.Valid,+ repeat(probe.Unavailable, MinimumPaired*2)...),+ Invalid: stream(+ probe.Invalid,+ repeat(probe.Rejected, MinimumPaired*2)...),+ Revoked: stream(+ probe.Revoked,+ repeat(probe.Unavailable, MinimumPaired*2)...),+ }++ if direction := Judge(+ answering,+ authzPolarity,+ ).Direction; direction != Denied {+ t.Fatalf(+ "a target answering 503 to everything was judged %s, want %s: it answered, and this control declares that its refusal",+ direction,+ Denied,+ )+ }+}+// The exposure figure has to describe the SAME set as the count beside it. The// count is the trailing run; a first-to-last span over the whole window// includes the period the target spent refusing, and quoting the two in onediff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.goindex b1307862..ab738752 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go@@ -183,6 +183,16 @@ const (// could not get, and the fault may well be in effect.reasonShiftUnarmed = "khaos could not establish the second phase, so this run measured what the target does with a definitive deny and nothing about what it does when the dependency cannot answer; those are different branches and a correct caller needs both"+ // Stated, and deliberately NOT a bypass. No credential got through that+ // would not have before: the target is still talking to the endpoint it+ // used during the baseline. Calling it PERMITTED would be khaos claiming a+ // bypass it did not observe; saying nothing would hide the operational+ // finding, which is that a DNS-based revocation on this name would not have+ // taken effect either.+ reasonRotationUnseen = "khaos %s and the target never asked again: %d queries during the baseline and ZERO during the fault window. The rotation did not reach it, so a DNS-based failover or endpoint revocation on that name would not have taken effect either. This is NOT a credential bypass and khaos reports none -- the target is still reaching the address it was reaching before the rotation"++ reasonRotationFollowed = "khaos %s and the target asked again %d time(s) during the fault window, so the rotation reached it: a DNS-based failover or revocation on that name would take effect"+reasonShiftNotDelivered = "the second phase was armed and confirmed, and khaos did not observe the target meeting it: it did not %s once during that window, so it answered every probe from something it had already cached and nothing khaos changed there reached it. Window one treats that observation as a void run rather than a finding, and so does this one")@@ -250,6 +260,13 @@ type Observation struct {Delivered int64 `json:"delivered,omitempty"`DeliveredWhat string `json:"delivered_what,omitempty"`+ // RotatedWhat names a zone rotation khaos performed, and this is the one+ // count read as a FINDING when it is zero: a target that never asked again+ // never saw the rotation. Research 05 section 6.5 is the polarity, and the+ // finding is deliberately NOT a bypass -- the target is still talking to+ // the endpoint it used during the baseline.+ RotatedWhat string `json:"rotated_what,omitempty"`+// DeliveredHint is what to check, in this fault's own terms. The guard is// shared and the causes are not: a name the target never resolved again and// a decision it never asked for again have nothing in common to advise on.@@ -512,6 +529,10 @@ func finish(verdict.Reasons = append(verdict.Reasons, storm)}+ if rotation := rotationReading(observation); rotation != "" {+ verdict.Reasons = append(verdict.Reasons, rotation)+ }+// EVERY direction is put to the declared polarity, including HELD. For a// certificate fault a target that carries on unchanged has ACCEPTED the// broken certificate, so "nothing looks wrong" is the finding.@@ -587,6 +608,30 @@ func leaked(observation Observation) []string {return reasons}+// rotationReading states whether a zone rotation reached the target at all,+// which is the whole question for the one fault that MOVES the zone rather than+// answering it wrongly. Zero queries during the fault window is the finding;+// khaos says it in both directions so a clean run is not silence.+func rotationReading(observation Observation) string {+ if observation.RotatedWhat == "" {+ return ""+ }++ if observation.Delivered == 0 {+ return sprintf(+ reasonRotationUnseen,+ observation.RotatedWhat,+ observation.Consulted,+ )+ }++ return sprintf(+ reasonRotationFollowed,+ observation.RotatedWhat,+ observation.Delivered,+ )+}+// retryStorm reports the target asking again about a credential the dependency// has already definitively refused. It is a REASON rather than a direction: no// credential got through, and what it tells an operator is that their callerdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/bind_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/bind_test.gonew file mode 100644index 00000000..c6990023--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/bind_test.go@@ -0,0 +1,44 @@+// ©AngelaMos | 2026+// bind_test.go++package idp++import (+ "errors"+ "testing"++ "github.com/CarterPerez-dev/khaos/internal/bindguard"+)++// This provider mints credentials that are deliberately defective -- `alg:+// none`, a signature over a zero-length key, a token carrying its own verifying+// key -- and serves the key set a target resolves them against. It had no bind+// guard at all while the fault proxy had carried one since M5, so the rule was+// a property of one tier rather than of khaos. An audit found it.+func TestAnExposedBindIsRefused(t *testing.T) {+ for _, addr := range []string{"0.0.0.0:45999", ":45999", "[::]:45999"} {+ provider, err := New(Config{})+ if err != nil {+ t.Fatalf("build a provider: %v", err)+ }++ err = provider.Listen(addr)+ if err == nil {+ bound, _ := provider.Addr()+ _ = provider.Close()+ t.Fatalf(+ "the provider bound %s, which is reachable from outside this host",+ bound,+ )+ }++ if !errors.Is(err, bindguard.ErrUnsafeBind) {+ t.Fatalf("%s: err = %v, want ErrUnsafeBind", addr, err)+ }+ }++ // And loopback MUST still be accepted, or the refusal is a broken listener+ // rather than a guard. `listening` binds loopback and fails the test if it+ // is refused, which is the positive half.+ _ = listening(t)+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.goindex 97dcde30..547cabc4 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/idp/provider.go@@ -12,6 +12,8 @@ import ("sync""sync/atomic""time"++ "github.com/CarterPerez-dev/khaos/internal/bindguard")const (@@ -43,6 +45,11 @@ const (shutdownGrace = 2 * time.Second+ // It mints the credentials khaos's own probes present and serves the key+ // set a target resolves them against, so it does not belong on an+ // interface that target cannot reach anyway.+ SubjectProvider = "an identity provider that mints deliberately defective credentials"+errAlreadyListening = "%w: khaos's identity provider is already listening on %s"errNotListening = "%w: khaos's identity provider is not listening, so it has no address"errNoRevocable = "%w: this run never issued a revocable credential, so there is nothing to revoke"@@ -202,6 +209,10 @@ func (p *Provider) clock() time.Time {// Listen binds the address the target was built to address, not khaos's choice.func (p *Provider) Listen(addr string) error {+ if err := bindguard.Check(addr, SubjectProvider); err != nil {+ return err+ }+// Guard and bind under ONE acquisition, or two callers both bind and the// second orphans the first listener and its goroutine.p.mu.Lock()diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.goindex 0bbd264b..34210db8 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go@@ -28,7 +28,7 @@ const (settleWindow = 250 * time.Millisecond- subjectProxy = "a fault proxy in front of a dependency under test"+ SubjectProxy = "a fault proxy in front of a dependency under test"probePayload = "GET / HTTP/1.1\r\nHost: khaos.confirm\r\nConnection: close\r\n\r\n"@@ -78,7 +78,7 @@ type Proxy struct {// Open binds the proxy and refuses an address that would expose it.func Open(listenAddr, upstream string) (*Proxy, error) {- if err := bindguard.Check(listenAddr, subjectProxy); err != nil {+ if err := bindguard.Check(listenAddr, SubjectProxy); err != nil {return nil, err}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/bind_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/bind_test.gonew file mode 100644index 00000000..2492d4a3--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/bind_test.go@@ -0,0 +1,58 @@+// ©AngelaMos | 2026+// bind_test.go++package pki++import (+ "errors"+ "testing"++ "github.com/CarterPerez-dev/khaos/internal/bindguard"+)++// BOTH listeners, because they are different exposures and each had none. The+// TLS one presents certificates khaos deliberately broke -- expired, revoked,+// wrongly named, signed with a weak key -- and the HTTP one serves the CRL and+// OCSP responses that decide whether a certificate is still good. The fault+// proxy had carried this guard since M5 and neither of these did, so the rule+// was a property of one tier rather than of khaos. An audit found it.+func TestBothListenersRefuseAnExposedBind(t *testing.T) {+ for _, addr := range []string{"0.0.0.0:45999", ":45999", "[::]:45999"} {+ t.Run(addr, func(t *testing.T) {+ revocation := authority_t(t)++ err := revocation.Listen(addr)+ if err == nil {+ _ = revocation.Close()+ t.Fatalf(+ "the revocation endpoint bound %s, which is reachable from outside this host",+ addr,+ )+ }++ if !errors.Is(err, bindguard.ErrUnsafeBind) {+ t.Fatalf("Listen(%q) = %v, want ErrUnsafeBind", addr, err)+ }++ dependency := authority_t(t)++ err = dependency.ListenTLS(addr)+ if err == nil {+ _ = dependency.Close()+ t.Fatalf(+ "the dependency endpoint bound %s, which is reachable from outside this host",+ addr,+ )+ }++ if !errors.Is(err, bindguard.ErrUnsafeBind) {+ t.Fatalf("ListenTLS(%q) = %v, want ErrUnsafeBind", addr, err)+ }+ })+ }++ // And loopback MUST still be accepted on both, or the refusal is a broken+ // tier rather than a guard. `listening` binds both and fails the test if+ // either is refused.+ _ = listening(t)+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.goindex af452e8d..eace378e 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/server.go@@ -12,6 +12,8 @@ import ("net/http""sync""time"++ "github.com/CarterPerez-dev/khaos/internal/bindguard")const (@@ -24,6 +26,14 @@ const (healthBody = "ok"maxOCSPBody = 64 << 10+ // Each listener names what it is, because one sentinel now serves every+ // listener khaos opens and a refusal has to say which was about to be+ // exposed. The CA signs for one hostname and the revocation endpoint+ // decides whether a certificate is still good, so neither belongs on an+ // interface the target under test cannot reach anyway.+ SubjectRevocation = "a certificate authority's revocation endpoint"+ SubjectDependency = "a dependency khaos presents deliberately broken certificates on"+errNoListener = "%w: the authority is not listening, so %s cannot be reached"errBindFailed = "%w: binding %q: %v"errAlreadyUp = "%w: the authority is already listening on %s"@@ -65,6 +75,10 @@ func (a *Authority) SetBase(base string) {}func (a *Authority) Listen(addr string) error {+ if err := bindguard.Check(addr, SubjectRevocation); err != nil {+ return err+ }+a.listener.mu.Lock()defer a.listener.mu.Unlock()@@ -109,6 +123,10 @@ func (a *Authority) Listen(addr string) error {// certificate is resolved per handshake rather than fixed at bind time, so// arming takes effect on the next connection.func (a *Authority) ListenTLS(addr string) error {+ if err := bindguard.Check(addr, SubjectDependency); err != nil {+ return err+ }+a.listener.mu.Lock()defer a.listener.mu.Unlock()diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/answered_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/answered_test.gonew file mode 100644index 00000000..f41f2739--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/answered_test.go@@ -0,0 +1,111 @@+// ©AngelaMos | 2026+// answered_test.go++package probe++import (+ "context"+ "net"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"+)++// A target ANSWERING that it cannot serve and a target not answering at all are+// different observations, and they were one disposition. Four experiments+// declare the first as their fail-closed outcome, so the conflation made a+// crashed target read as one that had correctly refused.+func TestA503AndADeadTargetAreDifferentDispositions(t *testing.T) {+ serving := httptest.NewServer(+ http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusServiceUnavailable)+ }),+ )+ defer serving.Close()++ answered := NewHTTP(HTTPOptions{+ Name: "answered",+ URL: serving.URL,+ Credential: Valid,+ Timeout: time.Second,+ }).Run(context.Background())++ if answered.Disposition != Unavailable {+ t.Fatalf(+ "a target answering 503 produced %s, want %s",+ answered.Disposition,+ Unavailable,+ )+ }++ if answered.Status != http.StatusServiceUnavailable {+ t.Fatalf("the status was not recorded: %d", answered.Status)+ }++ // A listener that is closed: the connection is refused, and nothing+ // answers. This is the case a declared fail-closed set must NOT cover.+ dead := deadAddress(t)++ refused := NewHTTP(HTTPOptions{+ Name: "refused",+ URL: "http://" + dead,+ Credential: Valid,+ Timeout: time.Second,+ }).Run(context.Background())++ if refused.Disposition != Unreachable {+ t.Fatalf(+ "a target that refused the connection produced %s, want %s",+ refused.Disposition,+ Unreachable,+ )+ }++ if refused.Status != 0 {+ t.Fatalf(+ "a probe that was never answered carries status %d; zero is what says nothing answered",+ refused.Status,+ )+ }+}++// Both are real evidence about the target and neither is a decision about the+// credential, so they agree on the two properties every rate is built from.+// Changing either would move the crash check for every experiment.+func TestNeitherCountsAsAnAnswerAndBothAreCountable(t *testing.T) {+ for _, disposition := range []Disposition{Unavailable, Unreachable} {+ if disposition.Answered() {+ t.Fatalf(+ "%s reads as the target having answered, which would let a target that served nothing read as HELD for every experiment that declares no closed set",+ disposition,+ )+ }++ if !disposition.Countable() {+ t.Fatalf(+ "%s is excluded from every rate, so a target in that state would be invisible rather than measured",+ disposition,+ )+ }+ }+}++// deadAddress is a port nothing is listening on, obtained by binding and+// releasing it, so the dial is REFUSED rather than timing out.+func deadAddress(t *testing.T) string {+ t.Helper()++ listener, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen: %v", err)+ }++ addr := listener.Addr().String()++ if err := listener.Close(); err != nil {+ t.Fatalf("close: %v", err)+ }++ return addr+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.goindex 59b88ca5..937aa019 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/http.go@@ -200,7 +200,13 @@ func transportDisposition(err error) Disposition {if errors.Is(err, context.Canceled) {return ProbeFailed}- return Unavailable++ // Unreachable and not Unavailable: nothing answered. The target refusing+ // the connection, resetting it or closing it without a response are all+ // the target failing to ANSWER, which is a different observation from a+ // 503 with a status line, and four experiments declare the second as their+ // fail-closed outcome.+ return Unreachable}// statusDisposition maps a status to what it says about the CREDENTIAL, not todiff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.goindex c1a0b4a3..896cd49b 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/probe.go@@ -15,12 +15,22 @@ const (// wrong rather than as a target failure.Unknown Disposition = "unknown"- Accepted Disposition = "accepted"- Rejected Disposition = "rejected"- Redirected Disposition = "redirected"- Throttled Disposition = "throttled"+ Accepted Disposition = "accepted"+ Rejected Disposition = "rejected"+ Redirected Disposition = "redirected"+ Throttled Disposition = "throttled"++ // Unavailable is the target ANSWERING that it cannot serve: a 5xx with a+ // status line. Unreachable is the target not answering at all. They were+ // one value, and the conflation made a control's declared fail-closed+ // outcome cover a target that had DIED: four experiments declare a 503 as+ // their refusal, so a crashed target read as one that failed closed, with a+ // reason saying the valid credential "stopped being accepted". Keeping them+ // apart is what lets a declaration name one without the other.Unavailable Disposition = "unavailable"- TimedOut Disposition = "timed-out"+ Unreachable Disposition = "unreachable"++ TimedOut Disposition = "timed-out"// ProbeFailed is khaos's own fault and is excluded from every rate.ProbeFailed Disposition = "probe-failed"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/freeport_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/freeport_test.gonew file mode 100644index 00000000..d23d47d7--- /dev/null+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/freeport_test.go@@ -0,0 +1,116 @@+// ©AngelaMos | 2026+// freeport_test.go++package rangelab++import (+ "net"+ "testing"+)++// Testing loopback alone was weaker than it read. The published container ports+// go to 127.0.0.1, but four tiers bind the Docker bridge gateway, and a port+// free on one is not evidence it is free on the other. An audit found it.+func TestAPortTakenOnAnyRequiredAddressIsNotFree(t *testing.T) {+ // A second loopback address stands in for the bridge gateway: it is a+ // different address on the same host, which is the whole property under+ // test, and it needs no daemon.+ const other = "127.0.0.2"++ probe, err := net.Listen("tcp", net.JoinHostPort(other, "0"))+ if err != nil {+ t.Skipf("this host has no second loopback address to bind: %v", err)+ }++ taken := probe.Addr().(*net.TCPAddr).Port++ defer func() { _ = probe.Close() }()++ // Free on 127.0.0.1 and taken on the other address, which is exactly the+ // case the old check passed.+ if !freeOn(taken, []string{loopback}) {+ t.Skipf(+ "port %d is also taken on loopback, so this case is not set up",+ taken,+ )+ }++ if freeOn(taken, []string{loopback, other}) {+ t.Fatalf(+ "port %d read as free while something is listening on it at %s; a tier binding that address would fail where khaos said it would not",+ taken,+ other,+ )+ }+}++// AND FreePort has to actually USE the extra addresses, which the case above+// does not prove: it calls the predicate directly, so a FreePort that ignored+// its argument passed it. Measured by gutting, which stayed green.+//+// This is deterministic rather than probabilistic. 192.0.2.1 is TEST-NET-1 and+// is not assigned to this host, so NO port can be bound on it -- every+// candidate fails, and the only way FreePort can succeed is by not testing the+// address it was given.+func TestFreePortActuallyTestsTheAddressesItIsGiven(t *testing.T) {+ const unbindable = "192.0.2.1"++ if _, err := net.Listen(+ "tcp",+ net.JoinHostPort(unbindable, "0"),+ ); err == nil {+ t.Skip(+ "this host can bind TEST-NET-1, so it cannot stand in for an address no port is free on",+ )+ }++ if port, err := FreePort(unbindable); err == nil {+ t.Fatalf(+ "FreePort returned %d while checking an address it cannot bind any port on, so it is not testing the addresses it is given",+ port,+ )+ }++ // And without it, the same call still has to answer, or the guard is a+ // range that cannot come up.+ if _, err := FreePort(); err != nil {+ t.Fatalf("FreePort with no extra addresses failed: %v", err)+ }+}++// And the allocator still has to return something usable, or the fix is a+// range that cannot come up.+func TestFreePortsStillAnswersWithEveryPortDistinct(t *testing.T) {+ const other = "127.0.0.2"++ chosen, err := FreePorts(other)+ if err != nil {+ t.Fatalf("FreePorts: %v", err)+ }++ seen := map[int]bool{}++ for _, port := range []int{+ chosen.Gateway, chosen.Proxied, chosen.Proxy, chosen.Verifier,+ chosen.IdP, chosen.TLS, chosen.Authority, chosen.Dependency,+ chosen.DNS, chosen.DBGateway, chosen.Secret, chosen.Env,+ chosen.Authz, chosen.AuthzDep,+ } {+ if port < portFloor || port >= portCeil {+ t.Fatalf("port %d is outside the high range khaos picks from", port)+ }++ if seen[port] {+ t.Fatalf("port %d was handed out twice", port)+ }++ seen[port] = true++ if !freeOn(port, []string{loopback, other}) {+ t.Fatalf(+ "port %d was handed out while taken on one of the addresses it was checked against",+ port,+ )+ }+ }+}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.goindex 1458d90e..c2a09c0b 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go@@ -123,9 +123,24 @@ const (// a 200 is read as an absence of an answer, and allowed. AuthzRetries// reaches the right verdict the wrong way, asking again every time it does// not like an answer, which is that section's retry storm.+ // AuthzCached is correct too, and it is the input khaos's own verdict+ // machinery was wrong about: a one-second decision cache plus a refusal+ // expressed as 503 rather than 403.AuthzDistinguishes = "distinguishes"AuthzCollapses = "collapses"AuthzRetries = "retries"+ AuthzCached = "cached"++ // EnvReresolve is whether the DNS gateway resolves its dependency's name on+ // every request. Both values are real target shapes: `reresolve` is a+ // client that opens a fresh connection per call, and `pinned` is the+ // DEFAULT behaviour of a Go service -- http.Transport resolves once when it+ // opens a connection and never again while that connection lives, measured+ // at two queries across forty-six requests. The rotation fault needs both.+ EnvReresolve = "KHAOS_RANGE_RERESOLVE"++ DNSReresolve = "reresolve"+ DNSPinned = "pinned"// SecretVariable is the variable the environment faults take away, which// khaos names rather than discovers: the fixture is configured to read its@@ -347,12 +362,15 @@ type Ports struct {}// FreePorts picks every host port the range needs in one go, all distinct.-func FreePorts() (Ports, error) {+// `also` is every further address a tier will bind one of them on, which the+// caller discovers from the daemon: four tiers bind the bridge gateway rather+// than loopback.+func FreePorts(also ...string) (Ports, error) {taken := map[int]bool{}pick := func() (int, error) {for range portTries {- port, err := FreePort()+ port, err := FreePort(also...)if err != nil {return 0, err}@@ -463,27 +481,47 @@ func orNone(declared string) string {return declared}-func FreePort() (int, error) {+// FreePort proves a candidate is free on every address a tier might bind it on,+// and `also` is what makes that true. Testing loopback alone was weaker than it+// read: the published container ports go to 127.0.0.1, but the proxy, the+// identity provider, the certificate authority and the authorization dependency+// all bind the DOCKER BRIDGE GATEWAY, and a port free on 127.0.0.1 is not+// evidence it is free on 172.17.0.1.+//+// A wildcard bind would answer the same question in one call and is not used:+// khaos refuses to bind a wildcard anywhere else, and a probe is not a good+// enough reason to make an exception an operator would find in `ss` output.+func FreePort(also ...string) (int, error) {for range portTries {//nolint:gosec // a bind candidate, immediately testedport := portFloor + rand.IntN(portCeil-portFloor)+ if freeOn(port, append([]string{loopback}, also...)) {+ return port, nil+ }+ }++ return 0, fmt.Errorf(errNoPort, portFloor, portCeil)+}++// freeOn requires EVERY address, because a port taken on any one of them is a+// port a tier can fail to bind.+func freeOn(port int, addresses []string) bool {+ for _, address := range addresses {listener, err := net.Listen("tcp",- net.JoinHostPort(loopback, strconv.Itoa(port)),+ net.JoinHostPort(address, strconv.Itoa(port)),)if err != nil {- continue+ return false}if err := listener.Close(); err != nil {- continue+ return false}-- return port, nil}- return 0, fmt.Errorf(errNoPort, portFloor, portCeil)+ return true}func (l Lab) GatewayURL() string {@@ -509,6 +547,7 @@ func (l Lab) Up(ctx context.Context,out io.Writer,failMode, cacheTTL, tlsMode, poolMode, secretMode, authzMode string,+ reresolve string,withDB bool,) error {environ := append(os.Environ(),@@ -536,6 +575,7 @@ func (l Lab) Up(EnvAuthzPort+"="+strconv.Itoa(l.AuthzPort),EnvAuthzDep+"="+strconv.Itoa(l.AuthzDepPort),EnvAuthzMode+"="+authzMode,+ EnvReresolve+"="+reresolve,)args := []string{"up", "--build", "--detach", "--wait"}diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.ymlindex 3b8e8855..b53e0db0 100644--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml@@ -282,7 +282,7 @@ services:# property of the fault class rather than a fixture defect, and khaos# reports it when it measures it -- but a fixture that can never# experience the fault cannot be what an exit gate asserts against.- KHAOS_RANGE_RERESOLVE: "true"+ KHAOS_RANGE_RERESOLVE: "${KHAOS_RANGE_RERESOLVE:-true}"labels:khaos.target: "true"khaos.range: "true"diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.shindex b1715aba..235e52c8 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/credential.sh@@ -867,15 +867,17 @@ declare -A WANT_AUTHZ=([collapses]="deviated / PERMITTED"[distinguishes]="passed / HELD"[retries]="passed / HELD"+ [cached]="passed / HELD")declare -A WANT_AUTHZ_EXIT=([collapses]=2[distinguishes]=0[retries]=0+ [cached]=0)-for mode in collapses distinguishes retries; do+for mode in collapses distinguishes retries cached; doif ! start_authz_range "$mode"; thencontinuefi@@ -965,6 +967,46 @@ if start_authz_range retries; then"That is the third branch"fi+echo "== the regression control: a correct caller that refuses with 503 =="++# THE INPUT THIS GATE MUST REJECT, and it could not until now. An audit found+# khaos reporting a CORRECT fail-closed caller as `deviated / PERMITTED` with+# "this target's fallback is to allow": the order-sensitive counts that stop+# khaos calling a short decision cache a bypass reset on a REFUSAL, and this+# experiment declares a 503 as its refusal, so a caller that served one request+# from a one-second cache and then correctly refused with 503s read as a bypass.+#+# The three modes above cannot produce it. They answer 403 and hold no cache, so+# the sequence is unreachable against them - which is exactly why the defect+# shipped under 204 green assertions. This mode answers 503 and caches for a+# second, in BOTH fault windows, so it exercises both halves of the fix:+# `AcceptedDespiteLapse` on window one's revoked stream and+# `AcceptedSinceLastRefusal` on window two's valid stream.+#+# VERIFIED by reverting the fix against this fixture on a live range: the same+# run returns `deviated / PERMITTED` exit 2 with the predicate gutted, and+# `passed / HELD` exit 0 with it in place.+if start_authz_range cached; then+ run_experiment credential.apikey_revoked++ assert_verdict "a caller that refuses with 503" "passed / HELD"+ assert_exit "a caller that refuses with 503" 0+ assert_confirmed "a caller that refuses with 503"++ assert_report_lacks \+ "a correct caller is not told its fallback is to allow" \+ "this target's fallback is to allow"+ assert_report_lacks \+ "and is not reported as having served in both phases" \+ "served in BOTH phases"+ assert_report_lacks \+ "and no bypass is claimed against it" \+ "was still accepted"+ assert_report_has \+ "it is credited with the third branch like any correct caller" \+ "That is the third branch"+fi+echo "== the authorization fixture is refused by name when it is not running =="# The refusal has to arrive with a range UP and this one fixture stopped, whichdiff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.shindex b327cf76..080876c6 100755--- a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh@@ -63,10 +63,18 @@ fail() { echo "FAIL $1"; failures=$((failures + 1)); }# The gateway's fail mode is fixed at container creation, deliberately, so a# gate spanning both directions has to restart the range. A gateway that could# be reconfigured mid-run would let khaos change the target it is measuring.+#+# The second argument is the DNS gateway's resolution behaviour, and it is fixed+# at creation for the same reason. `dns.ttl_rotate` needs both shapes: a client+# that re-resolves follows a zone rotation, and one that resolved once and+# reuses its connection never sees it.start_range() {+ local dns_mode="${2:-reresolve}"+"$BIN" range down --no-color >/dev/null 2>&1 || true- if ! "$BIN" range up --fail-mode "$1" --no-color --quiet >/dev/null 2>&1; then- fail "range up --fail-mode $1"+ if ! "$BIN" range up --fail-mode "$1" --dns-mode "$dns_mode" \+ --no-color --quiet >/dev/null 2>&1; then+ fail "range up --fail-mode $1 --dns-mode $dns_mode"return 1fireturn 0@@ -493,6 +501,67 @@ if start_range closed; thenfifi+echo "== the rotation: a zone change that does not reach the target =="++# `dns.ttl_rotate` is the one fault that MOVES the zone rather than answering it+# wrongly, so the honest path serves the new address afterwards. Research 05+# section 6.5 carries the polarity and it is NOT inverted: khaos rotates the+# name to the address `wrong_address` uses, so a target that FOLLOWS the+# rotation starts trusting a host on the strength of a name, which is the same+# defect from the same evidence and stays PERMITTED.+#+# Both directions, because the finding is which one happened:+#+# reresolve the target asks again, follows the rotation, and accepts what the+# new host approves -> deviated / PERMITTED exit 2+# pinned the target resolved once and reuses its connection, never sees+# the rotation -> passed / HELD exit 0, with the operational+# finding STATED and explicitly not called a bypass+#+# The pinned row is the one that needed a target shape the range did not have.+if start_range closed reresolve; then+ run_experiment dns.ttl_rotate++ assert_verdict "ttl_rotate against a re-resolving target" "deviated / PERMITTED"+ assert_exit "ttl_rotate against a re-resolving target" 2+ assert_confirmed "ttl_rotate against a re-resolving target"++ # The confirmation names BOTH addresses and says the answer came from the+ # honest path, which is what distinguishes a rotation from a redirect.+ assert_report_has \+ "the confirmation names the address the rotation replaced" \+ "before khaos moved it"+ assert_report_has \+ "and says the honest path serves it" \+ "from the resolver's own honest path rather than from a fault"+ assert_report_has \+ "the rotation is reported as having reached the target" \+ "so the rotation reached it"+fi++if start_range closed pinned; then+ run_experiment dns.ttl_rotate++ assert_verdict "ttl_rotate against a pinning target" "passed / HELD"+ assert_exit "ttl_rotate against a pinning target" 0+ assert_confirmed "ttl_rotate against a pinning target"++ # The finding has to be STATED, with both counts, and must NOT be called a+ # bypass: no credential got through that would not have before.+ assert_report_has \+ "the rotation is reported as not having reached the target" \+ "ZERO during the fault window"+ assert_report_has \+ "and the operational consequence is named" \+ "would not have taken effect either"+ assert_report_has \+ "and khaos explicitly reports no bypass" \+ "This is NOT a credential bypass"+ assert_report_lacks \+ "no credential is claimed to have been accepted" \+ "was accepted"+fi+echo "== the census: every catalogued fault was actually run =="# A catalogued experiment this gate never runs is a failure, for the same reason