Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(khaos): M8, the dns. family on a resolver that had to be a container

CarterPerez-dev007760030a39
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go
index 8acac666..c4778208 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway.go
@@ -70,17 +70,45 @@ type cachedDecision struct {
expires time.Time
}
-func newGateway(upstream, failMode string, cacheTTL time.Duration) *gateway {
+// newGateway builds the fixture. reresolve is what the DNS tier needs and is
+// worth the extra parameter, because without it that whole family is inert
+// against this gateway.
+//
+// Go's `http.Transport` keeps connections alive, so a client resolves its
+// upstream's name ONCE when it opens a connection and then never again for the
+// life of that connection. Measured on this fixture: 46 probes produced TWO
+// queries to khaos's resolver, both at the first request. A DNS fault armed
+// after that changes nothing the gateway can see, and khaos would report a clean
+// survival of a fault the target never experienced.
+//
+// That is a true property of the fault class rather than a fixture defect — it
+// is the DNS analogue of session resumption defeating a certificate probe — and
+// khaos reports it as such when it measures it. But a fixture that can NEVER
+// experience the fault cannot be the thing an exit gate asserts against, so this
+// gateway re-resolves per request. Research 05 section 11 ranks the
+// application's own connection pool as the caching layer most likely to void a
+// DNS experiment, which is exactly what it did here.
+func newGateway(
+ upstream, failMode string,
+ cacheTTL time.Duration,
+ reresolve bool,
+) *gateway {
if cacheTTL <= 0 {
cacheTTL = defaultCacheTTL
}
+ client := &http.Client{Timeout: upstreamTimeout}
+
+ if reresolve {
+ client.Transport = &http.Transport{DisableKeepAlives: true}
+ }
+
return &gateway{
upstream: upstream,
failMode: failMode,
cacheTTL: cacheTTL,
cache: map[string]cachedDecision{},
- client: &http.Client{Timeout: upstreamTimeout},
+ client: client,
}
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go
index 51029068..4a86dff1 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/gateway_test.go
@@ -91,7 +91,7 @@ func ask(t *testing.T, g *gateway, authorization string) int {
func TestFailClosedServesFromCacheThenRefuses(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailClosed, shortTTL)
+ g := newGateway(idp.server.URL, FailClosed, shortTTL, false)
if got := ask(t, g, bearerPrefix+goodToken); got != http.StatusOK {
t.Fatalf("baseline returned %d, want 200", got)
@@ -121,7 +121,7 @@ func TestFailClosedServesFromCacheThenRefuses(t *testing.T) {
func TestFailOpenHonoursACredentialItHadRefused(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailOpen, shortTTL)
+ g := newGateway(idp.server.URL, FailOpen, shortTTL, false)
if got := ask(
t,
@@ -152,7 +152,7 @@ func TestARefusalFromTheProviderIsAnAnswerNotAnOutage(t *testing.T) {
idp := newStubIDP(t)
idp.status.Store(int32(status))
- g := newGateway(idp.server.URL, FailOpen, shortTTL)
+ g := newGateway(idp.server.URL, FailOpen, shortTTL, false)
if got := ask(
t,
@@ -174,7 +174,7 @@ func TestARefusalFromTheProviderIsAnAnswerNotAnOutage(t *testing.T) {
func TestTheBearerSchemeIsRequired(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailClosed, shortTTL)
+ g := newGateway(idp.server.URL, FailClosed, shortTTL, false)
cases := map[string]int{
bearerPrefix + goodToken: http.StatusOK,
@@ -200,7 +200,7 @@ func TestTheBearerSchemeIsRequired(t *testing.T) {
func TestAFreshCredentialPerRequestDefeatsTheDecisionCache(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailOpen, time.Hour)
+ g := newGateway(idp.server.URL, FailOpen, time.Hour, false)
const probes = 3
@@ -293,7 +293,7 @@ func active(t *testing.T, base, token string) bool {
// the Docker version of this question cost two wrong predictions in a row.
func TestStaleRearmSurvivesSteadyTrafficAndLapsesOnAPause(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailStaleRearm, shortTTL)
+ g := newGateway(idp.server.URL, FailStaleRearm, shortTTL, false)
// The provider is up, so this caches a genuine allow.
if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
@@ -340,7 +340,7 @@ func TestStaleRearmSurvivesSteadyTrafficAndLapsesOnAPause(t *testing.T) {
// tell it apart from.
func TestWithoutACeilingStaleRearmWouldBeANoExpiryCache(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailStaleRearm, shortTTL)
+ g := newGateway(idp.server.URL, FailStaleRearm, shortTTL, false)
if ceiling := g.staleCeiling(); ceiling <= shortTTL {
t.Fatalf(
@@ -375,7 +375,7 @@ func TestWithoutACeilingStaleRearmWouldBeANoExpiryCache(t *testing.T) {
// copy.
func TestFailClosedNeverServesAStaleDecision(t *testing.T) {
idp := newStubIDP(t)
- g := newGateway(idp.server.URL, FailClosed, shortTTL)
+ g := newGateway(idp.server.URL, FailClosed, shortTTL, false)
if status := ask(t, g, bearerPrefix+goodToken); status != http.StatusOK {
t.Fatalf("baseline status %d, want 200", status)
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go
index 8ca0a530..e55c8a66 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/idp.go
@@ -28,6 +28,13 @@ const (
type introspection struct {
Active bool `json:"active"`
User string `json:"user,omitempty"`
+
+ // Asked is how many times the ROGUE provider has been introspected, and it
+ // is only ever set by that fixture. It is the measurement that separates a
+ // DNS redirect that landed from a dependency outage: a gateway that failed
+ // open because it could not resolve anything never asked anybody, and one
+ // that was redirected asked the wrong service.
+ Asked int64 `json:"asked,omitempty"`
}
type identityProvider struct {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
index 080c6915..ac6757ea 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/main.go
@@ -23,12 +23,20 @@ const (
roleGateway = "gateway"
roleVerifier = "verifier"
roleTLS = "tls"
+ roleRogue = "rogue"
+ roleResolver = "resolver"
roleHealth = "healthcheck"
envListen = "KHAOS_RANGE_LISTEN"
envUpstream = "KHAOS_RANGE_UPSTREAM"
- envFailMode = "KHAOS_RANGE_FAIL_MODE"
- envCacheTTL = "KHAOS_RANGE_CACHE_TTL"
+
+ // envReresolve makes this gateway resolve its dependency's name on EVERY
+ // request instead of once per kept-alive connection. Only the DNS gateway
+ // sets it; the other tiers are measured on the ordinary pooled path.
+ envReresolve = "KHAOS_RANGE_RERESOLVE"
+ trueValue = "true"
+ envFailMode = "KHAOS_RANGE_FAIL_MODE"
+ envCacheTTL = "KHAOS_RANGE_CACHE_TTL"
defaultListen = ":8080"
defaultUpstream = "http://idp:8080"
@@ -50,7 +58,7 @@ const (
// header timeout exists only to stop a wedged connection living forever.
readHeaderTimeout = 120 * time.Second
- usage = "usage: khaos-range <idp|gateway|verifier|tls|healthcheck>"
+ usage = "usage: khaos-range <idp|gateway|verifier|tls|rogue|resolver|healthcheck>"
selfURL = "http://127.0.0.1:8080/health"
healthWait = 2 * time.Second
errorFormat = "khaos-range: %v\n"
@@ -73,6 +81,13 @@ func run() error {
return selfCheck()
}
+ // The resolver is the one role that is not an HTTP service: it binds UDP and
+ // TCP on 53 and its own control surface, so it never reaches the server
+ // below.
+ if os.Args[1] == roleResolver {
+ return serveResolver(os.Args[2:])
+ }
+
handler, err := roleHandler(os.Args[1])
if err != nil {
return err
@@ -118,6 +133,7 @@ func roleHandler(role string) (http.Handler, error) {
envOr(envUpstream, defaultUpstream),
envOr(envFailMode, FailClosed),
durationOr(envCacheTTL, defaultCacheTTL),
+ envOr(envReresolve, "") == trueValue,
).routes(), nil
case roleVerifier:
return newVerifierGateway(
@@ -126,6 +142,8 @@ func roleHandler(role string) (http.Handler, error) {
durationOr(envKeyTTL, verifier.DefaultKeyTTL),
durationOr(envCacheTTL, verifier.DefaultDecisionTTL),
).routes(), nil
+ case roleRogue:
+ return newRogueProvider().routes(), nil
case roleTLS:
return newTLSGateway(
envOr(envUpstream, defaultUpstream),
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resolver.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resolver.go
new file mode 100644
index 00000000..691901fe
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/resolver.go
@@ -0,0 +1,131 @@
+// ©AngelaMos | 2026
+// resolver.go
+
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+)
+
+const (
+ flagControl = "control"
+ flagZone = "zone"
+
+ usageControl = "the zone name no fault ever touches"
+ usageZone = "a name this resolver answers authoritatively, as name=address; repeatable"
+
+ errZoneEntry = "%w: %q (expected name=address)"
+ errNoZone = "%w: pass --zone name=address at least once"
+)
+
+var errZoneFormat = errors.New("a --zone entry must be name=address")
+
+var errZoneMissing = errors.New(
+ "a chaos resolver needs at least one zone entry",
+)
+
+// serveResolver runs khaos's chaos resolver as a range service.
+//
+// It is the SAME `internal/dnsrv` package khaos runs outside the range, behind a
+// different entry point, and that matters for what the range proves. If the
+// range ran a separate implementation, every DNS experiment the gate asserts
+// would be asserting something an operator's khaos never executes.
+//
+// Outside the range khaos imports its own binary as a one-layer image, because
+// an operator's khaos has no source tree to build from. In here the image
+// already exists, so the resolver is an ordinary compose service with the labels
+// khaos matches on, and khaos finds it exactly as it finds one it started
+// itself.
+//
+// It does not go through roleHandler, because it is the one role that is not an
+// HTTP service: it binds UDP and TCP on 53 and its control surface separately.
+func serveResolver(args []string) error {
+ flags := flag.NewFlagSet(roleResolver, flag.ContinueOnError)
+
+ control := flags.String(flagControl, "", usageControl)
+
+ var entries zoneEntries
+ flags.Var(&entries, flagZone, usageZone)
+
+ if err := flags.Parse(args); err != nil {
+ return err
+ }
+
+ zone, err := entries.parse()
+ if err != nil {
+ return err
+ }
+
+ // From the ENVIRONMENT, for the measured reason in resolverArgs: a command
+ // line is world-readable in the host's process table and this is a secret.
+ resolver, err := dnsrv.New(dnsrv.Config{
+ Nonce: os.Getenv(dockerx.EnvResolverNonce),
+ Control: *control,
+ Zone: zone,
+ })
+ if err != nil {
+ return err
+ }
+
+ if err := resolver.Serve(); err != nil {
+ return err
+ }
+
+ if err := resolver.ServeControl(""); err != nil {
+ _ = resolver.Shutdown()
+ return err
+ }
+
+ ctx, stop := signal.NotifyContext(
+ context.Background(),
+ os.Interrupt,
+ syscall.SIGTERM,
+ )
+ defer stop()
+
+ <-ctx.Done()
+
+ return resolver.Shutdown()
+}
+
+// zoneEntries collects a repeated flag. It is a flag.Value rather than a
+// StringArray because this binary has no cobra, and the repetition matters: a
+// resolver with one zone entry has no control name to confirm against.
+type zoneEntries []string
+
+func (z *zoneEntries) String() string {
+ return strings.Join(*z, ",")
+}
+
+func (z *zoneEntries) Set(value string) error {
+ *z = append(*z, value)
+ return nil
+}
+
+func (z *zoneEntries) parse() (map[string]string, error) {
+ if len(*z) == 0 {
+ return nil, fmt.Errorf(errNoZone, errZoneMissing)
+ }
+
+ zone := make(map[string]string, len(*z))
+
+ for _, entry := range *z {
+ name, address, found := strings.Cut(entry, "=")
+ if !found || name == "" || address == "" {
+ return nil, fmt.Errorf(errZoneEntry, errZoneFormat, entry)
+ }
+ zone[name] = address
+ }
+
+ return zone, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/rogue.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/rogue.go
new file mode 100644
index 00000000..40705483
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/rogue.go
@@ -0,0 +1,77 @@
+// ©AngelaMos | 2026
+// rogue.go
+
+package main
+
+import (
+ "net/http"
+ "sync/atomic"
+)
+
+const (
+ routeRogueCount = "/khaos-rogue-count"
+
+ rogueUser = "whoever-asked"
+)
+
+// rogueProvider is the service a DNS redirect fault points the target at, and
+// it is what turns `dns.wrong_address` from an outage into a fail-open
+// demonstration.
+//
+// Pointing a redirect at an unroutable address only tests what an unreachable
+// dependency tests, which khaos already has three tiers for. Research 05
+// section 10 names the real finding: a hostname allowlist does not merely stop
+// working when the name resolves somewhere else, it ACTIVELY AUTHORISES the new
+// host. To observe that, the new host has to answer, and answer wrongly.
+//
+// So this approves everything. Every token is active, including the invalid
+// control khaos has never issued, which is exactly the shape of an attacker who
+// has won a name.
+//
+// It is NOT a khaos injector and never will be. It is a range fixture, in the
+// range image, started by the range's compose file, and it carries the consent
+// label like every other target in there.
+type rogueProvider struct {
+ asked atomic.Int64
+}
+
+func newRogueProvider() *rogueProvider {
+ return &rogueProvider{}
+}
+
+func (p *rogueProvider) routes() http.Handler {
+ mux := http.NewServeMux()
+
+ mux.HandleFunc(routeIntrospect, p.introspect)
+ mux.HandleFunc(routeRogueCount, p.count)
+ mux.HandleFunc(routeHealth, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+
+ return mux
+}
+
+// introspect approves anything. The token is not even read: a service that has
+// taken over a name it was never entitled to has no reason to check, and
+// pretending otherwise would make the fixture test a different bug.
+func (p *rogueProvider) introspect(w http.ResponseWriter, _ *http.Request) {
+ p.asked.Add(1)
+
+ writeJSON(w, introspection{Active: true, User: rogueUser})
+}
+
+// count is how the gate proves the redirect LANDED rather than inferring it
+// from the verdict.
+//
+// Without it a run that found a bypass is ambiguous: the gateway could have
+// failed open because it could not resolve its dependency at all, which is a
+// different fault's finding. A non-zero count here says the gateway resolved
+// the dependency's name to this service and asked it, which is the only
+// observation that distinguishes a redirect from an outage.
+func (p *rogueProvider) count(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, introspection{
+ Active: true,
+ User: rogueUser,
+ Asked: p.asked.Load(),
+ })
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/tlsgateway.go b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/tlsgateway.go
index 9cecb097..bd491967 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/tlsgateway.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/cmd/khaos-range/tlsgateway.go
@@ -69,7 +69,7 @@ func newTLSGateway(
cacheTTL time.Duration,
) *tlsGateway {
return &tlsGateway{
- auth: newGateway(upstream, failMode, cacheTTL),
+ auth: newGateway(upstream, failMode, cacheTTL, false),
mode: resolveTLSMode(tlsMode),
// Normalised to host:port, because Decide DIALS this and a URL is not
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/go.mod b/PROJECTS/intermediate/chaos-engineering-tool/go.mod
index cf346997..bc26108e 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/go.mod
+++ b/PROJECTS/intermediate/chaos-engineering-tool/go.mod
@@ -6,6 +6,7 @@ require (
charm.land/lipgloss/v2 v2.0.6
github.com/charmbracelet/colorprofile v0.4.3
github.com/containerd/errdefs v1.0.0
+ github.com/miekg/dns v1.1.73
github.com/moby/moby/api v1.56.1
github.com/moby/moby/client v0.6.1
github.com/reeflective/console v0.5.0
@@ -50,6 +51,7 @@ require (
go.opentelemetry.io/otel v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect
go.opentelemetry.io/otel/trace v1.35.0 // indirect
+ golang.org/x/net v0.58.0 // indirect
golang.org/x/sync v0.22.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
mvdan.cc/sh/v3 v3.13.1 // indirect
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/go.sum b/PROJECTS/intermediate/chaos-engineering-tool/go.sum
index 04f4c5d3..a49d2a67 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/go.sum
+++ b/PROJECTS/intermediate/chaos-engineering-tool/go.sum
@@ -64,6 +64,8 @@ github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBql
github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
+github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c=
@@ -115,6 +117,8 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20260529124908-c761662dc8c9 h1:4d4PbuBNwaxMXkXI8yiIYjydtMU+04RHeuSxJdgKftM=
golang.org/x/exp v0.0.0-20260529124908-c761662dc8c9/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
index 1b422b1c..bde74268 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment.go
@@ -16,6 +16,7 @@ import (
"strings"
"time"
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
"github.com/CarterPerez-dev/khaos/internal/dockerx"
"github.com/CarterPerez-dev/khaos/internal/experiment"
"github.com/CarterPerez-dev/khaos/internal/inject"
@@ -112,8 +113,15 @@ enforcing in the first place.`
errAuthorityBind = "khaos could not bind its certificate authority on %s, which is where the range's TLS gateway is already looking for it: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"
msgAuthorityUp = "khaos's certificate authority is serving the dependency on %s and revocation on %s, minting for %s"
- errProxyBind = "khaos could not bind its fault proxy on %s, which is where the range's proxied gateway is already looking for it: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"
- msgProxyUp = "khaos's fault proxy is listening on %s, forwarding to %s"
+
+ msgResolverUp = "khaos's chaos resolver %s is answering on %s, authoritative for %s, with %s as the control no fault touches"
+
+ errNoResolverOn = "no khaos chaos resolver is serving %s. The DNS family needs one, and the target has to have been CREATED pointing at it: `--dns` is fixed at container creation and Docker offers no way to change it afterwards. Run `khaos resolver up --network %[1]s` and recreate the target against the address it prints, or `khaos range up` for a stack that is already wired: %w"
+ errResolverSilent = "khaos's chaos resolver %s did not answer its control surface at %s: %w"
+
+ errTargetOffNetwork = "%s is not attached to %s, so it cannot be using the chaos resolver that serves it and khaos could not count its queries: %w"
+ errProxyBind = "khaos could not bind its fault proxy on %s, which is where the range's proxied gateway is already looking for it: %w. Something else took that port since the range came up; bring the range down and up again to pick a new one"
+ msgProxyUp = "khaos's fault proxy is listening on %s, forwarding to %s"
groupExperiments = "experiments"
titleExperiments = "Experiments:"
@@ -515,7 +523,7 @@ func runExperiment(
return err
}
- env, closeTiers, err := standUpTiers(ctx, out, work, spec)
+ env, closeTiers, err := standUpTiers(ctx, out, work, &spec, accepted)
if err != nil {
return err
}
@@ -787,6 +795,9 @@ func rangeSupplies() []string {
experiment.ParamAuthorityTLS,
experiment.ParamAuthorityBase,
experiment.ParamCertHost,
+ experiment.ParamDNSHost,
+ experiment.ParamResolverNetwork,
+ experiment.ParamDNSAddress,
}
}
@@ -803,6 +814,7 @@ const (
tierProxy
tierIdP
tierTLS
+ tierDNS
)
func tierOf(kind string) tier {
@@ -813,6 +825,8 @@ func tierOf(kind string) tier {
return tierIdP
case inject.NeedsPKI(kind):
return tierTLS
+ case inject.NeedsResolver(kind):
+ return tierDNS
default:
return tierDirect
}
@@ -889,6 +903,15 @@ func rangeDefaults(
url = lab.TLSProtectedURL()
target = rangelab.TLSGatewayName
+ case tierDNS:
+ // The DNS gateway, and the target is the GATEWAY rather than the
+ // dependency for the same reason again: the fault is applied to khaos's
+ // resolver, and the gateway is both the subject and the container whose
+ // consent khaos needs. The dependency is never touched at all here --
+ // it stays healthy and reachable, and the target simply cannot find it.
+ url = lab.DNSProtectedURL()
+ target = rangelab.DNSGatewayName
+
default:
// The direct tier probes the protected URL and names the provider
// container, both of which are already set above.
@@ -909,6 +932,18 @@ func rangeDefaults(
experiment.ParamAuthorityTLS: answers.AuthorityTLS,
experiment.ParamAuthorityBase: answers.AuthorityBase,
experiment.ParamCertHost: answers.CertHost,
+
+ experiment.ParamDNSHost: rangelab.DNSDependencyName,
+ experiment.ParamResolverNetwork: rangelab.NetworkName,
+
+ // The redirect faults point at the range's ROGUE provider rather than at
+ // the documentation address they default to, and that is what makes them
+ // findings instead of outages. The rogue approves every credential, so a
+ // gateway that follows the redirect without verifying who it reached
+ // serves the invalid control and khaos reports PERMITTED. Pointed at an
+ // unroutable address the same fault only reproduces what three other
+ // tiers already do to an unreachable dependency.
+ experiment.ParamDNSAddress: rangelab.RogueAddress,
}
}
@@ -971,19 +1006,32 @@ func fillFromRange(
)
}
- answers, err := askTheRange(ctx, reader, lab, missing)
+ answers, err := askTheRange(
+ ctx,
+ reader,
+ lab,
+ fillable(definition, supplied),
+ )
if err != nil {
return nil, err
}
defaults := rangeDefaults(lab, answers, at)
filled := maps.Clone(supplied)
- resolved := make([]string, 0, len(missing))
+
+ // Filled from the wider set, so a range answer beats a generic declaration
+ // default. See fillable.
+ answered := fillable(definition, supplied)
+ resolved := make([]string, 0, len(answered))
// Credentials are deliberately omitted from the note. Everything else khaos
// chose on the operator's behalf is named, because this line is the only thing
// telling them which stack they are about to break.
- for _, name := range missing {
+ for _, name := range answered {
+ if defaults[name] == "" {
+ continue
+ }
+
filled[name] = defaults[name]
if !credentialParam(name) {
resolved = append(
@@ -1035,6 +1083,47 @@ func partitionFillable(
return given, missing
}
+// fillable is what the range should ANSWER, which is a wider set than what is
+// missing, and separating the two is a correctness fix rather than tidying.
+//
+// `missing` drives the half-specified refusal, so a parameter the declaration
+// already defaults is deliberately not in it: demanding a value khaos supplies
+// itself sends the operator looking for nothing. But a default is a GENERIC
+// answer, and the range knows its own topology, so where both exist the range's
+// answer has to win.
+//
+// `--dns-address` is the first parameter where that mattered and it mattered a
+// lot. Its default is a documentation address with nothing on it, which makes a
+// redirect fault indistinguishable from the dependency outage three other tiers
+// already test. The range answers it with its rogue provider, which approves
+// every credential, and that is what turns the fault into the finding research 05
+// section 10 names: a hostname allowlist does not merely stop working when the
+// name moves, it authorises the new host. With the default winning, the gate's
+// own assertion caught it: the report named the documentation address.
+//
+// Precedence is operator, then range, then declaration.
+func fillable(
+ definition experiment.Definition,
+ supplied map[string]string,
+) []string {
+ answerable := rangeSupplies()
+ out := []string{}
+
+ for _, param := range definition.Params {
+ if !slices.Contains(answerable, param.Name) {
+ continue
+ }
+
+ if _, present := supplied[param.Name]; present {
+ continue
+ }
+
+ out = append(out, param.Name)
+ }
+
+ return out
+}
+
func flagList(names []string) string {
out := make([]string, 0, len(names))
for _, name := range names {
@@ -1145,7 +1234,8 @@ func standUpTiers(
ctx context.Context,
out io.Writer,
work *workspace.Workspace,
- spec experiment.Experiment,
+ spec *experiment.Experiment,
+ target dockerx.Target,
) (inject.Env, func(), error) {
env := work.Env()
@@ -1158,7 +1248,7 @@ func standUpTiers(
}
if spec.ProxyFault != nil {
- fault, err := startProxy(ctx, out, work.Docker, spec)
+ fault, err := startProxy(ctx, out, work.Docker, *spec)
if err != nil {
return env, closeAll, err
}
@@ -1173,7 +1263,7 @@ func standUpTiers(
// it and minting needs no listener. All that is left is to bind it where the
// target is already looking.
if spec.IdPTrap != nil {
- if err := startProvider(ctx, out, work.Docker, spec); err != nil {
+ if err := startProvider(ctx, out, work.Docker, *spec); err != nil {
return env, closeAll, err
}
@@ -1186,7 +1276,7 @@ func standUpTiers(
// The authority plays the dependency the target dials, so without it the
// baseline measures a dependency that is simply absent.
if spec.PKITrap != nil {
- if err := startAuthority(out, spec); err != nil {
+ if err := startAuthority(out, *spec); err != nil {
return env, closeAll, err
}
@@ -1196,9 +1286,164 @@ func standUpTiers(
env.PKITrap = *spec.PKITrap
}
+ // The resolver is ATTACHED, not started, and that is the one structural
+ // difference between this tier and the other three in-process ones.
+ //
+ // khaos cannot stand a resolver up at run time and have the target use it:
+ // `--dns` is fixed at container creation and Docker offers no API to change
+ // it on a running container, measured, so a resolver created now would be
+ // one nothing is pointed at. The resolver has to pre-exist the target, and
+ // khaos finds the one already serving the target's network.
+ if spec.DNSTrap != nil {
+ client, err := attachResolver(ctx, out, work.Docker, *spec)
+ if err != nil {
+ return env, closeAll, err
+ }
+
+ // The liveness guard is wired HERE and not by the builder, and the
+ // reason is structural: it needs the resolver client, which does not
+ // exist until the run finds one, and the target's container address,
+ // which needs the daemon. The identity tier can set its own counter at
+ // build time because it BUILDS its provider; this tier attaches to one.
+ //
+ // The address is what makes the count a proof of arrival rather than a
+ // number that is always non-zero. khaos's own confirmation queries come
+ // from the host; the target's are forwarded by its embedded resolver out
+ // of its own network namespace, so counting by source separates them.
+ if err := wireArrivalGuard(
+ ctx,
+ work.Docker,
+ spec,
+ client,
+ target,
+ ); err != nil {
+ return env, closeAll, err
+ }
+
+ // Disarmed on the way out, so the recovery window is measured through a
+ // WORKING resolver. Unlike the in-process tiers this authority outlives
+ // the khaos process, so leaving it armed would leave the target unable
+ // to resolve its dependency after khaos exited.
+ closers = append(closers, func() {
+ _, _ = client.Disarm(context.WithoutCancel(ctx))
+ })
+
+ env.Resolver = client
+ env.ResolverNetwork = spec.ResolverNetwork
+ env.DNSTrap = *spec.DNSTrap
+ }
+
return env, closeAll, nil
}
+// attachResolver finds the chaos resolver serving the target's network and
+// returns a client for it, with the fault disarmed and the confirmation
+// control read back out of the resolver itself.
+//
+// Everything here is read back rather than remembered, which is the discipline
+// the identity provider's most expensive bug established. The address comes
+// from the daemon, because Docker refuses a static one on a network whose
+// subnet it chose. The nonce comes off the container's label. And the control
+// name comes from the resolver's own health endpoint, because the resolver is
+// the only authority on which name it will refuse to fault.
+// wireArrivalGuard installs the count of the TARGET's own queries for the name
+// under fault, which is the proof-of-arrival control research 05 section 12
+// makes mandatory.
+//
+// Without it khaos cannot distinguish "the control failed closed" from "the
+// fault never reached the target", and the second is the likely one: `--dns` is
+// fixed at container creation and Docker offers no way to change it afterwards,
+// so a target that was not CREATED pointing at khaos's resolver never sends it a
+// single query and every verdict about it would be about nothing.
+func wireArrivalGuard(
+ ctx context.Context,
+ docker *dockerx.Client,
+ spec *experiment.Experiment,
+ client *dnsrv.Client,
+ target dockerx.Target,
+) error {
+ address, err := docker.NetworkAddress(ctx, target.ID, spec.ResolverNetwork)
+ if err != nil {
+ return fmt.Errorf(
+ errTargetOffNetwork,
+ target.Name,
+ spec.ResolverNetwork,
+ err,
+ )
+ }
+
+ host := spec.DNSHost
+
+ spec.Consulted = func() (int64, error) {
+ return client.Queries(context.WithoutCancel(ctx), address, host)
+ }
+
+ return nil
+}
+
+// requireDNSTier refuses a DNS experiment against a range with no DNS gateway,
+// BY NAME, before anything is probed.
+//
+// A missing gateway otherwise surfaces as a published port of zero, so the run
+// blames the target for not enforcing when khaos was pointing at nowhere.
+func requireDNSTier(lab rangelab.Lab, want func(string) bool) error {
+ if !want(experiment.ParamDNSHost) &&
+ !want(experiment.ParamResolverNetwork) {
+ return nil
+ }
+
+ return lab.RequireDNSTier()
+}
+
+func attachResolver(
+ ctx context.Context,
+ out io.Writer,
+ docker *dockerx.Client,
+ spec experiment.Experiment,
+) (*dnsrv.Client, error) {
+ resolver, err := docker.FindResolver(ctx, spec.ResolverNetwork)
+ if err != nil {
+ return nil, fmt.Errorf(errNoResolverOn, spec.ResolverNetwork, err)
+ }
+
+ control := dockerx.ResolverControlURL(resolver.Address)
+
+ client := dnsrv.NewClient(resolver.Address, control, resolver.Nonce, "")
+
+ health, err := client.Health(ctx)
+ if err != nil {
+ return nil, fmt.Errorf(errResolverSilent, resolver.Name, control, err)
+ }
+
+ client.ControlName = health.Control
+
+ // The zone is read back too, so a redirect's observable can name the honest
+ // address the fault replaced rather than restating the one it served.
+ state, err := client.State(ctx)
+ if err != nil {
+ return nil, fmt.Errorf(errResolverSilent, resolver.Name, control, err)
+ }
+
+ client.Zone = state.Zone
+
+ // Disarmed before the baseline, because the baseline is measured through
+ // this resolver and one still carrying a previous run's fault would make the
+ // two phases see different topologies.
+ if _, err := client.Disarm(ctx); err != nil {
+ return nil, err
+ }
+
+ note(out, fmt.Sprintf(
+ msgResolverUp,
+ resolver.Name,
+ resolver.Address,
+ spec.DNSHost,
+ health.Control,
+ ))
+
+ return client, nil
+}
+
// startAuthority stands khaos's certificate authority up on BOTH of its
// listeners, before the baseline.
//
@@ -1306,6 +1551,10 @@ func askTheRange(
answers.ProxyUpstream = upstream
}
+ if err := requireDNSTier(lab, want); err != nil {
+ return rangeAnswers{}, err
+ }
+
if want(experiment.ParamAuthority) {
authority, err := lab.AuthorityURL(ctx, reader)
if err != nil {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
index 98293f36..51771890 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/experiment_test.go
@@ -141,7 +141,7 @@ func TestRangeSuppliesIsExactlyTheKeySetOfRangeDefaults(t *testing.T) {
// allTiers is every tier the range serves. A test that iterated only the two it
// knew about would go on passing while a whole gateway's answers drifted.
func allTiers() []tier {
- return []tier{tierDirect, tierProxy, tierIdP, tierTLS}
+ return []tier{tierDirect, tierProxy, tierIdP, tierTLS, tierDNS}
}
func tierName(at tier) string {
@@ -152,6 +152,8 @@ func tierName(at tier) string {
return "against khaos's own provider"
case tierTLS:
return "against khaos's own certificate authority"
+ case tierDNS:
+ return "through khaos's own chaos resolver"
default:
return "direct"
}
@@ -172,6 +174,8 @@ func fullLab() rangelab.Lab {
Dependency: "https://host.docker.internal:30007",
AuthorityBase: "http://host.docker.internal:30006",
CertHost: "dependency.khaos.invalid",
+
+ DNSPort: 30009,
}
}
@@ -692,11 +696,16 @@ func TestTheRangeCanSatisfyEveryExperimentsRequiredParameters(t *testing.T) {
}
}
- if _, err := experiment.Build(
+ // Every catalogue entry BUILDS, including the two DNS faults khaos
+ // refuses to deliver. They are catalogued in order to be explained,
+ // and the refusal arrives when khaos is asked to arm one, which is
+ // what keeps this census able to walk the whole catalogue.
+ _, err := experiment.Build(
definition.Name,
supplied,
experiment.Timing{},
- ); err != nil {
+ )
+ if err != nil {
t.Fatalf(
"the range cannot satisfy %s, so it cannot run with zero flags: %v",
definition.Name,
@@ -707,7 +716,7 @@ func TestTheRangeCanSatisfyEveryExperimentsRequiredParameters(t *testing.T) {
}
}
-// The range carries FOUR gateways and they are not interchangeable. Each one
+// The range carries FIVE gateways and they are not interchangeable. Each one
// exists so one injection tier is measured on a path nothing else contaminates:
// a proxy-tier experiment arms a fault on the path only the proxied gateway
// uses, an identity-tier one arms a provider only the verifier gateway resolves
@@ -725,6 +734,7 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
tierProxy: lab.ProxiedProtectedURL(),
tierIdP: lab.VerifierProtectedURL(),
tierTLS: lab.TLSProtectedURL(),
+ tierDNS: lab.DNSProtectedURL(),
}
seen := map[string]tier{}
@@ -753,12 +763,17 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
definition.Name, definition.FaultKind, answered, want[at])
}
- // The two in-process tiers name their GATEWAY rather than the
+ // The three per-run tiers name their GATEWAY rather than the
// dependency, because that is the container khaos needs consent for
// and the one it actually measures. The fault lands on khaos's own
- // provider or authority, which is not a container at all, so naming
- // the dependency would ask consent for something this experiment
- // never touches and leave the measured service unchecked.
+ // provider, authority or resolver, which is not the dependency at
+ // all, so naming the dependency would ask consent for something
+ // this experiment never touches and leave the measured service
+ // unchecked.
+ //
+ // The DNS tier is the clearest case of it: the dependency stays
+ // healthy and reachable throughout, and what changes is only whether
+ // the gateway can find it.
wantTarget := rangelab.IDPName
switch at {
@@ -766,6 +781,8 @@ func TestEveryExperimentIsPointedAtTheGatewayItsFaultActuallyAffects(
wantTarget = rangelab.VerifierGatewayName
case tierTLS:
wantTarget = rangelab.TLSGatewayName
+ case tierDNS:
+ wantTarget = rangelab.DNSGatewayName
default:
// The direct and proxy tiers fault the dependency itself.
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
index 316f5664..c53c2203 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/rangecmd.go
@@ -71,6 +71,7 @@ running.`
msgTierProxied = "proxy tier %s, with khaos's fault proxy expected on host port %d"
msgTierVerifier = "identity tier %s, trusting khaos's provider as %s"
msgTierTLS = "certificate tier %s, dialling khaos's authority at %s as the %s client"
+ msgTierDNS = "dns tier %s, resolving %s through khaos's chaos resolver at %s"
)
func newRangeCmd() *cobra.Command {
@@ -261,6 +262,16 @@ func runRangeUp(
return err
}
+ // A fresh nonce per `range up`, for the same reason khaos mints a fresh
+ // key set per run: a nonce that outlived the range would let anything
+ // that had seen a previous one arm the resolver, and the resolver shares
+ // a network with every target in there.
+ nonce, err := newNonce()
+ if err != nil {
+ return err
+ }
+ lab.ResolverNonce = nonce
+
err = lab.Up(ctx, progress, failMode, cacheTTL, tlsMode)
if err == nil {
break
@@ -384,5 +395,17 @@ func describeEntryPoints(
)
}
+ if lab.DNSPort != 0 {
+ say(
+ out,
+ fmt.Sprintf(
+ msgTierDNS,
+ lab.DNSProtectedURL(),
+ rangelab.DNSDependencyName,
+ rangelab.ResolverAddress,
+ ),
+ )
+ }
+
fmt.Fprintln(out)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/resolvercmd.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/resolvercmd.go
new file mode 100644
index 00000000..2683c8a0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/resolvercmd.go
@@ -0,0 +1,427 @@
+// ©AngelaMos | 2026
+// resolvercmd.go
+
+package cli
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/spf13/cobra"
+)
+
+const (
+ FlagZone = "zone"
+ FlagControl = "control"
+ FlagNetwork = "network"
+
+ serveResolverShort = "serve khaos's chaos DNS resolver (run inside khaos's own container)"
+ serveResolverLong = `serve-resolver runs khaos's authoritative chaos DNS resolver.
+
+It is not meant to be run by hand. khaos imports its own binary as a one-layer
+container image and starts this subcommand inside it, because an unprivileged
+process cannot bind port 53 and Docker's --dns carries an address with no port.
+
+The resolver listens on 53 inside its own network namespace, publishes nothing
+to the host, and is armed over a nonce-authorised control surface on the
+container's own address.`
+
+ resolverShort = "manage khaos's chaos DNS resolver on a network"
+ resolverUpShort = "start a chaos resolver on a network and print what to point a target at"
+ resolverDownShort = "remove khaos's chaos resolver from a network"
+ resolverStatusShort = "show khaos's chaos resolvers"
+
+ zoneUsage = "a name this resolver answers authoritatively, as name=address; repeatable"
+ controlUsage = "the zone name no fault ever touches, used as the confirmation control"
+ networkUsage = "the Docker network to serve, which must be the target's own"
+
+ resolverNamePrefix = "khaos-resolver-"
+ nonceBytes = dnsrv.NonceBytes
+
+ // readyTimeout and readyInterval are how long khaos waits for the resolver's
+ // health endpoint. A container starting from a one-file image has nothing to
+ // initialise, so this only ever covers scheduling.
+ readyTimeout = 15 * time.Second
+ readyInterval = 100 * time.Millisecond
+
+ errZoneFormat = "%w: %q (expected name=address)"
+ errNoZone = "%w: pass --zone name=address at least once"
+ errNoNetwork = "%w: pass --network"
+ errResolverUp = "the chaos resolver on %s never became ready at %s"
+ errNoResolvers = "no chaos resolver is running"
+
+ resolverUpFormat = `chaos resolver serving %s
+
+ container %s
+ address %s
+ control %s
+
+Point the target at it. --dns is fixed at container creation and Docker offers
+no way to change it on a running container, so the target has to be recreated:
+
+ docker run --dns %s ...
+
+ # or, in compose:
+ services:
+ <your-service>:
+ dns: ["%s"]
+
+khaos will refuse to report a verdict until it has seen a query arrive from the
+target, so a target that was not recreated is reported as an invalid run rather
+than as one that passed.
+`
+
+ resolverStatusHeader = "NETWORK ADDRESS CONTAINER"
+ resolverStatusRow = "%-26s %-16s %s\n"
+ resolverDownFormat = "removed the chaos resolver on %s\n"
+)
+
+var ErrZoneFormat = errors.New("a --zone entry must be name=address")
+
+var ErrNoZone = errors.New("a chaos resolver needs at least one zone entry")
+
+var ErrNoNetwork = errors.New("a chaos resolver needs a network to serve")
+
+// newServeResolverCmd is hidden because it is khaos's own plumbing: it runs
+// inside the container khaos builds from its own binary, and an operator running
+// it on the host would be told it cannot bind port 53.
+func newServeResolverCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: dockerx.ResolverCommand,
+ Short: serveResolverShort,
+ Long: serveResolverLong,
+ Hidden: true,
+ Args: cobra.NoArgs,
+ RunE: runServeResolver,
+ }
+
+ cmd.Flags().StringArray(FlagZone, nil, zoneUsage)
+ cmd.Flags().String(FlagControl, "", controlUsage)
+
+ return cmd
+}
+
+func runServeResolver(cmd *cobra.Command, _ []string) error {
+ entries, err := cmd.Flags().GetStringArray(FlagZone)
+ if err != nil {
+ return err
+ }
+
+ zone, err := parseZone(entries)
+ if err != nil {
+ return err
+ }
+
+ // From the ENVIRONMENT, not a flag. A command line is world-readable in
+ // `/proc/<pid>/cmdline` (mode 444, measured) and the nonce is the only
+ // authorisation for arming and disarming khaos's DNS faults.
+ nonce := os.Getenv(dockerx.EnvResolverNonce)
+
+ control, err := cmd.Flags().GetString(FlagControl)
+ if err != nil {
+ return err
+ }
+
+ resolver, err := dnsrv.New(dnsrv.Config{
+ Nonce: nonce,
+ Zone: zone,
+ Control: control,
+ })
+ if err != nil {
+ return err
+ }
+
+ if err := resolver.Serve(); err != nil {
+ return err
+ }
+
+ if err := resolver.ServeControl(""); err != nil {
+ _ = resolver.Shutdown()
+ return err
+ }
+
+ // The container is the fault's lifetime, so the signal handler exists to
+ // make `docker stop` a clean shutdown rather than a kill. There is nothing
+ // to roll back here: the fault is the armed state, and it dies with the
+ // process that holds it.
+ stop := make(chan os.Signal, 1)
+ signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
+ <-stop
+
+ return resolver.Shutdown()
+}
+
+func parseZone(entries []string) (map[string]string, error) {
+ if len(entries) == 0 {
+ return nil, fmt.Errorf(errNoZone, ErrNoZone)
+ }
+
+ zone := make(map[string]string, len(entries))
+
+ for _, entry := range entries {
+ name, address, found := strings.Cut(entry, "=")
+ if !found || name == "" || address == "" {
+ return nil, fmt.Errorf(errZoneFormat, ErrZoneFormat, entry)
+ }
+ zone[name] = address
+ }
+
+ return zone, nil
+}
+
+func newResolverCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "resolver",
+ Short: resolverShort,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ return cmd.Help()
+ },
+ }
+
+ cmd.AddCommand(
+ newResolverUpCmd(),
+ newResolverDownCmd(),
+ newResolverStatusCmd(),
+ )
+
+ return cmd
+}
+
+func newResolverUpCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "up",
+ Short: resolverUpShort,
+ Args: cobra.NoArgs,
+ RunE: runResolverUp,
+ }
+
+ cmd.Flags().String(FlagNetwork, "", networkUsage)
+ cmd.Flags().StringArray(FlagZone, nil, zoneUsage)
+ cmd.Flags().String(FlagControl, "", controlUsage)
+
+ return cmd
+}
+
+func runResolverUp(cmd *cobra.Command, _ []string) error {
+ networkName, err := cmd.Flags().GetString(FlagNetwork)
+ if err != nil {
+ return err
+ }
+ if networkName == "" {
+ return fmt.Errorf(errNoNetwork, ErrNoNetwork)
+ }
+
+ entries, err := cmd.Flags().GetStringArray(FlagZone)
+ if err != nil {
+ return err
+ }
+
+ zone, err := parseZone(entries)
+ if err != nil {
+ return err
+ }
+
+ control, err := cmd.Flags().GetString(FlagControl)
+ if err != nil {
+ return err
+ }
+ if control == "" {
+ control = dockerx.ResolverControlName(networkName)
+ zone[control] = anyZoneAddress(zone)
+ }
+
+ ctx := cmd.Context()
+
+ docker, err := dockerx.Open(ctx)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = docker.Close() }()
+
+ nonce, err := newNonce()
+ if err != nil {
+ return err
+ }
+
+ resolver, err := docker.RunResolver(ctx, dockerx.ResolverSpec{
+ Network: networkName,
+ Name: resolverNamePrefix + networkName,
+ Nonce: nonce,
+ Zone: zone,
+ Control: control,
+ })
+ if err != nil {
+ return err
+ }
+
+ if err := waitReady(ctx, resolver); err != nil {
+ return err
+ }
+
+ _, err = fmt.Fprintf(
+ cmd.OutOrStdout(),
+ resolverUpFormat,
+ resolver.Network,
+ resolver.Name,
+ resolver.Address,
+ dockerx.ResolverControlURL(resolver.Address),
+ resolver.Address,
+ resolver.Address,
+ )
+
+ return err
+}
+
+// waitReady blocks until the resolver answers its own health endpoint, so
+// `resolver up` never prints an address that is not yet listening and an
+// operator never recreates a target against a resolver that failed to start.
+func waitReady(ctx context.Context, resolver dockerx.Resolver) error {
+ client := dnsrv.NewClient(
+ resolver.Address,
+ dockerx.ResolverControlURL(resolver.Address),
+ resolver.Nonce,
+ "",
+ )
+
+ deadline := time.Now().Add(readyTimeout)
+
+ for time.Now().Before(deadline) {
+ if client.Healthy(ctx) {
+ return nil
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(readyInterval):
+ }
+ }
+
+ return fmt.Errorf(
+ errResolverUp,
+ resolver.Network,
+ dockerx.ResolverControlURL(resolver.Address),
+ )
+}
+
+func newResolverDownCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "down",
+ Short: resolverDownShort,
+ Args: cobra.NoArgs,
+ RunE: runResolverDown,
+ }
+
+ cmd.Flags().String(FlagNetwork, "", networkUsage)
+
+ return cmd
+}
+
+func runResolverDown(cmd *cobra.Command, _ []string) error {
+ networkName, err := cmd.Flags().GetString(FlagNetwork)
+ if err != nil {
+ return err
+ }
+ if networkName == "" {
+ return fmt.Errorf(errNoNetwork, ErrNoNetwork)
+ }
+
+ ctx := cmd.Context()
+
+ docker, err := dockerx.Open(ctx)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = docker.Close() }()
+
+ resolver, err := docker.FindResolver(ctx, networkName)
+ if err != nil {
+ return err
+ }
+
+ if err := docker.RemoveResolver(ctx, resolver.ID); err != nil {
+ return err
+ }
+
+ _, err = fmt.Fprintf(cmd.OutOrStdout(), resolverDownFormat, networkName)
+
+ return err
+}
+
+func newResolverStatusCmd() *cobra.Command {
+ return &cobra.Command{
+ Use: "status",
+ Short: resolverStatusShort,
+ Args: cobra.NoArgs,
+ RunE: runResolverStatus,
+ }
+}
+
+func runResolverStatus(cmd *cobra.Command, _ []string) error {
+ ctx := cmd.Context()
+
+ docker, err := dockerx.Open(ctx)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = docker.Close() }()
+
+ resolvers, err := docker.Resolvers(ctx)
+ if err != nil {
+ return err
+ }
+
+ out := cmd.OutOrStdout()
+
+ if len(resolvers) == 0 {
+ _, err := fmt.Fprintln(out, errNoResolvers)
+ return err
+ }
+
+ if _, err := fmt.Fprintln(out, resolverStatusHeader); err != nil {
+ return err
+ }
+
+ for _, resolver := range resolvers {
+ if _, err := fmt.Fprintf(
+ out,
+ resolverStatusRow,
+ resolver.Network,
+ resolver.Address,
+ resolver.Name,
+ ); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func newNonce() (string, error) {
+ buffer := make([]byte, nonceBytes)
+ if _, err := rand.Read(buffer); err != nil {
+ return "", err
+ }
+ return hex.EncodeToString(buffer), nil
+}
+
+// anyZoneAddress gives the generated control name the same address as a real
+// zone entry, so the control resolves to something that exists. It is only ever
+// used for the name khaos generates when the operator did not name one.
+func anyZoneAddress(zone map[string]string) string {
+ for _, address := range zone {
+ return address
+ }
+ return ""
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
index 4fb80dcf..4a279923 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/cli/root.go
@@ -48,6 +48,8 @@ func NewRoot() *cobra.Command {
newTargetsCmd(),
newInjectCmd(),
newRangeCmd(),
+ newResolverCmd(),
+ newServeResolverCmd(),
newCatalogCmd(),
newPlanCmd(),
newRunCmd(),
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go
new file mode 100644
index 00000000..cfe1f588
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/client.go
@@ -0,0 +1,346 @@
+// ©AngelaMos | 2026
+// client.go
+
+package dnsrv
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "time"
+
+ "github.com/miekg/dns"
+)
+
+const (
+ // probeTimeout has to clear the largest delay khaos can arm plus the time a
+ // client spends noticing a truncated answer and re-asking over TCP. At the
+ // MaxDelay ceiling of 3.999s a tighter budget would time out on khaos's own
+ // confirmation and report the resolver as dead.
+ probeTimeout = 10 * time.Second
+
+ controlTimeout = 5 * time.Second
+
+ errControlStatus = "%w: %s answered %s"
+ errControlCall = "reach the resolver's control surface at %s: %w"
+ errExchange = "ask %s for %s over %s: %w"
+ errDecode = "decode the resolver's reply from %s: %w"
+)
+
+var ErrControlRefused = errors.New(
+ "the resolver's control surface refused this request",
+)
+
+// Client is how khaos, on the host, talks to a resolver running in a container.
+// Both addresses are the CONTAINER's own, reached directly: the host routes to
+// container addresses on a bridge network, so neither the DNS port nor the
+// control port is ever published, which is what keeps the whole tier at zero
+// host ports.
+type Client struct {
+ // DNS is the resolver's address:port, used for arming confirmation by
+ // MEASURING what it serves.
+ DNS string
+
+ // Control is the base URL of the resolver's control surface.
+ Control string
+
+ // Nonce authorises every control call. Without it the target, which shares
+ // the network, could arm or disarm khaos's own fault.
+ Nonce string
+
+ // ControlName is the zone name no fault touches, used as the confirmation
+ // oracle's positive control.
+ ControlName string
+
+ // Zone is the honest answer the resolver serves for each name it owns, read
+ // back from it rather than remembered. A redirect's observable needs it to
+ // name the address the fault replaced.
+ Zone map[string]string
+
+ http *http.Client
+}
+
+func NewClient(dnsAddr, controlURL, nonce, controlName string) *Client {
+ return &Client{
+ DNS: dnsAddr,
+ Control: controlURL,
+ Nonce: nonce,
+ ControlName: controlName,
+ http: &http.Client{Timeout: controlTimeout},
+ }
+}
+
+func (c *Client) Arm(ctx context.Context, trap Trap) (Stats, error) {
+ body, err := json.Marshal(trap)
+ if err != nil {
+ return Stats{}, err
+ }
+
+ return c.call(ctx, http.MethodPost, PathArm, body)
+}
+
+func (c *Client) Disarm(ctx context.Context) (Stats, error) {
+ return c.call(ctx, http.MethodPost, PathDisarm, nil)
+}
+
+func (c *Client) State(ctx context.Context) (Stats, error) {
+ return c.call(ctx, http.MethodGet, PathState, nil)
+}
+
+// Healthy reports whether the resolver is up, without the nonce, so khaos can
+// wait for a container to come up before it has read the nonce back out of it.
+func (c *Client) Healthy(ctx context.Context) bool {
+ request, err := http.NewRequestWithContext(
+ ctx,
+ http.MethodGet,
+ c.Control+PathHealthz,
+ nil,
+ )
+ if err != nil {
+ return false
+ }
+
+ response, err := c.client().Do(request)
+ if err != nil {
+ return false
+ }
+ defer func() { _ = response.Body.Close() }()
+ _, _ = io.Copy(io.Discard, response.Body)
+
+ return response.StatusCode == http.StatusOK
+}
+
+func (c *Client) call(
+ ctx context.Context,
+ method string,
+ path string,
+ body []byte,
+) (Stats, error) {
+ var reader io.Reader
+ if body != nil {
+ reader = bytes.NewReader(body)
+ }
+
+ request, err := http.NewRequestWithContext(
+ ctx,
+ method,
+ c.Control+path,
+ reader,
+ )
+ if err != nil {
+ return Stats{}, err
+ }
+ request.Header.Set(NonceHeader, c.Nonce)
+
+ response, err := c.client().Do(request)
+ if err != nil {
+ return Stats{}, fmt.Errorf(errControlCall, c.Control+path, err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ payload, err := io.ReadAll(
+ io.LimitReader(response.Body, controlBodyLimit),
+ )
+ if err != nil {
+ return Stats{}, err
+ }
+
+ if response.StatusCode != http.StatusOK {
+ return Stats{}, fmt.Errorf(
+ errControlStatus,
+ ErrControlRefused,
+ path,
+ firstLine(payload, response.Status),
+ )
+ }
+
+ var stats Stats
+ if err := json.Unmarshal(payload, &stats); err != nil {
+ return Stats{}, fmt.Errorf(errDecode, path, err)
+ }
+
+ return stats, nil
+}
+
+func (c *Client) client() *http.Client {
+ if c.http == nil {
+ c.http = &http.Client{Timeout: controlTimeout}
+ }
+ return c.http
+}
+
+// answer is one measured DNS exchange. Everything the confirmation oracle
+// decides on is in here, and all of it came off the wire.
+type answer struct {
+ rcode int
+ truncated bool
+ records []dns.RR
+ elapsed time.Duration
+ err error
+
+ // authoritative is the AA bit, and it is what separates the armed fault
+ // from khaos's own failure paths. Five paths in serveFault answer SERVFAIL,
+ // so the rcode alone could not tell the armed arm from the default.
+ authoritative bool
+}
+
+func (a answer) addresses() []string {
+ out := make([]string, 0, len(a.records))
+ for _, rr := range a.records {
+ switch record := rr.(type) {
+ case *dns.A:
+ out = append(out, record.A.String())
+ case *dns.AAAA:
+ out = append(out, record.AAAA.String())
+ }
+ }
+ return out
+}
+
+func (a answer) cnames() []string {
+ out := make([]string, 0, len(a.records))
+ for _, rr := range a.records {
+ if record, ok := rr.(*dns.CNAME); ok {
+ out = append(out, dns.CanonicalName(record.Target))
+ }
+ }
+ return out
+}
+
+// ttls returns every TTL in the answer section, so a confirmation can require
+// the armed value rather than merely the presence of a record.
+func (a answer) ttls() []uint32 {
+ out := make([]uint32, 0, len(a.records))
+ for _, rr := range a.records {
+ if rr == nil || rr.Header().Rrtype == dns.TypeOPT {
+ continue
+ }
+ out = append(out, rr.Header().Ttl)
+ }
+ return out
+}
+
+func (a answer) resolved() bool {
+ return a.err == nil &&
+ a.rcode == dns.RcodeSuccess &&
+ len(a.addresses()) > 0
+}
+
+// ask performs one exchange and records what came back, including the failure.
+// A timeout is data here rather than an error to propagate: TruncationNoTCP's
+// whole observable is that neither transport ever completes.
+func (c *Client) ask(name, proto string) answer {
+ client := &dns.Client{Net: proto, Timeout: probeTimeout}
+
+ message := new(dns.Msg)
+ message.SetQuestion(dns.Fqdn(name), dns.TypeA)
+
+ start := time.Now()
+ response, _, err := client.Exchange(message, c.dnsAddr())
+ elapsed := time.Since(start)
+
+ if err != nil {
+ return answer{
+ elapsed: elapsed,
+ err: fmt.Errorf(errExchange, c.DNS, name, proto, err),
+ }
+ }
+
+ return answer{
+ rcode: response.Rcode,
+ truncated: response.Truncated,
+ records: response.Answer,
+ elapsed: elapsed,
+ authoritative: response.Authoritative,
+ }
+}
+
+func (c *Client) dnsAddr() string {
+ if _, _, err := net.SplitHostPort(c.DNS); err != nil {
+ return net.JoinHostPort(c.DNS, fmt.Sprint(DefaultPort))
+ }
+ return c.DNS
+}
+
+func firstLine(payload []byte, fallback string) string {
+ text := string(bytes.TrimSpace(payload))
+ if text == "" {
+ return fallback
+ }
+
+ if index := bytes.IndexByte([]byte(text), '\n'); index > 0 {
+ return text[:index]
+ }
+
+ return text
+}
+
+// Queries reads how many times one source asked khaos's resolver about one name
+// it is authoritative for.
+//
+// A count of zero is the sentinel every guard downstream branches on, so a
+// failed read must not become one: it would make khaos report the TARGET as
+// never having asked when khaos simply could not count.
+func (c *Client) Queries(
+ ctx context.Context,
+ source, zoneName string,
+) (int64, error) {
+ stats, err := c.State(ctx)
+ if err != nil {
+ return 0, err
+ }
+
+ return stats.Queries[QueryKey(source, zoneName)], nil
+}
+
+// Health reads the resolver's liveness endpoint, which also reports the control
+// name no fault touches.
+//
+// khaos reads that name back out of the resolver rather than remembering what
+// it passed, for the reason the identity provider's most expensive bug
+// established: the resolver is the only authority on which name it will refuse
+// to fault, and a confirmation built on khaos's own idea of it would refute
+// every fault the moment the two disagreed.
+func (c *Client) Health(ctx context.Context) (Health, error) {
+ request, err := http.NewRequestWithContext(
+ ctx,
+ http.MethodGet,
+ c.Control+PathHealthz,
+ nil,
+ )
+ if err != nil {
+ return Health{}, err
+ }
+
+ response, err := c.client().Do(request)
+ if err != nil {
+ return Health{}, fmt.Errorf(errControlCall, c.Control+PathHealthz, err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ payload, err := io.ReadAll(io.LimitReader(response.Body, controlBodyLimit))
+ if err != nil {
+ return Health{}, err
+ }
+
+ if response.StatusCode != http.StatusOK {
+ return Health{}, fmt.Errorf(
+ errControlStatus,
+ ErrControlRefused,
+ PathHealthz,
+ firstLine(payload, response.Status),
+ )
+ }
+
+ var health Health
+ if err := json.Unmarshal(payload, &health); err != nil {
+ return Health{}, fmt.Errorf(errDecode, PathHealthz, err)
+ }
+
+ return health, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go
new file mode 100644
index 00000000..d3268a6d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control.go
@@ -0,0 +1,219 @@
+// ©AngelaMos | 2026
+// control.go
+
+package dnsrv
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "time"
+)
+
+const (
+ // ControlPort is where the resolver's control surface listens INSIDE its
+ // container. Nothing is published to the host, and khaos reaches it on the
+ // container's own address the same way it reaches the resolver's DNS port,
+ // so this never collides with anything on the operator's machine and never
+ // needs remapping.
+ ControlPort = 9053
+
+ // NonceHeader authorises a control request. The resolver shares a network
+ // with the target, so the target can reach this surface; the nonce is what
+ // stops it arming or disarming khaos's own fault. khaos generates it,
+ // passes it at container creation, and reads it back out of the container
+ // rather than remembering what it chose.
+ NonceHeader = "X-Khaos-Nonce"
+
+ // NonceBytes is 16, so a target cannot find the nonce by trying. Every
+ // refused attempt is counted and reported in Stats, which turns guessing
+ // into evidence rather than noise.
+ NonceBytes = 16
+
+ PathArm = "/arm"
+ PathDisarm = "/disarm"
+ PathState = "/state"
+ PathHealthz = "/healthz"
+
+ controlReadTimeout = 5 * time.Second
+ controlWriteTimeout = 5 * time.Second
+ controlHeaderTimeout = 3 * time.Second
+ controlBodyLimit = 4 << 10
+)
+
+const (
+ errControlBind = "bind the resolver's control surface on %s: %w"
+ errControlUp = "%w: this resolver's control surface is already serving"
+ errControlReply = "encode the control reply: %w"
+)
+
+var ErrControlServing = errors.New(
+ "that resolver's control surface is already serving",
+)
+
+// Health is what /healthz answers. It is a liveness signal and deliberately NOT
+// nonce-protected, because khaos has to be able to wait for the container to
+// come up before it has read the nonce back out of it, and because a readiness
+// probe that needs a secret is one khaos cannot use from a `range status`.
+//
+// It reports nothing about the armed fault. A target that reads it learns only
+// that a resolver exists, which it already knows from its own resolv.conf.
+type Health struct {
+ OK bool `json:"ok"`
+ Upstream string `json:"upstream"`
+ Control string `json:"control"`
+}
+
+// ServeControl binds the control surface and returns once it is accepting.
+func (r *Resolver) ServeControl(listen string) error {
+ if listen == "" {
+ listen = fmt.Sprintf(":%d", ControlPort)
+ }
+
+ mux := http.NewServeMux()
+ mux.HandleFunc(PathHealthz, r.handleHealth)
+ mux.HandleFunc(PathState, r.guard(r.handleState))
+ mux.HandleFunc(PathArm, r.guard(r.handleArm))
+ mux.HandleFunc(PathDisarm, r.guard(r.handleDisarm))
+
+ server := &http.Server{
+ Handler: mux,
+ ReadTimeout: controlReadTimeout,
+ WriteTimeout: controlWriteTimeout,
+ ReadHeaderTimeout: controlHeaderTimeout,
+ }
+
+ listener, err := net.Listen(tcpNet, listen)
+ if err != nil {
+ return fmt.Errorf(errControlBind, listen, err)
+ }
+
+ r.mu.Lock()
+ if r.control != nil {
+ r.mu.Unlock()
+ _ = listener.Close()
+ return fmt.Errorf(errControlUp, ErrControlServing)
+ }
+ r.control = server
+ r.controlAddr = listener.Addr().String()
+ r.mu.Unlock()
+
+ // The goroutine serves the LOCALS it was handed. Reading them back off the
+ // struct is the exact shape that gave `internal/pki` three data races, and
+ // the plain test suite was green for both of its new packages.
+ go func() {
+ _ = server.Serve(listener)
+ }()
+
+ return nil
+}
+
+// ControlAddr reports where the control surface bound, which a test needs when
+// it asked for port 0.
+func (r *Resolver) ControlAddr() string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.controlAddr
+}
+
+func (r *Resolver) shutdownControl() error {
+ r.mu.Lock()
+ server := r.control
+ r.control = nil
+ r.controlAddr = ""
+ r.mu.Unlock()
+
+ if server == nil {
+ return nil
+ }
+
+ return server.Close()
+}
+
+// guard refuses a request carrying the wrong nonce and COUNTS it. A refusal
+// that is merely returned tells khaos nothing; a refusal that is counted is
+// evidence khaos can read in the run record.
+func (r *Resolver) guard(
+ next func(http.ResponseWriter, *http.Request),
+) func(http.ResponseWriter, *http.Request) {
+ return func(w http.ResponseWriter, req *http.Request) {
+ if req.Header.Get(NonceHeader) != r.cfg.Nonce || r.cfg.Nonce == "" {
+ r.countDenied()
+ http.Error(
+ w,
+ http.StatusText(http.StatusForbidden),
+ http.StatusForbidden,
+ )
+ return
+ }
+
+ r.countControl()
+ next(w, req)
+ }
+}
+
+func (r *Resolver) handleHealth(w http.ResponseWriter, _ *http.Request) {
+ r.writeJSON(w, Health{
+ OK: true,
+ Upstream: r.cfg.Upstream,
+ Control: r.cfg.Control,
+ })
+}
+
+func (r *Resolver) handleState(w http.ResponseWriter, _ *http.Request) {
+ r.writeJSON(w, r.Stats())
+}
+
+// handleArm installs a fault and answers with the resolver's resulting state,
+// so the caller gets a synchronous acknowledgement of what is now armed.
+//
+// The acknowledgement is deliberately NOT what khaos confirms the fault by. A
+// state echo says which branch was chosen, not what a target would receive, and
+// the PKI tier's inert-authority control caught fifteen traps confirming
+// themselves off exactly that kind of read-back. Confirmation measures the
+// SERVED answer.
+func (r *Resolver) handleArm(w http.ResponseWriter, req *http.Request) {
+ var trap Trap
+
+ decoder := json.NewDecoder(
+ http.MaxBytesReader(w, req.Body, controlBodyLimit),
+ )
+ if err := decoder.Decode(&trap); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ if err := r.Arm(trap); err != nil {
+ http.Error(w, err.Error(), http.StatusUnprocessableEntity)
+ return
+ }
+
+ r.writeJSON(w, r.Stats())
+}
+
+func (r *Resolver) handleDisarm(w http.ResponseWriter, _ *http.Request) {
+ r.Disarm()
+ r.writeJSON(w, r.Stats())
+}
+
+func (r *Resolver) writeJSON(w http.ResponseWriter, payload any) {
+ w.Header().Set("Content-Type", "application/json")
+
+ if err := json.NewEncoder(w).Encode(payload); err != nil {
+ r.countError()
+ }
+}
+
+func (r *Resolver) countControl() {
+ r.mu.Lock()
+ r.controls++
+ r.mu.Unlock()
+}
+
+func (r *Resolver) countDenied() {
+ r.mu.Lock()
+ r.denied++
+ r.mu.Unlock()
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control_test.go
new file mode 100644
index 00000000..627e9f0a
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/control_test.go
@@ -0,0 +1,253 @@
+// ©AngelaMos | 2026
+// control_test.go
+
+package dnsrv
+
+import (
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/miekg/dns"
+)
+
+// Every test in this file exists because a gutting pass deleted the mechanism it
+// covers and the whole suite stayed green. They are the controls, not the
+// features.
+
+// TestArmValidates is the control for Arm's call to Validate.
+//
+// Deleting that call left the suite green: the masked-fault test called
+// Validate directly, and the zone guard sits in Arm's own body, so nothing
+// asserted that arming a trap validates it at all. The control surface reaches
+// Arm from outside khaos, so a masked fault or an over-ceiling delay arriving
+// there has to be refused by Arm itself and not by the caller remembering to
+// check first.
+func TestArmValidates(t *testing.T) {
+ cases := []struct {
+ name string
+ trap Trap
+ want error
+ }{
+ {
+ name: "a masked fault",
+ trap: Trap{Kind: Refused, Host: testHost},
+ want: ErrMasked,
+ },
+ {
+ name: "a delay above the measured ceiling",
+ trap: Trap{
+ Kind: SlowResponse,
+ Host: testHost,
+ Delay: 30 * time.Second,
+ },
+ want: ErrDelayRange,
+ },
+ {
+ name: "an unknown fault",
+ trap: Trap{Kind: "not-a-fault", Host: testHost},
+ want: ErrUnknownKind,
+ },
+ {
+ name: "a redirect to something that is not an address",
+ trap: Trap{Kind: WrongAddress, Host: testHost, Address: "nope"},
+ want: ErrBadAddress,
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ if err := resolver.Arm(test.trap); !errors.Is(err, test.want) {
+ t.Errorf("Arm error %v, want %v", err, test.want)
+ }
+
+ if _, err := client.Arm(t.Context(), test.trap); err == nil {
+ t.Error("the control surface armed a trap Arm must refuse")
+ }
+ })
+ }
+}
+
+// TestObserveRefutesAgainstAForeignResolver is the control for Observe's
+// control-name check.
+//
+// Deleting that check left the suite green, because a dead resolver also fails
+// the fault-side query. The case it really guards is worse and quieter: khaos
+// pointed at a resolver it never armed. A resolver that NXDOMAINs everything
+// answers NXDOMAIN for the faulted name too, so without the control half khaos
+// confirms a fault it did not inject and reports a finding about a resolver that
+// was never part of the experiment.
+func TestObserveRefutesAgainstAForeignResolver(t *testing.T) {
+ foreign := testUpstreamServer(t)
+
+ client := NewClient(foreign, "http://127.0.0.1:1", testNonce, testControl)
+
+ observable, confirmed, err := client.Observe(trapFor(NXDomain))
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if confirmed {
+ t.Fatalf("confirmed NXDOMAIN against a resolver khaos never armed: %s",
+ observable)
+ }
+}
+
+// TestTTLObservationRequiresADifference is the control for ttlObservation's
+// control half.
+//
+// That arm was unreachable through the live resolver, because khaos always
+// serves HonestTTL for the control name, so deleting it changed nothing. It
+// encodes a real rule even so: the fault is a DIFFERENCE from what the same
+// resolver serves for a name it is not faulting, and a zone that happened to
+// serve a zero TTL would otherwise confirm TTLZero against a resolver with
+// nothing armed.
+func TestTTLObservationRequiresADifference(t *testing.T) {
+ trap := Trap{Kind: TTLZero, Host: testHost}
+
+ withTTL := func(ttl uint32) answer {
+ record, err := dns.NewRR("x.test. 0 IN A 10.0.0.1")
+ if err != nil {
+ t.Fatalf("build record: %v", err)
+ }
+ record.Header().Ttl = ttl
+ return answer{records: []dns.RR{record}}
+ }
+
+ _, confirmed, _ := ttlObservation(
+ trap,
+ withTTL(0),
+ withTTL(HonestTTL),
+ testControl,
+ )
+ if !confirmed {
+ t.Error("did not confirm a zero TTL the control did not share")
+ }
+
+ _, confirmed, _ = ttlObservation(
+ trap,
+ withTTL(0),
+ withTTL(0),
+ testControl,
+ )
+ if confirmed {
+ t.Error("confirmed a zero TTL where the CONTROL also carried zero, " +
+ "which is not a difference and therefore not this fault")
+ }
+}
+
+// TestTruncationObservablesRejectEachOthersAnswers is the control for the
+// truncation arms.
+//
+// Both faults set the bit over UDP, so UDP alone cannot separate them and every
+// arm that does lived behind a live resolver where deleting one went unnoticed.
+// Driving the pure functions directly gives each arm an input it must reject.
+func TestTruncationObservablesRejectEachOthersAnswers(t *testing.T) {
+ resolved := func() answer {
+ record, err := dns.NewRR("x.test. 600 IN A 10.0.0.1")
+ if err != nil {
+ t.Fatalf("build record: %v", err)
+ }
+ return answer{records: []dns.RR{record}}
+ }
+
+ completeTCP := resolved()
+ truncatedEmptyTCP := answer{truncated: true}
+ truncatedWithRecordsTCP := resolved()
+ truncatedWithRecordsTCP.truncated = true
+ unreachableTCP := answer{err: errors.New("connection refused")}
+ servfailTCP := answer{rcode: dns.RcodeServerFailure}
+
+ trap := Trap{Kind: Truncation, Host: testHost}
+ noTCPTrap := Trap{Kind: TruncationNoTCP, Host: testHost}
+ truncatedUDP := answer{truncated: true}
+ plainUDP := resolved()
+
+ cases := []struct {
+ name string
+ confirm bool
+ got func() (string, bool, error)
+ }{
+ {
+ name: "truncation with a complete TCP answer confirms",
+ confirm: true,
+ got: func() (string, bool, error) {
+ return truncObservation(trap, truncatedUDP, completeTCP)
+ },
+ },
+ {
+ name: "truncation refutes an untruncated UDP answer",
+ confirm: false,
+ got: func() (string, bool, error) {
+ return truncObservation(trap, plainUDP, completeTCP)
+ },
+ },
+ {
+ name: "truncation refutes the no-TCP fault's TCP answer",
+ confirm: false,
+ got: func() (string, bool, error) {
+ return truncObservation(trap, truncatedUDP, truncatedEmptyTCP)
+ },
+ },
+ {
+ name: "truncation refutes an unreachable TCP path",
+ confirm: false,
+ got: func() (string, bool, error) {
+ return truncObservation(trap, truncatedUDP, unreachableTCP)
+ },
+ },
+ {
+ name: "no-TCP confirms a truncated empty TCP answer",
+ confirm: true,
+ got: func() (string, bool, error) {
+ return truncNoTCPObservation(
+ noTCPTrap, truncatedUDP, truncatedEmptyTCP,
+ )
+ },
+ },
+ {
+ name: "no-TCP refutes an unreachable TCP path",
+ confirm: false,
+ got: func() (string, bool, error) {
+ return truncNoTCPObservation(
+ noTCPTrap, truncatedUDP, unreachableTCP,
+ )
+ },
+ },
+ {
+ name: "no-TCP refutes a TCP answer that is not truncated",
+ confirm: false,
+ got: func() (string, bool, error) {
+ return truncNoTCPObservation(
+ noTCPTrap,
+ truncatedUDP,
+ servfailTCP,
+ )
+ },
+ },
+ {
+ name: "no-TCP refutes a truncated TCP answer that still resolves",
+ confirm: false,
+ got: func() (string, bool, error) {
+ return truncNoTCPObservation(
+ noTCPTrap, truncatedUDP, truncatedWithRecordsTCP,
+ )
+ },
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ observable, confirmed, err := test.got()
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+ if confirmed != test.confirm {
+ t.Errorf("confirmed=%t, want %t: %s",
+ confirmed, test.confirm, observable)
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go
new file mode 100644
index 00000000..17c6b178
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/helpers_test.go
@@ -0,0 +1,162 @@
+// ©AngelaMos | 2026
+// helpers_test.go
+
+package dnsrv
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/miekg/dns"
+)
+
+const (
+ testHost = "dependency.khaos.test."
+ testControl = "control.khaos.test."
+ testAddress = "10.77.77.7"
+ testNonce = "test-nonce-0123456789abcdef"
+ testUpstream = "upstream.elsewhere.test."
+ testUpAddr = "10.88.88.8"
+ loopbackAny = "127.0.0.1:0"
+)
+
+// testUpstreamServer is a resolver khaos is NOT authoritative for, so the
+// forwarding path can be exercised without reaching anything real. It answers
+// exactly one name, which is what makes "khaos forwarded this" distinguishable
+// from "khaos invented an answer".
+func testUpstreamServer(t *testing.T) string {
+ t.Helper()
+
+ mux := dns.NewServeMux()
+ mux.HandleFunc(".", func(w dns.ResponseWriter, req *dns.Msg) {
+ reply := new(dns.Msg)
+ reply.SetReply(req)
+
+ if len(req.Question) > 0 &&
+ dns.CanonicalName(req.Question[0].Name) == testUpstream &&
+ req.Question[0].Qtype == dns.TypeA {
+ rr, err := dns.NewRR(
+ fmt.Sprintf("%s 42 IN A %s", testUpstream, testUpAddr),
+ )
+ if err == nil {
+ reply.Answer = append(reply.Answer, rr)
+ }
+ } else {
+ reply.SetRcode(req, dns.RcodeNameError)
+ }
+
+ _ = w.WriteMsg(reply)
+ })
+
+ conn, listener, err := bindBoth(loopbackAny)
+ if err != nil {
+ t.Fatalf("bind test upstream: %v", err)
+ }
+
+ udp := &dns.Server{PacketConn: conn, Handler: mux}
+ tcp := &dns.Server{Listener: listener, Handler: mux}
+
+ started := make(chan struct{}, 2)
+ udp.NotifyStartedFunc = func() { started <- struct{}{} }
+ tcp.NotifyStartedFunc = func() { started <- struct{}{} }
+
+ go func() { _ = udp.ActivateAndServe() }()
+ go func() { _ = tcp.ActivateAndServe() }()
+
+ <-started
+ <-started
+
+ t.Cleanup(func() {
+ _ = udp.Shutdown()
+ _ = tcp.Shutdown()
+ })
+
+ return conn.LocalAddr().String()
+}
+
+// startResolver brings up a resolver and its control surface on loopback high
+// ports and returns a client pointed at both.
+//
+// Nothing here needs port 53 or a container, which is the property that makes
+// the whole tier testable: the container exists because khaos cannot BIND 53,
+// not because the resolver needs one.
+func startResolver(t *testing.T) (*Resolver, *Client) {
+ t.Helper()
+
+ upstream := testUpstreamServer(t)
+
+ resolver, err := New(Config{
+ Listen: loopbackAny,
+ Upstream: upstream,
+ Nonce: testNonce,
+ Control: testControl,
+ Zone: map[string]string{
+ testHost: testAddress,
+ testControl: testAddress,
+ },
+ })
+ if err != nil {
+ t.Fatalf("new resolver: %v", err)
+ }
+
+ if err := resolver.Serve(); err != nil {
+ t.Fatalf("serve: %v", err)
+ }
+
+ if err := resolver.ServeControl(loopbackAny); err != nil {
+ t.Fatalf("serve control: %v", err)
+ }
+
+ t.Cleanup(func() { _ = resolver.Shutdown() })
+
+ client := NewClient(
+ resolver.Addr(),
+ "http://"+resolver.ControlAddr(),
+ testNonce,
+ testControl,
+ )
+
+ return resolver, client
+}
+
+// ask is the test's own exchange, used where a test needs to see the raw reply
+// rather than the oracle's verdict.
+func ask(t *testing.T, client *Client, name, proto string) answer {
+ t.Helper()
+ return client.ask(name, proto)
+}
+
+func mustArm(t *testing.T, client *Client, trap Trap) {
+ t.Helper()
+
+ if _, err := client.Arm(t.Context(), trap); err != nil {
+ t.Fatalf("arm %s: %v", trap.Kind, err)
+ }
+}
+
+// deliverableKinds is every fault khaos actually serves. Tests iterate it so a
+// new kind cannot be added without the controls below covering it.
+func deliverableKinds() []Kind {
+ out := make([]Kind, 0, len(Kinds()))
+ for _, name := range Kinds() {
+ kind := Kind(name)
+ if Deliverable(kind) {
+ out = append(out, kind)
+ }
+ }
+ return out
+}
+
+// trapFor builds a minimal valid trap for a kind, so a table-driven control can
+// cover every kind without restating magnitudes.
+func trapFor(kind Kind) Trap {
+ trap := Trap{Kind: kind, Host: testHost}
+
+ if kind == SlowResponse {
+ // Well above the oracle's differential margin and well under the
+ // embedded resolver's ceiling, so the control is not deciding on noise.
+ trap.Delay = slowMargin * 2
+ }
+
+ return trap
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go
new file mode 100644
index 00000000..541e733c
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe.go
@@ -0,0 +1,473 @@
+// ©AngelaMos | 2026
+// observe.go
+
+package dnsrv
+
+import (
+ "fmt"
+ "slices"
+ "time"
+
+ "github.com/miekg/dns"
+)
+
+const (
+ // slowMargin is how much slower the faulted name has to be than the control
+ // before khaos will call a delay confirmed. It is a differential rather than
+ // an absolute threshold, which is the proxy tier's most expensive lesson
+ // restated: an absolute threshold confirms `hang` or `latency` for free
+ // against any dependency that is simply slow.
+ slowMargin = 500 * time.Millisecond
+
+ // chainMinimum is how many CNAME hops have to be present before a chain is
+ // a chain. One CNAME is an ordinary alias and every second zone has them;
+ // the fault is the indirection, so two is the floor.
+ chainMinimum = 2
+
+ // unknownHonest stands in when khaos has not read the resolver's zone back.
+ // It is named rather than blank so a reader can tell a missing read-back from
+ // an address that happens to be empty.
+ unknownHonest = "the address it really has"
+)
+
+const (
+ observeNXDomain = "answering NXDOMAIN for %s while still resolving %s"
+ observeServFail = "answering SERVFAIL for %s while still resolving %s"
+ observeSlow = "answering %s correctly after %s, against %s for the control on the same resolver"
+ observeWrong = "answering %s with %v instead of %s"
+ observeChain = "answering %s with a %d-hop CNAME ending at %s"
+ observeLoop = "answering %s with a CNAME cycle through %v"
+ observeTrunc = "setting the truncation bit for %s over UDP with an empty answer, and serving %v completely over TCP"
+ observeTruncNo = "setting the truncation bit for %s with an empty answer over UDP and over TCP, so no transport ever completes"
+ observeTTL = "answering %s with a TTL of %d, against %d for %s on the same resolver"
+
+ refuteControl = "the control name %s did not resolve through this resolver (%v), so the resolver is broken rather than lying"
+ refuteNoFault = "%s still resolves to %v through this resolver, so nothing is armed against it"
+ refuteRcode = "%s answered %s rather than %s"
+ refuteNotAuthoritative = "%s answered %s without the authoritative bit, so it came from khaos's own failure path rather than from the armed fault"
+ refuteSlowUnanswered = "%s was never answered at all (%v), which is an unresponsive resolver rather than a slow one; khaos cannot report a delay it did not receive"
+ refuteSlowNotAnswer = "%s answered %s with %d addresses, and this fault serves the HONEST answer late: a slow failure is a different fault"
+ refuteSlowWrong = "%s answered %v late instead of %s, so the answer was changed as well as delayed and this fault changes only the timing"
+ refuteWrong = "%s answered %v, which does not contain %s"
+ refuteWrongIndirect = "%s answered the right address through CNAMEs %v: that is the chain fault, not a direct redirect"
+ refuteChain = "%s answered %d CNAME hops, and a chain needs %d"
+ refuteChainCycle = "%s answered CNAMEs %v, which cycle: that is the loop fault, not a chain"
+ refuteTruncReach = "%s could not be reached over TCP at all (%v), which is an unreachable resolver rather than one that truncates on both transports"
+ refuteLoop = "%s answered CNAMEs %v, which do not form a cycle"
+ refuteSlow = "%s answered in %s against the control's %s, a difference under the %s margin"
+ refuteTruncUDP = "%s was not truncated over UDP"
+ refuteTruncTCP = "%s was %s over TCP, and this fault requires the opposite"
+ refuteTTL = "%s answered TTLs %v, and this fault serves %d"
+ refuteUnknown = "no observable is defined for %q, so it cannot be confirmed"
+ refuteMasked = "%s is a fault khaos refuses to deliver, so there is nothing to confirm"
+)
+
+// Observe measures what the resolver actually SERVES and reports the observable
+// alongside the verdict. It never reads the armed state back.
+//
+// That distinction is the whole design of this function. A state echo from
+// /arm says which branch was chosen, not what a target would receive, and the
+// PKI tier's inert-authority control caught fifteen traps confirming themselves
+// off exactly that kind of read-back. Here the resolver is asked the same
+// question the target asks, over the same transports, and the answer is the
+// evidence.
+//
+// Every observation is TWO-SIDED. The faulted name has to show the fault AND
+// the control name has to keep resolving, because those two failures look
+// identical from one query: a resolver that has crashed answers nothing for
+// both, and calling that a confirmed NXDOMAIN fault would be a finding
+// manufactured out of khaos's own breakage.
+func (c *Client) Observe(trap Trap) (string, bool, error) {
+ if !Deliverable(trap.Kind) {
+ return "", false, fmt.Errorf(
+ errMasked,
+ ErrMasked,
+ fmt.Sprintf(refuteMasked, string(trap.Kind)),
+ )
+ }
+
+ control := c.ask(c.ControlName, udpNet)
+ if !control.resolved() {
+ return refutation(
+ refuteControl,
+ c.ControlName,
+ controlDetail(control),
+ )
+ }
+
+ faulted := c.ask(trap.Host, udpNet)
+
+ switch trap.Kind {
+ case NXDomain:
+ return rcodeObservation(
+ trap, faulted, dns.RcodeNameError, observeNXDomain, c.ControlName,
+ )
+
+ case ServFail:
+ return rcodeObservation(
+ trap,
+ faulted,
+ dns.RcodeServerFailure,
+ observeServFail,
+ c.ControlName,
+ )
+
+ case WrongAddress:
+ return wrongObservation(trap, faulted, c.honest(trap.Host))
+
+ case CNAMEChain:
+ return chainObservation(trap, faulted)
+
+ case CNAMELoop:
+ return loopObservation(trap, faulted)
+
+ case SlowResponse:
+ return slowObservation(trap, faulted, control, c.honest(trap.Host))
+
+ case Truncation:
+ return truncObservation(trap, faulted, c.ask(trap.Host, tcpNet))
+
+ case TruncationNoTCP:
+ return truncNoTCPObservation(trap, faulted, c.ask(trap.Host, tcpNet))
+
+ case TTLZero, TTLLong:
+ return ttlObservation(trap, faulted, control, c.ControlName)
+
+ default:
+ return refutation(refuteUnknown, string(trap.Kind))
+ }
+}
+
+// rcodeObservation requires the rcode AND the authoritative bit.
+//
+// AA is what separates the armed fault from khaos's own breakage. Five paths in
+// serveFault answer SERVFAIL — the armed arm, the unreachable-kind arm, the
+// default, a record-building failure and a forwarding failure — so deleting
+// `case ServFail:` fell through to `default`, served the same rcode, and left
+// every test and the whole gate green. khaos IS authoritative for a name in its
+// own zone, so the fault sets AA and the failure paths do not, which makes the
+// two distinguishable from the outside by exactly the thing that differs.
+func rcodeObservation(
+ trap Trap,
+ faulted answer,
+ want int,
+ observed string,
+ controlName string,
+) (string, bool, error) {
+ if faulted.err != nil {
+ return refutation(refuteRcode, trap.Host, faulted.err, rcodeName(want))
+ }
+
+ if faulted.rcode != want {
+ return refutation(
+ refuteRcode,
+ trap.Host,
+ rcodeName(faulted.rcode),
+ rcodeName(want),
+ )
+ }
+
+ if !faulted.authoritative {
+ return refutation(refuteNotAuthoritative, trap.Host, rcodeName(want))
+ }
+
+ return fmt.Sprintf(observed, trap.Host, controlName), true, nil
+}
+
+// wrongObservation requires the wrong address to be answered DIRECTLY, with no
+// CNAME in the reply, and the isolation control is what forced that too.
+//
+// Once the chain fault was made to end at the same address — which it has to, or
+// the indirection goes nowhere and there is nothing for a control to have failed
+// to notice — its final A record satisfied this observable and the two faults
+// alibi each other again. The claims really are different: this one is "the name
+// resolves somewhere else", and the chain's is "the name resolves somewhere else
+// THROUGH indirection that defeats matching on the name you asked for".
+func wrongObservation(
+ trap Trap,
+ faulted answer,
+ honest string,
+) (string, bool, error) {
+ addresses := faulted.addresses()
+ want := trap.address()
+
+ if !slices.Contains(addresses, want) {
+ return refutation(refuteWrong, trap.Host, addresses, want)
+ }
+
+ if hops := faulted.cnames(); len(hops) > 0 {
+ return refutation(refuteWrongIndirect, trap.Host, hops)
+ }
+
+ // The HONEST address, not the one khaos was told to serve. Naming the served
+ // address on both sides of "instead of" states the same thing twice and
+ // tells a reader nothing about what was replaced.
+ if honest == "" {
+ honest = unknownHonest
+ }
+
+ return fmt.Sprintf(observeWrong, trap.Host, addresses, honest), true, nil
+}
+
+// honest reports the address this resolver really serves for a name, read back
+// from the resolver rather than assumed.
+func (c *Client) honest(host string) string {
+ if c.Zone == nil {
+ return ""
+ }
+
+ return c.Zone[dns.CanonicalName(host)]
+}
+
+// chainObservation requires hops that do NOT cycle, and the isolation control
+// is what forced that.
+//
+// A loop also has three CNAME hops, so a hop count alone confirmed CNAMELoop's
+// answer as a chain. Two observables that accept the same reply alibi each
+// other: gut whichever one builds the records and the other's control stays
+// green. The chain's claim is indirection ENDING somewhere, so it has to reject
+// indirection that goes nowhere.
+func chainObservation(trap Trap, faulted answer) (string, bool, error) {
+ hops := faulted.cnames()
+ if len(hops) < chainMinimum {
+ return refutation(refuteChain, trap.Host, len(hops), chainMinimum)
+ }
+
+ if hasCycle(hops) {
+ return refutation(refuteChainCycle, trap.Host, hops)
+ }
+
+ return fmt.Sprintf(
+ observeChain,
+ trap.Host,
+ len(hops),
+ hops[len(hops)-1],
+ ), true, nil
+}
+
+// loopObservation requires an actual cycle rather than merely several hops,
+// because a chain also has several hops and the two faults would then alibi
+// each other: gut whichever one produces the records and the other's control
+// still passes.
+func loopObservation(trap Trap, faulted answer) (string, bool, error) {
+ hops := faulted.cnames()
+
+ if !hasCycle(hops) {
+ return refutation(refuteLoop, trap.Host, hops)
+ }
+
+ return fmt.Sprintf(observeLoop, trap.Host, hops), true, nil
+}
+
+func hasCycle(hops []string) bool {
+ seen := make(map[string]bool, len(hops))
+ for _, hop := range hops {
+ if seen[hop] {
+ return true
+ }
+ seen[hop] = true
+ }
+ return false
+}
+
+// slowObservation requires the slow answer to be an ANSWER, and the first
+// version did not.
+//
+// It was the only observable in this file that never looked at `faulted.err` or
+// `faulted.resolved()`. `ask` records a failure as `answer{elapsed, err}` and
+// `probeTimeout` is 10s, and `miekg/dns` Client.Exchange is one dial, one write,
+// one read with no retry — so a single dropped UDP datagram on khaos's own
+// confirmation query produced `elapsed ≈ 10s` with a non-nil error and this
+// returned CONFIRMED, wording it as "answering <name> after 10s". khaos had
+// received nothing. It then measured a full fault window and attributed what it
+// saw to a delay it could not prove was served.
+//
+// Every sibling already failed safe on exactly this — both truncation arms
+// refute an unreachable TCP path BY NAME. The lesson was written in this file
+// and not applied to the UDP case.
+//
+// The honest-address check is the second half. The kind promises an answer that
+// is "correct but late", so a slow NXDOMAIN or a slow redirect satisfied the old
+// predicate and the report said it answered correctly. The ordered cross-product
+// cannot catch that, because it only ever confronts this fault with faults that
+// are FAST.
+func slowObservation(
+ trap Trap,
+ faulted answer,
+ control answer,
+ honest string,
+) (string, bool, error) {
+ if faulted.err != nil {
+ return refutation(refuteSlowUnanswered, trap.Host, faulted.err)
+ }
+
+ if !faulted.resolved() {
+ return refutation(
+ refuteSlowNotAnswer,
+ trap.Host,
+ rcodeName(faulted.rcode),
+ len(faulted.addresses()),
+ )
+ }
+
+ if honest != "" && !slices.Contains(faulted.addresses(), honest) {
+ return refutation(
+ refuteSlowWrong,
+ trap.Host,
+ faulted.addresses(),
+ honest,
+ )
+ }
+
+ if faulted.elapsed-control.elapsed < slowMargin {
+ return refutation(
+ refuteSlow,
+ trap.Host,
+ faulted.elapsed.Round(time.Millisecond),
+ control.elapsed.Round(time.Millisecond),
+ slowMargin,
+ )
+ }
+
+ return fmt.Sprintf(
+ observeSlow,
+ trap.Host,
+ faulted.elapsed.Round(time.Millisecond),
+ control.elapsed.Round(time.Millisecond),
+ ), true, nil
+}
+
+// truncObservation needs BOTH halves. A truncated UDP answer alone does not
+// distinguish this fault from TruncationNoTCP, and the difference between them
+// is the entire point: one tests whether the client makes the RFC 5966 retry,
+// the other tests whether the retry is permitted to leave the network.
+//
+// The decision is split from the measuring so each arm has a control. Every arm
+// below was reachable only through a live resolver before, and the gutting pass
+// showed what that costs: deleting one left the whole suite green.
+func truncObservation(
+ trap Trap,
+ overUDP answer,
+ overTCP answer,
+) (string, bool, error) {
+ if !overUDP.truncated {
+ return refutation(refuteTruncUDP, trap.Host)
+ }
+
+ if overTCP.err != nil {
+ return refutation(refuteTruncReach, trap.Host, overTCP.err)
+ }
+
+ if overTCP.truncated {
+ return refutation(refuteTruncTCP, trap.Host, "also truncated")
+ }
+
+ if !overTCP.resolved() {
+ return refutation(
+ refuteTruncTCP,
+ trap.Host,
+ "not answered completely",
+ )
+ }
+
+ return fmt.Sprintf(
+ observeTrunc,
+ trap.Host,
+ overTCP.addresses(),
+ ), true, nil
+}
+
+// truncNoTCPObservation requires the truncation bit ON TCP, not merely a TCP
+// path that produced no answer. "The resolver truncates on both transports" and
+// "TCP is unreachable" are different findings and only the first is this fault.
+func truncNoTCPObservation(
+ trap Trap,
+ overUDP answer,
+ overTCP answer,
+) (string, bool, error) {
+ if !overUDP.truncated {
+ return refutation(refuteTruncUDP, trap.Host)
+ }
+
+ if overTCP.err != nil {
+ return refutation(refuteTruncReach, trap.Host, overTCP.err)
+ }
+
+ if !overTCP.truncated {
+ return refutation(refuteTruncTCP, trap.Host, "not truncated")
+ }
+
+ if overTCP.resolved() {
+ return refutation(refuteTruncTCP, trap.Host, "answered completely")
+ }
+
+ return fmt.Sprintf(observeTruncNo, trap.Host), true, nil
+}
+
+// ttlObservation requires the armed TTL on the faulted name AND a different one
+// on the control.
+//
+// The control half is load-bearing rather than decorative. A zone that happens
+// to serve a zero TTL would confirm TTLZero against a resolver with nothing
+// armed, so the fault has to be a DIFFERENCE from what the same resolver serves
+// for a name it is not faulting.
+func ttlObservation(
+ trap Trap,
+ faulted answer,
+ control answer,
+ controlName string,
+) (string, bool, error) {
+ want := trap.ttl()
+ ttls := faulted.ttls()
+
+ if len(ttls) == 0 || !allEqual(ttls, want) {
+ return refutation(refuteTTL, trap.Host, ttls, want)
+ }
+
+ controlTTLs := control.ttls()
+ if len(controlTTLs) > 0 && allEqual(controlTTLs, want) {
+ return refutation(refuteTTL, controlName, controlTTLs, want)
+ }
+
+ return fmt.Sprintf(
+ observeTTL,
+ trap.Host,
+ want,
+ firstTTL(controlTTLs),
+ controlName,
+ ), true, nil
+}
+
+func allEqual(values []uint32, want uint32) bool {
+ for _, value := range values {
+ if value != want {
+ return false
+ }
+ }
+ return len(values) > 0
+}
+
+func firstTTL(values []uint32) uint32 {
+ if len(values) == 0 {
+ return 0
+ }
+ return values[0]
+}
+
+func refutation(format string, args ...any) (string, bool, error) {
+ return fmt.Sprintf(format, args...), false, nil
+}
+
+func controlDetail(control answer) any {
+ if control.err != nil {
+ return control.err
+ }
+ return rcodeName(control.rcode)
+}
+
+func rcodeName(rcode int) string {
+ if name, ok := dns.RcodeToString[rcode]; ok {
+ return name
+ }
+ return fmt.Sprint(rcode)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe_test.go
new file mode 100644
index 00000000..5219e2de
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/observe_test.go
@@ -0,0 +1,394 @@
+// ©AngelaMos | 2026
+// observe_test.go
+
+package dnsrv
+
+import (
+ "errors"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/miekg/dns"
+)
+
+func is(err, target error) bool {
+ return errors.Is(err, target)
+}
+
+func contains(text, substring string) bool {
+ return strings.Contains(text, substring)
+}
+
+// TestEveryDeliverableFaultConfirms is the positive half. It iterates the
+// catalogue rather than a hand-written list, so a new fault with no observable
+// fails here instead of shipping as a fault that cannot be confirmed.
+func TestEveryDeliverableFaultConfirms(t *testing.T) {
+ for _, kind := range deliverableKinds() {
+ t.Run(string(kind), func(t *testing.T) {
+ _, client := startResolver(t)
+
+ trap := trapFor(kind)
+ mustArm(t, client, trap)
+
+ observable, confirmed, err := client.Observe(trap)
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if !confirmed {
+ t.Fatalf("%s did not confirm: %s", kind, observable)
+ }
+
+ if observable == "" {
+ t.Error("confirmed with no observable, so the run record has " +
+ "nothing a reader can audit")
+ }
+ })
+ }
+}
+
+// TestInertResolverConfirmsNothing is the negative control, and it is the
+// control that matters most in this package.
+//
+// An inert trap arms the resolver with a fault it then does not apply, so every
+// observable has to disappear. Without this, a confirmation that read the armed
+// STATE back rather than the served answer would pass against a resolver doing
+// nothing at all — which is precisely the bug the PKI tier's inert-authority
+// control caught three separate times, each one a trap reading the trap record
+// instead of built state.
+func TestInertResolverConfirmsNothing(t *testing.T) {
+ for _, kind := range deliverableKinds() {
+ t.Run(string(kind), func(t *testing.T) {
+ _, client := startResolver(t)
+
+ trap := trapFor(kind)
+ trap.Inert = true
+ mustArm(t, client, trap)
+
+ observable, confirmed, err := client.Observe(trap)
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if confirmed {
+ t.Fatalf("%s confirmed against an INERT resolver: %s",
+ kind, observable)
+ }
+ })
+ }
+}
+
+// TestNothingArmedConfirmsNothing is the second half of the negative control.
+// Inert exercises the arming path with the mutation suppressed; this exercises
+// no arming at all, which is the state a run is in during its baseline.
+func TestNothingArmedConfirmsNothing(t *testing.T) {
+ for _, kind := range deliverableKinds() {
+ t.Run(string(kind), func(t *testing.T) {
+ _, client := startResolver(t)
+
+ observable, confirmed, err := client.Observe(trapFor(kind))
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if confirmed {
+ t.Fatalf(
+ "%s confirmed with nothing armed: %s",
+ kind,
+ observable,
+ )
+ }
+ })
+ }
+}
+
+// TestObservingAWrongFaultRefutes is the isolation control: each fault's
+// observable must reject the OTHER faults' answers.
+//
+// Two checks that both accept the same input alibi each other, so gutting
+// either leaves the suite green. Here the resolver really serves fault A while
+// khaos asks whether fault B is confirmed, for every ordered pair, and the
+// answer has to be no.
+func TestObservingAWrongFaultRefutes(t *testing.T) {
+ kinds := deliverableKinds()
+
+ for _, armedKind := range kinds {
+ for _, askedKind := range kinds {
+ if armedKind == askedKind {
+ continue
+ }
+
+ name := string(armedKind) + "-asked-as-" + string(askedKind)
+
+ t.Run(name, func(t *testing.T) {
+ _, client := startResolver(t)
+
+ mustArm(t, client, trapFor(armedKind))
+
+ observable, confirmed, err := client.Observe(trapFor(askedKind))
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if confirmed {
+ t.Fatalf(
+ "a resolver serving %s confirmed %s: %s",
+ armedKind, askedKind, observable,
+ )
+ }
+ })
+ }
+ }
+}
+
+// TestObserveRefutesWhenTheControlIsBroken proves the oracle is two-sided. A
+// resolver that has stopped answering altogether looks exactly like a confirmed
+// negative fault from one query, and reporting that as a finding would
+// manufacture a defect out of khaos's own breakage.
+func TestObserveRefutesWhenTheControlIsBroken(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ mustArm(t, client, trapFor(NXDomain))
+
+ if _, confirmed, err := client.Observe(trapFor(NXDomain)); err != nil ||
+ !confirmed {
+ t.Fatalf("precondition: NXDOMAIN should confirm first (err=%v)", err)
+ }
+
+ if err := resolver.Shutdown(); err != nil {
+ t.Fatalf("shutdown: %v", err)
+ }
+
+ observable, confirmed, err := client.Observe(trapFor(NXDomain))
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+
+ if confirmed {
+ t.Fatalf("confirmed a fault against a DEAD resolver: %s", observable)
+ }
+}
+
+func TestObserveRefusesAMaskedFault(t *testing.T) {
+ _, client := startResolver(t)
+
+ for _, kind := range []Kind{Refused, Timeout} {
+ t.Run(string(kind), func(t *testing.T) {
+ _, _, err := client.Observe(Trap{Kind: kind, Host: testHost})
+ if err == nil {
+ t.Fatal("observed a fault khaos refuses to deliver")
+ }
+
+ if !is(err, ErrMasked) {
+ t.Errorf("error %v, want %v", err, ErrMasked)
+ }
+ })
+ }
+}
+
+// TestControlSurfaceRefusesTheWrongNonce matters because the resolver shares a
+// network with the target. The nonce is the only thing stopping the target from
+// disarming the fault being measured on it.
+func TestControlSurfaceRefusesTheWrongNonce(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ mustArm(t, client, trapFor(NXDomain))
+
+ imposter := NewClient(
+ resolver.Addr(),
+ "http://"+resolver.ControlAddr(),
+ "wrong-nonce",
+ testControl,
+ )
+
+ if _, err := imposter.Disarm(t.Context()); err == nil {
+ t.Fatal("a wrong nonce disarmed the fault")
+ }
+
+ if _, confirmed, err := client.Observe(trapFor(NXDomain)); err != nil ||
+ !confirmed {
+ t.Errorf("the fault did not survive the refused disarm (err=%v)", err)
+ }
+
+ if denied := resolver.Stats().Denied; denied == 0 {
+ t.Error("a refused control request was not counted, so khaos has no " +
+ "evidence it happened")
+ }
+}
+
+// TestHealthNeedsNoNonce is deliberate: khaos has to be able to wait for the
+// container to come up before it has read the nonce back out of it.
+func TestHealthNeedsNoNonce(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ anonymous := NewClient(
+ resolver.Addr(),
+ "http://"+resolver.ControlAddr(),
+ "",
+ testControl,
+ )
+
+ if !anonymous.Healthy(t.Context()) {
+ t.Fatal("health needed a nonce")
+ }
+}
+
+// TestHealthLeaksNothingAboutTheFault checks the other side of that decision.
+// The target can reach this surface, so an unauthenticated endpoint must not
+// tell it which fault is armed.
+func TestHealthLeaksNothingAboutTheFault(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ mustArm(t, client, trapFor(WrongAddress))
+
+ response, err := http.Get("http://" + resolver.ControlAddr() + PathHealthz)
+ if err != nil {
+ t.Fatalf("get health: %v", err)
+ }
+ defer func() { _ = response.Body.Close() }()
+
+ body := make([]byte, controlBodyLimit)
+ n, _ := response.Body.Read(body)
+ text := string(body[:n])
+
+ for _, leak := range []string{
+ string(WrongAddress), DefaultWrongAddress, testNonce,
+ } {
+ if contains(text, leak) {
+ t.Errorf("health leaked %q: %s", leak, text)
+ }
+ }
+}
+
+// TestDisarmRestoresTheHonestAnswer is the revert path. The recovery phase is
+// measured through a WORKING resolver, so a disarm that left the fault in place
+// would score the recovery window against a target still under fault.
+func TestDisarmRestoresTheHonestAnswer(t *testing.T) {
+ _, client := startResolver(t)
+
+ mustArm(t, client, trapFor(NXDomain))
+
+ if _, err := client.Disarm(t.Context()); err != nil {
+ t.Fatalf("disarm: %v", err)
+ }
+
+ got := ask(t, client, testHost, udpNet)
+ if !got.resolved() {
+ t.Fatalf("%s did not resolve after disarm: rcode=%s err=%v",
+ testHost, rcodeName(got.rcode), got.err)
+ }
+}
+
+// TestSlowObservationIsADifferential is the proxy tier's lesson applied here. An
+// absolute threshold confirms a delay for free against any resolver that is
+// simply slow, so the observable has to be a DIFFERENCE from the same
+// resolver's own control.
+func TestSlowObservationIsADifferential(t *testing.T) {
+ trap := Trap{Kind: SlowResponse, Host: testHost, Delay: time.Second}
+
+ answered := func(elapsed time.Duration) answer {
+ record, err := dns.NewRR("x.test. 600 IN A " + testAddress)
+ if err != nil {
+ t.Fatalf("build record: %v", err)
+ }
+ return answer{records: []dns.RR{record}, elapsed: elapsed}
+ }
+
+ bothSlow := answered(4 * time.Second)
+ control := answered(4 * time.Second)
+
+ if _, confirmed, _ := slowObservation(
+ trap, bothSlow, control, testAddress,
+ ); confirmed {
+ t.Error("confirmed a delay where the control was equally slow, which " +
+ "is an absolute threshold wearing a differential's name")
+ }
+
+ fast := answered(10 * time.Millisecond)
+
+ if _, confirmed, _ := slowObservation(
+ trap, answered(4*time.Second), fast, testAddress,
+ ); !confirmed {
+ t.Error("did not confirm a delay the control did not share")
+ }
+}
+
+// TestSlowObservationRefusesAQueryItNeverGotAnAnswerTo is the control for the
+// blocker this function shipped with.
+//
+// `ask` records a failure as an elapsed time plus an error, and probeTimeout is
+// 10s, so a single dropped UDP datagram on khaos's own confirmation query looked
+// exactly like a 10-second delay. The function returned CONFIRMED and worded it
+// as "answering <name> after 10s" — khaos reporting a delay it never received,
+// and then measuring a whole fault window on the strength of it.
+func TestSlowObservationRefusesAQueryItNeverGotAnAnswerTo(t *testing.T) {
+ trap := Trap{Kind: SlowResponse, Host: testHost, Delay: time.Second}
+ fast := answer{elapsed: time.Millisecond}
+
+ cases := []struct {
+ name string
+ faulted answer
+ }{
+ {
+ name: "a timed-out query",
+ faulted: answer{
+ elapsed: 10 * time.Second,
+ err: errors.New("i/o timeout"),
+ },
+ },
+ {
+ name: "a slow NXDOMAIN, which is a different fault",
+ faulted: answer{
+ elapsed: 4 * time.Second,
+ rcode: dns.RcodeNameError,
+ },
+ },
+ {
+ name: "a slow empty answer",
+ faulted: answer{elapsed: 4 * time.Second},
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ observable, confirmed, err := slowObservation(
+ trap, test.faulted, fast, testAddress,
+ )
+ if err != nil {
+ t.Fatalf("observe: %v", err)
+ }
+ if confirmed {
+ t.Fatalf("confirmed a delay from something that was not an "+
+ "answer: %s", observable)
+ }
+ })
+ }
+}
+
+// TestSlowObservationRefusesAnAnswerThatAlsoMoved keeps the fault to ONE
+// property. The kind promises the honest answer late, so a late redirect is two
+// faults reported as one.
+func TestSlowObservationRefusesAnAnswerThatAlsoMoved(t *testing.T) {
+ record, err := dns.NewRR("x.test. 600 IN A 198.51.100.66")
+ if err != nil {
+ t.Fatalf("build record: %v", err)
+ }
+
+ moved := answer{records: []dns.RR{record}, elapsed: 4 * time.Second}
+
+ observable, confirmed, _ := slowObservation(
+ Trap{Kind: SlowResponse, Host: testHost},
+ moved,
+ answer{elapsed: time.Millisecond},
+ testAddress,
+ )
+
+ if confirmed {
+ t.Fatalf(
+ "confirmed a delay on an answer that also changed: %s",
+ observable,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go
new file mode 100644
index 00000000..34d925bf
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/records.go
@@ -0,0 +1,209 @@
+// ©AngelaMos | 2026
+// records.go
+
+package dnsrv
+
+import (
+ "errors"
+ "fmt"
+ "net"
+
+ "github.com/miekg/dns"
+)
+
+const (
+ // chainLabel and loopLabel prefix the hops khaos synthesizes. They are
+ // deliberately readable rather than random, because the whole point of the
+ // CNAME faults is that an operator reading the target's logs can see the
+ // name it actually connected to was not the name it asked for.
+ chainLabel = "hop"
+ loopLabel = "cycle"
+
+ // chainExit is the zone the chain ends OUTSIDE. `.invalid` is reserved by
+ // RFC 2606 and can never resolve publicly, so a target that follows the
+ // chain and connects anyway is observably following it rather than quietly
+ // reaching a real service.
+ chainExit = "khaos-elsewhere.invalid."
+
+ errRecord = "build a %s record for %s: %w"
+)
+
+// ErrNoRecord is returned when a question asks for a type khaos's synthesizing
+// faults do not produce. It is not a failure: a target asking for SRV while
+// khaos is faulting an address record should be forwarded, not answered with
+// something invented.
+var ErrNoRecord = errors.New(
+ "this fault does not synthesize an answer for that query type",
+)
+
+// addressAnswer is WrongAddress: the name resolves, to somewhere nobody
+// approved. It answers A for an IPv4 address and AAAA for an IPv6 one, and it
+// answers the question that was ASKED rather than both, because a target that
+// asked only for A and received a AAAA has been sent something it never
+// requested and the fault would be about khaos rather than about the address.
+func addressAnswer(question dns.Question, trap Trap) ([]dns.RR, error) {
+ address := trap.address()
+
+ parsed := net.ParseIP(address)
+ if parsed == nil {
+ return nil, fmt.Errorf(errBadAddress, ErrBadAddress, address)
+ }
+
+ isV4 := parsed.To4() != nil
+
+ switch {
+ case question.Qtype == dns.TypeA && isV4:
+ return newRR(dns.TypeA, question.Name, trap.ttl(), address)
+ case question.Qtype == dns.TypeAAAA && !isV4:
+ return newRR(dns.TypeAAAA, question.Name, trap.ttl(), address)
+ default:
+ // The family the target asked for is not the family of the address khaos
+ // was given, so there is no honest record to make. An empty NOERROR is
+ // the right answer: the name exists and has no record of that type,
+ // which is exactly true.
+ return nil, nil
+ }
+}
+
+// chainAnswer is CNAMEChain: a multi-hop CNAME whose last hop lands in a
+// different zone from the one the target asked about.
+//
+// It is built as a `file`-zone-shaped full chain rather than a single CNAME,
+// because research 05 section 8.2 measured that a resolver answering one CNAME
+// at a time is a different thing to test than one answering the whole chain, and
+// the whole chain is what a real authoritative server returns.
+func chainAnswer(question dns.Question, trap Trap) ([]dns.RR, error) {
+ if question.Qtype != dns.TypeA && question.Qtype != dns.TypeAAAA {
+ return nil, nil
+ }
+
+ ttl := trap.ttl()
+ name := dns.CanonicalName(question.Name)
+
+ records := make([]dns.RR, 0, chainHops)
+
+ // chainHops counts CNAME records, so the bound is inclusive.
+ for hop := 1; hop <= chainHops; hop++ {
+ next := fmt.Sprintf("%s%d.%s", chainLabel, hop, chainExit)
+ rr, err := newRR(dns.TypeCNAME, name, ttl, next)
+ if err != nil {
+ return nil, err
+ }
+ records = append(records, rr...)
+ name = next
+ }
+
+ final, err := addressAnswer(dns.Question{
+ Name: name,
+ Qtype: question.Qtype,
+ }, trap)
+ if err != nil {
+ return nil, err
+ }
+
+ return append(records, final...), nil
+}
+
+// loopAnswer is CNAMELoop: two CNAMEs pointing at each other, plus the entry
+// hop. The server answering it does not hang, which the research measured; what
+// is under test is the CLIENT's loop handling, and a client with none spins or
+// exhausts its own retry budget.
+func loopAnswer(question dns.Question, trap Trap) ([]dns.RR, error) {
+ if question.Qtype != dns.TypeA && question.Qtype != dns.TypeAAAA {
+ return nil, nil
+ }
+
+ ttl := trap.ttl()
+ asked := dns.CanonicalName(question.Name)
+ first := fmt.Sprintf("%s1.%s", loopLabel, chainExit)
+ second := fmt.Sprintf("%s2.%s", loopLabel, chainExit)
+
+ hops := [][2]string{
+ {asked, first},
+ {first, second},
+ {second, first},
+ }
+
+ records := make([]dns.RR, 0, len(hops))
+ for _, hop := range hops {
+ rr, err := newRR(dns.TypeCNAME, hop[0], ttl, hop[1])
+ if err != nil {
+ return nil, err
+ }
+ records = append(records, rr...)
+ }
+
+ return records, nil
+}
+
+// rettl rewrites every TTL in an answer, which is how the two TTL faults corrupt
+// exactly one property of a real answer rather than inventing one.
+//
+// It rewrites the authority and additional sections too. A caching resolver
+// keyed off an SOA's TTL in the authority section would otherwise honour a
+// lifetime khaos did not set, and the fault would be partially applied with
+// nothing saying so.
+func rettl(msg *dns.Msg, ttl uint32) *dns.Msg {
+ if msg == nil {
+ return nil
+ }
+
+ for _, section := range [][]dns.RR{msg.Answer, msg.Ns, msg.Extra} {
+ for _, rr := range section {
+ if rr == nil || rr.Header().Rrtype == dns.TypeOPT {
+ continue
+ }
+ rr.Header().Ttl = ttl
+ }
+ }
+
+ return msg
+}
+
+// validAddress refuses a WrongAddress magnitude that is not an address, before
+// the trap reaches the journal. The fault's whole observable is the address the
+// target connected to, so a malformed one would arm, answer an empty NOERROR
+// and be reported as a target that resolved nothing.
+func validAddress(address string) error {
+ if net.ParseIP(address) == nil {
+ return fmt.Errorf(errBadAddress, ErrBadAddress, address)
+ }
+ return nil
+}
+
+func newRR(
+ rrtype uint16,
+ name string,
+ ttl uint32,
+ value string,
+) ([]dns.RR, error) {
+ text := fmt.Sprintf(
+ "%s %d IN %s %s",
+ dns.Fqdn(name),
+ ttl,
+ dns.TypeToString[rrtype],
+ dns.Fqdn(value),
+ )
+
+ if rrtype == dns.TypeA || rrtype == dns.TypeAAAA {
+ text = fmt.Sprintf(
+ "%s %d IN %s %s",
+ dns.Fqdn(name),
+ ttl,
+ dns.TypeToString[rrtype],
+ value,
+ )
+ }
+
+ rr, err := dns.NewRR(text)
+ if err != nil {
+ return nil, fmt.Errorf(
+ errRecord,
+ dns.TypeToString[rrtype],
+ name,
+ err,
+ )
+ }
+
+ return []dns.RR{rr}, nil
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go
new file mode 100644
index 00000000..ba54f56e
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver.go
@@ -0,0 +1,807 @@
+// ©AngelaMos | 2026
+// resolver.go
+
+package dnsrv
+
+import (
+ "errors"
+ "fmt"
+ "maps"
+ "net"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/miekg/dns"
+)
+
+const (
+ // DefaultPort is the port the resolver listens on INSIDE its container. It
+ // is 53 and it has to be, because `--dns` carries an address and no port:
+ // the CLI calls a port malformed and the daemon rejects it at container
+ // start with `ParseAddr(...): unexpected character`, after create has
+ // already succeeded. resolv.conf has no port field either.
+ //
+ // This is also why the resolver is a container and not a goroutine in khaos.
+ // An unprivileged process gets `bind: permission denied` on port 53 at every
+ // address, loopback included, while high ports on the same addresses bind
+ // fine. Inside a container the process is root in its own namespace and
+ // NET_BIND_SERVICE is in Docker's default set, so no host port is published
+ // and khaos still holds no capability.
+ DefaultPort = 53
+
+ // DefaultUpstream is the resolver container's OWN embedded resolver.
+ //
+ // Forwarding there rather than to a public address is what keeps khaos
+ // keyless and offline-capable: the resolver inherits exactly the resolution
+ // the host has, with no configuration and no third party. There is no loop,
+ // because khaos's resolver container is never created with `--dns` pointing
+ // at itself, and khaos refuses to start one that is.
+ DefaultUpstream = "127.0.0.11:53"
+
+ // forwardTimeout is under the embedded resolver's own 4s `extIOTimeout`, so
+ // a slow upstream surfaces here as a forward error khaos can read rather
+ // than as the target's query dying one layer up where nothing records it.
+ forwardTimeout = 3 * time.Second
+
+ udpNet = "udp"
+ tcpNet = "tcp"
+
+ // bindAttempts only ever matters for an ephemeral port: the kernel can hand
+ // out a UDP port that something else already holds in the TCP space.
+ bindAttempts = 8
+)
+
+const (
+ errBindFailed = "bind the chaos resolver on %s: %w"
+ errUpstreamSet = "%w: the resolver's upstream is %s, which is itself"
+ errAlreadyUp = "%w: this resolver is already serving"
+ errNotUp = "%w: this resolver is not serving"
+ errEmptyZone = "%w: a resolver with an empty zone has nothing it could fault"
+ errZoneAddress = "%w: the zone maps %s to %q"
+ errControlName = "%w: %q is not in the zone, so a confirmation could never use it as a control"
+ errFaultedHost = "%w: %s is the confirmation control and cannot also be the faulted name"
+ errOutsideZone = "%w: %s is not a name this resolver is authoritative for (zone: %s)"
+ errNoZoneTarget = "%w: this resolver was given no confirmation control name"
+)
+
+var ErrUpstreamLoop = errors.New(
+ "a resolver whose upstream is itself would answer every query with itself",
+)
+
+var ErrAlreadyServing = errors.New("that resolver is already serving")
+
+var ErrNotServing = errors.New("that resolver is not serving")
+
+var ErrEmptyZone = errors.New("a chaos resolver needs a zone to be wrong about")
+
+var ErrZoneAddress = errors.New("a zone entry must map a name to an IP address")
+
+var ErrNoControl = errors.New(
+ "a chaos resolver needs a control name its faults never touch",
+)
+
+var ErrControlIsHost = errors.New(
+ "the confirmation control cannot be the name under fault",
+)
+
+var ErrOutsideZone = errors.New(
+ "this resolver is not authoritative for that name",
+)
+
+// Config is everything the resolver needs to exist. It is deliberately small:
+// the FAULT is not here, because a resolver is armed in band after it is
+// running, for a reason that is measured rather than stylistic. Docker refuses
+// `--ip` on a network whose subnet it chose, so khaos cannot pin the resolver's
+// address, so re-creating the container to arm it would move the address the
+// target was created to dial and the target would be talking to nothing.
+type Config struct {
+ // Listen is the address:port to bind. Empty binds every address on
+ // DefaultPort, which is correct inside a container and is why khaos never
+ // binds a wildcard on the host.
+ Listen string
+
+ // Upstream answers everything OUTSIDE the zone. A resolver that NXDOMAINed
+ // unrelated names would break the target for reasons the experiment never
+ // claimed, and the measured property that makes this safe is that the
+ // target's own embedded resolver answers container names locally and never
+ // forwards them here at all.
+ Upstream string
+
+ // Nonce authorises the control surface. khaos generates it, passes it at
+ // container creation, and reads it back out of the container rather than
+ // remembering what it chose, the same way the range reads its ports back
+ // from the containers that will dial them.
+ Nonce string
+
+ // Zone is the set of names khaos is AUTHORITATIVE for, mapped to the honest
+ // address each one really has.
+ //
+ // This exists because of a measured constraint rather than for convenience.
+ // A container name or network alias can never be faulted: the target's own
+ // embedded resolver answers those out of its own table and never forwards
+ // them to an upstream, so khaos would never see the query. The faulted name
+ // therefore has to be one the embedded resolver cannot answer — and then
+ // NOBODY would answer it, and the baseline would measure a target whose
+ // dependency never resolved. So khaos serves it honestly itself, and the
+ // fault is a deviation from khaos's own correct answer.
+ Zone map[string]string
+
+ // Control is the name in Zone that no fault ever touches. It is the
+ // confirmation oracle's positive control: khaos serves it from the same
+ // table, over the same transports, through the same handler, so a control
+ // that stops resolving means khaos's resolver is broken rather than lying.
+ Control string
+}
+
+// Resolver is khaos's authoritative chaos resolver. One instance serves one
+// network, is armed and disarmed in band, and attributes every query it answers
+// to the address it came from.
+type Resolver struct {
+ cfg Config
+ zone map[string]string
+
+ mu sync.Mutex
+ trap Trap
+ servers []*dns.Server
+ arrival map[string]int64
+ queries map[string]int64
+ faulted int64
+ honest int64
+ forward int64
+ errs int64
+ denied int64
+
+ control *http.Server
+ controlAddr string
+ controls int64
+}
+
+// New builds a resolver without binding anything. Nothing listens until Serve,
+// which is what makes the whole package unit-testable in process on a high port
+// with no container and no privilege.
+func New(cfg Config) (*Resolver, error) {
+ if cfg.Listen == "" {
+ cfg.Listen = fmt.Sprintf(":%d", DefaultPort)
+ }
+ if cfg.Upstream == "" {
+ cfg.Upstream = DefaultUpstream
+ }
+
+ if sameAddr(cfg.Listen, cfg.Upstream) {
+ return nil, fmt.Errorf(errUpstreamSet, ErrUpstreamLoop, cfg.Upstream)
+ }
+
+ if len(cfg.Zone) == 0 {
+ return nil, fmt.Errorf(errEmptyZone, ErrEmptyZone)
+ }
+
+ zone := make(map[string]string, len(cfg.Zone))
+ for name, address := range cfg.Zone {
+ if net.ParseIP(address) == nil {
+ return nil, fmt.Errorf(
+ errZoneAddress,
+ ErrZoneAddress,
+ name,
+ address,
+ )
+ }
+ zone[dns.CanonicalName(name)] = address
+ }
+
+ if cfg.Control == "" {
+ return nil, fmt.Errorf(errNoZoneTarget, ErrNoControl)
+ }
+
+ if _, ok := zone[dns.CanonicalName(cfg.Control)]; !ok {
+ return nil, fmt.Errorf(errControlName, ErrNoControl, cfg.Control)
+ }
+
+ return &Resolver{
+ cfg: cfg,
+ zone: zone,
+ arrival: map[string]int64{},
+ queries: map[string]int64{},
+ }, nil
+}
+
+// Zone reports the names this resolver is authoritative for, so khaos can state
+// them in a plan and a gate can census them.
+func (r *Resolver) Zone() map[string]string {
+ return maps.Clone(r.zone)
+}
+
+// Serve binds UDP and TCP and returns once both are accepting. Both transports
+// are mandatory and not a nicety: a client that sees the truncation bit re-asks
+// over TCP, the embedded resolver forwards on the transport the client used, and
+// a UDP-only resolver would turn every truncation experiment into a timeout.
+func (r *Resolver) Serve() error {
+ mux := dns.NewServeMux()
+ mux.HandleFunc(".", r.handle)
+
+ r.mu.Lock()
+ if len(r.servers) > 0 {
+ r.mu.Unlock()
+ return fmt.Errorf(errAlreadyUp, ErrAlreadyServing)
+ }
+ r.mu.Unlock()
+
+ conn, listener, err := bindBoth(r.cfg.Listen)
+ if err != nil {
+ return err
+ }
+
+ servers := []*dns.Server{
+ {PacketConn: conn, Net: udpNet, Handler: mux},
+ {Listener: listener, Net: tcpNet, Handler: mux},
+ }
+
+ r.mu.Lock()
+ r.servers = servers
+ r.mu.Unlock()
+
+ // Each goroutine serves the LOCAL it was handed, never a field it reads back
+ // off the struct. `internal/pki` shipped three data races doing the
+ // opposite: its serve goroutines dereferenced a field that mutex-held
+ // shutdown nils, and the plain test suite was green for both new packages.
+ ready := make(chan error, len(servers))
+ for _, server := range servers {
+ srv := server
+ srv.NotifyStartedFunc = func() { ready <- nil }
+ go func() {
+ if err := srv.ActivateAndServe(); err != nil {
+ ready <- fmt.Errorf(errBindFailed, srv.Net, err)
+ }
+ }()
+ }
+
+ var failures []error
+ for range servers {
+ if err := <-ready; err != nil {
+ failures = append(failures, err)
+ }
+ }
+
+ if len(failures) > 0 {
+ _ = r.Shutdown()
+ return errors.Join(failures...)
+ }
+
+ return nil
+}
+
+// bindBoth puts UDP and TCP on the SAME port, and that is a correctness
+// requirement rather than tidiness.
+//
+// Handing `dns.Server` an `Addr` with port 0 and letting each transport bind
+// itself gives them DIFFERENT ports, because UDP and TCP are separate port
+// spaces and the kernel picks independently. A resolver like that answers over
+// UDP and is unreachable over TCP, so every client that sees the truncation bit
+// and re-asks per RFC 5966 gets `connection refused` — the truncation fault
+// becomes an outage, and the no-TCP variant confirms itself for free against a
+// port that was never listening. A test caught it; at the real :53 both happen
+// to land on the same number and nothing would have.
+func bindBoth(listen string) (net.PacketConn, net.Listener, error) {
+ udpAddr, err := net.ResolveUDPAddr(udpNet, listen)
+ if err != nil {
+ return nil, nil, fmt.Errorf(errBindFailed, udpNet, err)
+ }
+
+ var last error
+
+ for attempt := 0; attempt < bindAttempts; attempt++ {
+ conn, err := net.ListenUDP(udpNet, udpAddr)
+ if err != nil {
+ return nil, nil, fmt.Errorf(errBindFailed, udpNet, err)
+ }
+
+ listener, err := net.Listen(tcpNet, conn.LocalAddr().String())
+ if err == nil {
+ return conn, listener, nil
+ }
+
+ _ = conn.Close()
+ last = fmt.Errorf(errBindFailed, tcpNet, err)
+
+ // An explicit port cannot be retried into success, so the only case
+ // worth another attempt is an ephemeral one the kernel chose for UDP
+ // that something else already holds in the TCP space.
+ if udpAddr.Port != 0 {
+ break
+ }
+ }
+
+ return nil, nil, last
+}
+
+// Addr reports where the resolver actually bound, which matters because a test
+// asks for port 0 and has to find out what it got.
+func (r *Resolver) Addr() string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ for _, srv := range r.servers {
+ if srv.Net == udpNet && srv.PacketConn != nil {
+ return srv.PacketConn.LocalAddr().String()
+ }
+ }
+
+ return r.cfg.Listen
+}
+
+// Shutdown closes both DNS transports and the control surface. The control
+// surface goes down too, and unconditionally, because a resolver whose DNS is
+// gone but whose /arm still answers is one khaos could arm and then confirm
+// against nothing.
+func (r *Resolver) Shutdown() error {
+ r.mu.Lock()
+ servers := r.servers
+ r.servers = nil
+ r.mu.Unlock()
+
+ failures := []error{r.shutdownControl()}
+
+ if len(servers) == 0 {
+ failures = append(failures, fmt.Errorf(errNotUp, ErrNotServing))
+ }
+
+ for _, srv := range servers {
+ if err := srv.Shutdown(); err != nil {
+ failures = append(failures, err)
+ }
+ }
+
+ return errors.Join(failures...)
+}
+
+// Arm installs a fault. An inert trap is still installed, because the negative
+// control has to travel the same path as a live one: a control that skipped
+// arming would prove only that the resolver was never asked.
+func (r *Resolver) Arm(trap Trap) error {
+ if err := trap.Validate(); err != nil {
+ return err
+ }
+
+ if trap.Kind != PassThrough {
+ if err := r.inZone(trap.Host); err != nil {
+ return err
+ }
+
+ if dns.CanonicalName(trap.Host) == dns.CanonicalName(r.cfg.Control) {
+ return fmt.Errorf(errFaultedHost, ErrControlIsHost, trap.Host)
+ }
+ }
+
+ r.mu.Lock()
+ r.trap = trap
+ r.mu.Unlock()
+
+ return nil
+}
+
+// inZone refuses a fault aimed at a name this resolver does not answer.
+//
+// Without it the most likely operator mistake is also the most silent one: a
+// trap on a name khaos is not authoritative for arms cleanly, every query for
+// it is forwarded untouched, the target resolves it perfectly, and the run
+// reports a clean survival against a fault that was never applied to anything.
+func (r *Resolver) inZone(host string) error {
+ if _, ok := r.zone[dns.CanonicalName(host)]; ok {
+ return nil
+ }
+
+ names := make([]string, 0, len(r.zone))
+ for name := range r.zone {
+ names = append(names, name)
+ }
+
+ return fmt.Errorf(
+ errOutsideZone,
+ ErrOutsideZone,
+ host,
+ joinWith(sorted(names), ", "),
+ )
+}
+
+func (r *Resolver) Disarm() {
+ r.mu.Lock()
+ r.trap = Trap{}
+ r.mu.Unlock()
+}
+
+func (r *Resolver) armed() Trap {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.trap
+}
+
+// Arrivals reports how many queries came from one address. It is attributed by
+// SOURCE rather than counted in total, and that is the difference between a
+// proof of arrival and a number that is always non-zero.
+//
+// khaos queries its own resolver to confirm the fault, from the host, so its
+// source is the bridge gateway. The target's queries are forwarded by the
+// embedded resolver from the TARGET's own network namespace, so their source is
+// the target's container address. Counting both together would make "the target
+// used khaos's resolver" true on every run, including the runs where the target
+// never used it at all.
+func (r *Resolver) Arrivals(from string) int64 {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.arrival[from]
+}
+
+// Stats is what the control surface reports and what khaos reads to decide
+// whether a run is interpretable at all.
+type Stats struct {
+ Armed Kind `json:"armed"`
+ Host string `json:"host"`
+ Inert bool `json:"inert"`
+ Faulted int64 `json:"faulted"`
+ Honest int64 `json:"honest"`
+ Forward int64 `json:"forwarded"`
+ Errors int64 `json:"forward_errors"`
+ Control int64 `json:"control_requests"`
+ Denied int64 `json:"control_denied"`
+ Arrivals map[string]int64 `json:"arrivals"`
+
+ // Zone is the honest answer khaos serves for each name it owns, read back
+ // rather than remembered. A redirect's observable needs it to name the
+ // address the fault replaced rather than restating the one it served.
+ Zone map[string]string `json:"zone"`
+
+ // Queries counts only the queries khaos was AUTHORITATIVE for, keyed by
+ // source address and zone name.
+ //
+ // Arrivals alone cannot carry the proof-of-arrival control, and that is the
+ // whole reason this exists. A target resolving anything else through khaos's
+ // resolver -- a public name, another service -- makes its arrival count
+ // non-zero without ever asking about the dependency under fault, so the
+ // guard would pass for a target that never experienced the experiment. It
+ // would also break the TTL faults outright, whose whole measurement is HOW
+ // MANY times the faulted name was asked for.
+ Queries map[string]int64 `json:"queries"`
+}
+
+func (r *Resolver) Stats() Stats {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ return Stats{
+ Armed: r.trap.Kind,
+ Host: r.trap.Host,
+ Inert: r.trap.Inert,
+ Faulted: r.faulted,
+ Honest: r.honest,
+ Forward: r.forward,
+ Errors: r.errs,
+ Control: r.controls,
+ Denied: r.denied,
+ Arrivals: maps.Clone(r.arrival),
+ Queries: maps.Clone(r.queries),
+ Zone: maps.Clone(r.zone),
+ }
+}
+
+func (r *Resolver) handle(w dns.ResponseWriter, req *dns.Msg) {
+ if req == nil || len(req.Question) == 0 {
+ return
+ }
+
+ question := req.Question[0]
+
+ r.record(sourceAddr(w.RemoteAddr()))
+
+ name, authoritative := r.lookup(question.Name)
+ if !authoritative {
+ r.reply(w, req, r.forwardQuery(req, w))
+ return
+ }
+
+ r.recordQuery(sourceAddr(w.RemoteAddr()), name)
+
+ trap := r.armed()
+ if !r.applies(trap, name) {
+ r.countHonest()
+ r.reply(w, req, r.honestAnswer(req, question, name))
+ return
+ }
+
+ r.countFaulted()
+ r.reply(w, req, r.serveFault(trap, req, question, name, w))
+}
+
+// lookup resolves a query name to the zone entry it belongs to, and reports
+// whether khaos is authoritative for it at all.
+//
+// Matching is on the WHOLE name or on the name plus a suffix, because a
+// container's resolv.conf carries a forced `search` domain. A target asking for
+// `dependency.khaos.test` was measured arriving as both
+// `dependency.khaos.test.` and `dependency.khaos.test.mynetworksettings.com.`,
+// and that is not cosmetic: under a negative fault the absolute attempt fails
+// and the client RETRIES with the search suffix, so a resolver that faulted only
+// the exact name would answer the retry honestly and defeat its own fault.
+func (r *Resolver) lookup(name string) (string, bool) {
+ asked := dns.CanonicalName(name)
+
+ if _, ok := r.zone[asked]; ok {
+ return asked, true
+ }
+
+ for entry := range r.zone {
+ if strings.HasPrefix(asked, strings.TrimSuffix(entry, ".")+".") {
+ return entry, true
+ }
+ }
+
+ return "", false
+}
+
+func (r *Resolver) applies(trap Trap, zoneName string) bool {
+ if trap.Kind == PassThrough || trap.Inert || trap.Host == "" {
+ return false
+ }
+
+ return dns.CanonicalName(trap.Host) == zoneName
+}
+
+// honestAnswer is what khaos serves for a zone name with nothing armed against
+// it, and what it serves for the control name always. Every baseline is
+// measured through this, so it has to be indistinguishable from a correct
+// resolver's answer.
+func (r *Resolver) honestAnswer(
+ req *dns.Msg,
+ question dns.Question,
+ zoneName string,
+) *dns.Msg {
+ return r.synthesize(req, question, Trap{
+ Host: zoneName,
+ Address: r.zone[zoneName],
+ }, addressAnswer)
+}
+
+// serveFault produces the faulted answer. Every kind acts on khaos's OWN honest
+// answer for the name, which is the design rule from the PKI tier restated:
+// every trap corrupts exactly ONE property. A fault that synthesized its own
+// address as a side effect of being slow would be two faults reported as one.
+func (r *Resolver) serveFault(
+ trap Trap,
+ req *dns.Msg,
+ question dns.Question,
+ zoneName string,
+ w dns.ResponseWriter,
+) *dns.Msg {
+ honest := Trap{Host: zoneName, Address: r.zone[zoneName]}
+
+ switch trap.Kind {
+ case NXDomain:
+ return authoritativeRcode(req, dns.RcodeNameError)
+
+ case ServFail:
+ return authoritativeRcode(req, dns.RcodeServerFailure)
+
+ case WrongAddress:
+ return r.synthesize(req, question, Trap{
+ Host: zoneName,
+ Address: trap.address(),
+ }, addressAnswer)
+
+ case CNAMEChain:
+ // The chain ends at the trap's address, not the honest one, and that is
+ // what makes it a finding rather than decoration. Research 05 section 10
+ // states the fault precisely: if name matching happens on the name that
+ // was ASKED rather than on the target the connection goes to, a CNAME
+ // out of the allowlisted zone passes the check while the traffic goes
+ // somewhere else. Ending the chain at the honest address would test the
+ // indirection and change nothing about where the traffic went, so there
+ // would be nothing for a control to have failed to notice.
+ return r.synthesize(req, question, Trap{
+ Host: zoneName,
+ Address: trap.address(),
+ }, chainAnswer)
+
+ case CNAMELoop:
+ return r.synthesize(req, question, honest, loopAnswer)
+
+ case SlowResponse:
+ time.Sleep(trap.delay())
+ return r.synthesize(req, question, honest, addressAnswer)
+
+ case Truncation:
+ // Truncated with nothing in it over UDP, complete over TCP: the shape a
+ // correct truncating server has. Emptying the UDP answer is what makes
+ // the fault about the RETRY rather than about the bit, because a client
+ // that ignores `tc` and keeps the records it was given has not been
+ // tested by a truncated message that was complete anyway.
+ reply := r.synthesize(req, question, honest, addressAnswer)
+ if protoOf(w) == udpNet {
+ reply.Truncated = true
+ reply.Answer = nil
+ }
+ return reply
+
+ case TruncationNoTCP:
+ // Truncated and empty on BOTH transports, so the client exhausts UDP and
+ // TCP and still has nothing. This is research 05 row 9's actual security
+ // question — whether DNS over TCP is permitted at all — rather than the
+ // fallback question above.
+ reply := r.synthesize(req, question, honest, addressAnswer)
+ reply.Truncated = true
+ reply.Answer = nil
+ return reply
+
+ case TTLZero, TTLLong:
+ return rettl(
+ r.synthesize(req, question, honest, addressAnswer),
+ trap.ttl(),
+ )
+
+ case Refused, Timeout, PassThrough:
+ // Unreachable: Validate refuses both masked kinds before a trap can be
+ // armed, and applies() excludes PassThrough. SERVFAIL rather than a
+ // silent empty reply means a future kind that forgets its branch is
+ // observably broken instead of looking like a target that resolved
+ // nothing.
+ return new(dns.Msg).SetRcode(req, dns.RcodeServerFailure)
+
+ default:
+ return new(dns.Msg).SetRcode(req, dns.RcodeServerFailure)
+ }
+}
+
+// authoritativeRcode is an rcode khaos is answering ON PURPOSE, for a name it
+// owns. Every other SERVFAIL in this file is khaos failing at something and is
+// left NON-authoritative, which is what makes the two distinguishable from
+// outside: deleting the ServFail arm used to fall through to `default`, serve
+// the same rcode, and leave every test and the whole gate green.
+//
+// It is also the protocol-correct answer. khaos IS the authority for its zone,
+// so an NXDOMAIN or a SERVFAIL it decides on carries AA, and one it stumbles
+// into does not.
+func authoritativeRcode(req *dns.Msg, rcode int) *dns.Msg {
+ reply := new(dns.Msg).SetRcode(req, rcode)
+ reply.Authoritative = true
+
+ return reply
+}
+
+func (r *Resolver) synthesize(
+ req *dns.Msg,
+ question dns.Question,
+ trap Trap,
+ build func(dns.Question, Trap) ([]dns.RR, error),
+) *dns.Msg {
+ reply := new(dns.Msg)
+ reply.SetReply(req)
+ reply.Authoritative = true
+
+ records, err := build(question, trap)
+ if err != nil {
+ r.countError()
+ return new(dns.Msg).SetRcode(req, dns.RcodeServerFailure)
+ }
+
+ reply.Answer = records
+
+ return reply
+}
+
+func (r *Resolver) reply(w dns.ResponseWriter, req, msg *dns.Msg) {
+ if msg == nil {
+ msg = new(dns.Msg).SetRcode(req, dns.RcodeServerFailure)
+ }
+
+ if err := w.WriteMsg(msg); err != nil {
+ r.countError()
+ }
+}
+
+func (r *Resolver) forwardQuery(req *dns.Msg, w dns.ResponseWriter) *dns.Msg {
+ client := &dns.Client{Net: protoOf(w), Timeout: forwardTimeout}
+
+ resp, _, err := client.Exchange(req.Copy(), r.cfg.Upstream)
+ if err != nil || resp == nil {
+ r.countError()
+ return new(dns.Msg).SetRcode(req, dns.RcodeServerFailure)
+ }
+
+ r.countForward()
+ resp.Id = req.Id
+
+ return resp
+}
+
+func (r *Resolver) record(source string) {
+ r.mu.Lock()
+ r.arrival[source]++
+ r.mu.Unlock()
+}
+
+// recordQuery counts a query for a name khaos answers authoritatively, keyed by
+// source AND name, because both predicates built on it need the pair.
+func (r *Resolver) recordQuery(source, zoneName string) {
+ r.mu.Lock()
+ r.queries[QueryKey(source, zoneName)]++
+ r.mu.Unlock()
+}
+
+// QueryKey is the one place the composite key is spelled, so the resolver
+// counting and khaos reading cannot disagree about its shape.
+func QueryKey(source, zoneName string) string {
+ return source + " " + dns.CanonicalName(zoneName)
+}
+
+// Queries reports how many times one source asked about one name khaos is
+// authoritative for.
+func (r *Resolver) Queries(source, zoneName string) int64 {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.queries[QueryKey(source, zoneName)]
+}
+
+func (r *Resolver) countFaulted() {
+ r.mu.Lock()
+ r.faulted++
+ r.mu.Unlock()
+}
+
+func (r *Resolver) countHonest() {
+ r.mu.Lock()
+ r.honest++
+ r.mu.Unlock()
+}
+
+func (r *Resolver) countForward() {
+ r.mu.Lock()
+ r.forward++
+ r.mu.Unlock()
+}
+
+func (r *Resolver) countError() {
+ r.mu.Lock()
+ r.errs++
+ r.mu.Unlock()
+}
+
+func protoOf(w dns.ResponseWriter) string {
+ if _, ok := w.RemoteAddr().(*net.TCPAddr); ok {
+ return tcpNet
+ }
+ return udpNet
+}
+
+func sourceAddr(addr net.Addr) string {
+ if addr == nil {
+ return ""
+ }
+
+ host, _, err := net.SplitHostPort(addr.String())
+ if err != nil {
+ return addr.String()
+ }
+
+ return host
+}
+
+func sameAddr(listen, upstream string) bool {
+ lh, lp, err := net.SplitHostPort(listen)
+ if err != nil {
+ return false
+ }
+
+ uh, up, err := net.SplitHostPort(upstream)
+ if err != nil {
+ return false
+ }
+
+ if lp != up {
+ return false
+ }
+
+ return lh != "" && lh == uh
+}
+
+func joinWith(parts []string, sep string) string {
+ return strings.Join(parts, sep)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver_test.go
new file mode 100644
index 00000000..1ae669f0
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/resolver_test.go
@@ -0,0 +1,425 @@
+// ©AngelaMos | 2026
+// resolver_test.go
+
+package dnsrv
+
+import (
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/miekg/dns"
+)
+
+// TestUnarmedResolverIsIndistinguishable is the baseline guard. Every
+// experiment's baseline is measured through this resolver, so an unarmed one
+// answering anything other than the honest address would void every run before
+// a fault was ever applied.
+func TestUnarmedResolverIsIndistinguishable(t *testing.T) {
+ _, client := startResolver(t)
+
+ for _, name := range []string{testHost, testControl} {
+ got := ask(t, client, name, udpNet)
+
+ if !got.resolved() {
+ t.Fatalf("%s did not resolve unarmed: rcode=%s err=%v",
+ name, rcodeName(got.rcode), got.err)
+ }
+
+ if addresses := got.addresses(); !slices.Contains(
+ addresses,
+ testAddress,
+ ) {
+ t.Errorf("%s answered %v, want %s", name, addresses, testAddress)
+ }
+
+ if got.truncated {
+ t.Errorf("%s was truncated unarmed", name)
+ }
+
+ for _, ttl := range got.ttls() {
+ if ttl != HonestTTL {
+ t.Errorf("%s TTL %d, want %d", name, ttl, HonestTTL)
+ }
+ }
+ }
+}
+
+// TestOutsideZoneIsForwarded proves the resolver does not break the target's
+// unrelated name resolution. A chaos resolver that NXDOMAINed everything it was
+// not authoritative for would be a dependency outage wearing a DNS fault's name.
+func TestOutsideZoneIsForwarded(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ mustArm(t, client, trapFor(NXDomain))
+
+ got := ask(t, client, testUpstream, udpNet)
+
+ if !got.resolved() {
+ t.Fatalf("%s did not resolve through the forward path: rcode=%s err=%v",
+ testUpstream, rcodeName(got.rcode), got.err)
+ }
+
+ if addresses := got.addresses(); !slices.Contains(addresses, testUpAddr) {
+ t.Errorf("forwarded answer %v, want %s", addresses, testUpAddr)
+ }
+
+ if stats := resolver.Stats(); stats.Forward == 0 {
+ t.Error("the resolver recorded no forwarded query")
+ }
+}
+
+// TestSearchSuffixIsFaulted is the measured sharp edge, and the one most likely
+// to be lost in a refactor.
+//
+// A container's resolv.conf carries a forced `search` domain. Under a negative
+// fault the absolute query fails and the client RETRIES with the suffix
+// appended, which was observed arriving as `<name>.mynetworksettings.com.`. A
+// resolver matching only the exact name answers that retry honestly and defeats
+// its own fault, and the run reports a target that resolved its dependency
+// fine.
+// The assertion is on the resolver's own FAULTED counter, not on the rcode, and
+// that is the second half of the lesson. The first version checked for NXDOMAIN
+// and passed with the suffix match deleted, because an unmatched name is
+// forwarded and the test upstream answers NXDOMAIN for anything it does not
+// know. The test was right about the behaviour and wrong about the cause, which
+// is the failure mode the gutting pass exists to find.
+func TestSearchSuffixIsFaulted(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ mustArm(t, client, trapFor(NXDomain))
+
+ suffixed := testHost + "mynetworksettings.com."
+
+ before := resolver.Stats()
+ got := ask(t, client, suffixed, udpNet)
+ after := resolver.Stats()
+
+ if after.Faulted-before.Faulted != 1 {
+ t.Fatalf("search-suffixed %s was not faulted (faulted %d->%d, "+
+ "forwarded %d->%d): the client's retry escaped the fault",
+ suffixed,
+ before.Faulted, after.Faulted,
+ before.Forward, after.Forward,
+ )
+ }
+
+ if got.rcode != dns.RcodeNameError {
+ t.Errorf("search-suffixed %s answered %s, want NXDOMAIN",
+ suffixed, rcodeName(got.rcode))
+ }
+}
+
+func TestFaultsServeWhatTheyClaim(t *testing.T) {
+ type check func(*testing.T, answer)
+
+ hasRcode := func(want int) check {
+ return func(t *testing.T, got answer) {
+ if got.rcode != want {
+ t.Errorf("rcode %s, want %s",
+ rcodeName(got.rcode), rcodeName(want))
+ }
+ }
+ }
+
+ cases := []struct {
+ kind Kind
+ trap Trap
+ check check
+ }{
+ {
+ kind: NXDomain,
+ check: hasRcode(dns.RcodeNameError),
+ },
+ {
+ kind: ServFail,
+ check: hasRcode(dns.RcodeServerFailure),
+ },
+ {
+ kind: WrongAddress,
+ trap: Trap{Address: "203.0.113.9"},
+ check: func(t *testing.T, got answer) {
+ if addresses := got.addresses(); !slices.Contains(
+ addresses,
+ "203.0.113.9",
+ ) {
+ t.Errorf("addresses %v, want 203.0.113.9", addresses)
+ }
+ if slices.Contains(got.addresses(), testAddress) {
+ t.Error("the honest address survived a redirect fault")
+ }
+ },
+ },
+ {
+ kind: CNAMEChain,
+ check: func(t *testing.T, got answer) {
+ if hops := got.cnames(); len(hops) < chainMinimum {
+ t.Errorf(
+ "%d CNAME hops, want at least %d",
+ len(hops),
+ chainMinimum,
+ )
+ }
+ if hasCycle(got.cnames()) {
+ t.Error(
+ "a chain must not be a cycle, or the two faults alibi each other",
+ )
+ }
+ },
+ },
+ {
+ kind: CNAMELoop,
+ check: func(t *testing.T, got answer) {
+ if !hasCycle(got.cnames()) {
+ t.Errorf("CNAMEs %v do not form a cycle", got.cnames())
+ }
+ },
+ },
+ {
+ kind: Truncation,
+ check: func(t *testing.T, got answer) {
+ if !got.truncated {
+ t.Error("tc not set over UDP")
+ }
+ if len(got.addresses()) != 0 {
+ t.Error(
+ "a truncated UDP answer still carried records, so a " +
+ "client ignoring tc is never tested",
+ )
+ }
+ },
+ },
+ {
+ kind: TruncationNoTCP,
+ check: func(t *testing.T, got answer) {
+ if !got.truncated {
+ t.Error("tc not set over UDP")
+ }
+ },
+ },
+ {
+ kind: TTLZero,
+ check: func(t *testing.T, got answer) {
+ for _, ttl := range got.ttls() {
+ if ttl != 0 {
+ t.Errorf("TTL %d, want 0", ttl)
+ }
+ }
+ if len(got.addresses()) == 0 {
+ t.Error(
+ "a TTL fault must still resolve, or it is an outage",
+ )
+ }
+ },
+ },
+ {
+ kind: TTLLong,
+ check: func(t *testing.T, got answer) {
+ for _, ttl := range got.ttls() {
+ if ttl != LongTTL {
+ t.Errorf("TTL %d, want %d", ttl, LongTTL)
+ }
+ }
+ },
+ },
+ {
+ kind: SlowResponse,
+ trap: Trap{Delay: 700 * time.Millisecond},
+ check: func(t *testing.T, got answer) {
+ if got.elapsed < 700*time.Millisecond {
+ t.Errorf("answered in %s, want at least 700ms", got.elapsed)
+ }
+ if len(got.addresses()) == 0 {
+ t.Error("a slow answer must still be the honest answer")
+ }
+ },
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(string(test.kind), func(t *testing.T) {
+ _, client := startResolver(t)
+
+ trap := test.trap
+ trap.Kind = test.kind
+ trap.Host = testHost
+
+ mustArm(t, client, trap)
+
+ test.check(t, ask(t, client, testHost, udpNet))
+
+ // The control is checked on every case rather than once, because a
+ // resolver that crashed on arming would otherwise read as a fault
+ // that landed for every kind in the table.
+ if control := ask(
+ t,
+ client,
+ testControl,
+ udpNet,
+ ); !control.resolved() {
+ t.Errorf(
+ "the control name stopped resolving under %s: rcode=%s err=%v",
+ test.kind,
+ rcodeName(control.rcode),
+ control.err,
+ )
+ }
+ })
+ }
+}
+
+// TestTruncationCompletesOverTCP and its sibling are what separate the two
+// truncation faults. Both set tc over UDP, so UDP alone cannot tell them apart,
+// and the difference is the whole finding: one tests whether the client makes
+// the RFC 5966 retry, the other whether the retry is allowed off the network.
+func TestTruncationCompletesOverTCP(t *testing.T) {
+ _, client := startResolver(t)
+
+ mustArm(t, client, trapFor(Truncation))
+
+ overTCP := ask(t, client, testHost, tcpNet)
+
+ if !overTCP.resolved() {
+ t.Fatalf("TCP did not complete: rcode=%s truncated=%t err=%v",
+ rcodeName(overTCP.rcode), overTCP.truncated, overTCP.err)
+ }
+
+ if overTCP.truncated {
+ t.Error("the TCP answer was still truncated, which is the other fault")
+ }
+}
+
+func TestTruncationNoTCPNeverCompletes(t *testing.T) {
+ _, client := startResolver(t)
+
+ mustArm(t, client, trapFor(TruncationNoTCP))
+
+ overTCP := ask(t, client, testHost, tcpNet)
+
+ if overTCP.resolved() {
+ t.Fatalf("TCP completed with %v, so this is the other truncation fault",
+ overTCP.addresses())
+ }
+
+ if !overTCP.truncated {
+ t.Error("tc not set over TCP")
+ }
+}
+
+// TestArrivalsAreAttributedBySource guards the proof-of-arrival control. A total
+// counter is non-zero on every run, including the runs where the target never
+// used khaos's resolver at all, because khaos's own confirmation queries land in
+// it.
+func TestArrivalsAreAttributedBySource(t *testing.T) {
+ resolver, client := startResolver(t)
+
+ before := resolver.Arrivals("127.0.0.1")
+
+ ask(t, client, testHost, udpNet)
+ ask(t, client, testHost, udpNet)
+
+ after := resolver.Arrivals("127.0.0.1")
+
+ if after-before != 2 {
+ t.Errorf("recorded %d arrivals from 127.0.0.1, want 2", after-before)
+ }
+
+ if other := resolver.Arrivals("10.99.99.99"); other != 0 {
+ t.Errorf("recorded %d arrivals from an address that never asked", other)
+ }
+}
+
+func TestResolverRefusesConfigThatCannotWork(t *testing.T) {
+ cases := []struct {
+ name string
+ cfg Config
+ want error
+ }{
+ {
+ name: "upstream is itself",
+ cfg: Config{
+ Listen: "10.0.0.5:53",
+ Upstream: "10.0.0.5:53",
+ Control: testControl,
+ Zone: map[string]string{testControl: testAddress},
+ },
+ want: ErrUpstreamLoop,
+ },
+ {
+ name: "empty zone",
+ cfg: Config{Listen: loopbackAny, Control: testControl},
+ want: ErrEmptyZone,
+ },
+ {
+ name: "zone entry is not an address",
+ cfg: Config{
+ Listen: loopbackAny,
+ Control: testControl,
+ Zone: map[string]string{testControl: "not-an-address"},
+ },
+ want: ErrZoneAddress,
+ },
+ {
+ name: "no control name",
+ cfg: Config{
+ Listen: loopbackAny,
+ Zone: map[string]string{testHost: testAddress},
+ },
+ want: ErrNoControl,
+ },
+ {
+ name: "control name is not in the zone",
+ cfg: Config{
+ Listen: loopbackAny,
+ Control: "somewhere.else.test.",
+ Zone: map[string]string{testHost: testAddress},
+ },
+ want: ErrNoControl,
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ _, err := New(test.cfg)
+ if err == nil {
+ t.Fatal("accepted a config that cannot work")
+ }
+ if !is(err, test.want) {
+ t.Errorf("error %v, want %v", err, test.want)
+ }
+ })
+ }
+}
+
+// TestArmRefusesAnOutOfZoneHost is the guard against the quietest possible
+// operator mistake: a fault aimed at a name khaos does not answer arms fine,
+// every query for it is forwarded untouched, and the run reports a clean
+// survival against a fault that was applied to nothing.
+func TestArmRefusesAnOutOfZoneHost(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ err := resolver.Arm(Trap{Kind: NXDomain, Host: "not.in.the.zone.test."})
+ if err == nil {
+ t.Fatal("armed a fault on a name the resolver does not answer")
+ }
+
+ if !is(err, ErrOutsideZone) {
+ t.Errorf("error %v, want %v", err, ErrOutsideZone)
+ }
+}
+
+// TestArmRefusesFaultingTheControl stops the confirmation oracle's positive
+// control from being the thing under fault, which would make every observation
+// self-refuting.
+func TestArmRefusesFaultingTheControl(t *testing.T) {
+ resolver, _ := startResolver(t)
+
+ err := resolver.Arm(Trap{Kind: NXDomain, Host: testControl})
+ if err == nil {
+ t.Fatal("armed a fault on the confirmation control")
+ }
+
+ if !is(err, ErrControlIsHost) {
+ t.Errorf("error %v, want %v", err, ErrControlIsHost)
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go
new file mode 100644
index 00000000..06577c87
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap.go
@@ -0,0 +1,402 @@
+// ©AngelaMos | 2026
+// trap.go
+
+package dnsrv
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "time"
+)
+
+// Kind names one way khaos makes name resolution wrong for a target. Every kind
+// maps to a row of the injectable fault matrix in research 05 section 9, and the
+// two rows khaos cannot deliver are kinds here as well, so that asking for one
+// gets an explanation rather than silence.
+//
+// What separates these from the other families is that the fault is not in the
+// dependency at all. The dependency is healthy; the target cannot find it, or
+// finds something else. That reaches a different branch of the target's code
+// from an unreachable dependency and from one presenting a bad certificate, and
+// research 05 section 10 is the argument for why those branches are where the
+// fail-open lives.
+type Kind string
+
+const (
+ // PassThrough is the zero value: the resolver answers correctly by
+ // forwarding. A resolver with nothing armed has to be indistinguishable from
+ // the one the target would otherwise use, because every baseline is measured
+ // through it.
+ PassThrough Kind = ""
+
+ // NXDomain is a definite negative answer. The client stops immediately
+ // without trying other servers, and a Go target gets `IsNotFound`. It is the
+ // cleanest demonstration in research 05 section 10: a control whose
+ // allowlist entry is a hostname cannot evaluate itself when the name does
+ // not resolve, and a great deal of code reads "could not evaluate" as "not a
+ // violation". Row 1.
+ NXDomain Kind = "nxdomain"
+
+ // ServFail is a TEMPORARY failure, and that single word is the difference
+ // from NXDomain: clients with retry logic retry it, so it also measures
+ // whether a transient resolution failure turns into a retry storm or a hang
+ // rather than a decision. Row 2.
+ ServFail Kind = "servfail"
+
+ // SlowResponse answers correctly but late. Its ceiling is not a preference:
+ // Docker's embedded resolver gives up at `extIOTimeout`, measured at exactly
+ // 4.001s, and above that this fault silently becomes Timeout, which khaos
+ // refuses. So the magnitude is bounded at MaxDelay and anything larger is
+ // refused by name. Row 5.
+ SlowResponse Kind = "slow-response"
+
+ // WrongAddress is the money experiment of the family. The name resolves, to
+ // an address the operator never approved. A hostname allowlist does not
+ // merely stop working here, it actively authorises the wrong host, and TLS
+ // is the only thing left that can object. Row 6.
+ WrongAddress Kind = "wrong-address"
+
+ // CNAMEChain answers with a multi-hop CNAME ending outside the queried zone.
+ // It asks whether name-based matching happens on the name that was ASKED or
+ // on the target the connection actually goes to, which research 05 section
+ // 10 lists as the fourth distinct failure of hostname allowlisting. Row 10.
+ CNAMEChain Kind = "cname-chain"
+
+ // CNAMELoop answers with a CNAME cycle. The research measured CoreDNS
+ // serving one without hanging and noted explicitly that the CLIENT's loop
+ // handling is the thing under test, which no other kind here covers.
+ CNAMELoop Kind = "cname-loop"
+
+ // Truncation sets the `tc` bit on UDP and answers completely over TCP, which
+ // is what a correct truncating server does. The client must notice and
+ // re-ask over TCP per RFC 5966.
+ //
+ // This kind exists because of a CORRECTION rather than in spite of one.
+ // Research 05 section 5 recorded truncation as absorbed entirely by the
+ // embedded resolver, which would have made the fault undeliverable. It is
+ // not: measured 2026-10-13, `tc=1` reaches the client in every truncating
+ // shape and the client's TCP retry arrives here. The original measurement
+ // used `dig` without `+ignore`, so dig performed the fallback itself and
+ // reported the clean answer. Row 9.
+ Truncation Kind = "truncation"
+
+ // TruncationNoTCP sets `tc` and then never completes over TCP either. It is
+ // a different question from Truncation and it is the one research 05 row 9
+ // names as the security control: whether DNS over TCP is actually permitted
+ // by the egress policy. A deny-all-TCP-53 rule turns every large answer into
+ // total resolution failure.
+ //
+ // Measured consequence worth knowing before reading a verdict: to a Go
+ // target this lands as `no such host` with `IsNotFound=true`, which is
+ // NXDomain's classification. khaos names which one it served because the
+ // target cannot.
+ TruncationNoTCP Kind = "truncation-no-tcp"
+
+ // TTLZero and TTLLong act on the answer's lifetime rather than its content.
+ // They are the two kinds whose fault may be a measured NO-OP: Go has no
+ // answer cache and ignores TTL entirely, proven against a 604800 TTL, so
+ // against a Go target these test nothing.
+ //
+ // khaos does not guess. It counts the target's own queries during the fault
+ // window, and a target that re-queries per probe is reported UNSUPPORTED,
+ // the same way the revocation group reports a target that never asked.
+ // Rows 7 and 8.
+ TTLZero Kind = "ttl-zero"
+ TTLLong Kind = "ttl-long"
+
+ // Refused and Timeout are REFUSED BY KHAOS, and they are kinds rather than
+ // omissions on purpose. Docker's embedded resolver rewrites both: REFUSED
+ // becomes SERVFAIL because `forwardExtDNS` treats it as a reason to try the
+ // next upstream, and a dropped query becomes SERVFAIL at a hard 4.001s.
+ //
+ // Delivering SERVFAIL while claiming REFUSED would be worse than refusing,
+ // because the whole point is which branch of the target's code runs, and a
+ // real library branches on the rcode. An operator who asks for one of these
+ // gets the reason and the name of the mechanism instead of a wrong
+ // measurement. Rows 3 and 4.
+ Refused Kind = "refused"
+ Timeout Kind = "timeout"
+)
+
+const (
+ // MaxDelay is one millisecond under the measured `extIOTimeout` ceiling. At
+ // or above 4.000s the embedded resolver answers SERVFAIL itself at 4.001s
+ // and the delay never reaches the client, so a larger magnitude would be
+ // Timeout wearing SlowResponse's name.
+ MaxDelay = 3999 * time.Millisecond
+
+ // DefaultDelay sits well inside the ceiling and above any plausible
+ // sub-second client timeout, so the fault is observable without being at the
+ // edge of the window khaos is allowed to use.
+ DefaultDelay = 3 * time.Second
+
+ // LongTTL is one week, matching the research's own measurement against Go.
+ LongTTL = 604800
+
+ // HonestTTL is what an unfaulted khaos answer carries. It matches the TTL
+ // Docker's own embedded resolver puts on its internal answers, so a target
+ // comparing them sees nothing unusual.
+ HonestTTL = 600
+
+ // DefaultWrongAddress is in TEST-NET-2, which RFC 5737 reserves for
+ // documentation. Nothing can be running there, so a target that connects is
+ // observably connecting to an address its operator never approved rather
+ // than to some other real service.
+ DefaultWrongAddress = "198.51.100.66"
+)
+
+const (
+ describeNXDomain = "answer %s that the dependency's name does not exist"
+ describeServFail = "answer %s that resolving the dependency's name failed temporarily"
+ describeSlow = "hold %s's lookup of the dependency for %s before answering correctly"
+ describeWrong = "answer %s that the dependency is at %s, an address nobody approved"
+ describeChain = "answer %s with a %d-hop CNAME ending outside the name it asked for"
+ describeLoop = "answer %s with a CNAME cycle"
+ describeTrunc = "answer %s over UDP with the truncation bit set, and completely over TCP"
+ describeTruncNo = "answer %s with the truncation bit set over UDP and refuse to complete over TCP either"
+ describeTTLZero = "answer %s correctly with a TTL of zero, so nothing may cache it"
+ describeTTLLong = "answer %s correctly with a TTL of %d seconds, so a cache pins it for a week"
+
+ describeRefused = "answer %s REFUSED, which khaos cannot deliver"
+ describeTimeout = "drop %s's query entirely, which khaos cannot deliver"
+
+ describeUnknown = "an unrecognised DNS fault"
+)
+
+const (
+ errUnknownKind = "%w: %q (known faults: %s)"
+ errMasked = "%w: %s"
+ errDelayRange = "%w: %s was asked for a %s delay, and the ceiling is %s"
+ errNoHost = "%w: %s needs the dependency's name, and none was given"
+ errBadAddress = "%w: %q is not an IP address"
+
+ // maskedRefused and maskedTimeout are the refusals themselves. They name the
+ // mechanism, what the target would actually have received, and what to ask
+ // for instead, because a refusal that does not say what to do next is a dead
+ // end rather than a diagnosis.
+ maskedRefused = "Docker's embedded resolver rewrites REFUSED to SERVFAIL before the target sees it. `forwardExtDNS` treats REFUSED as a reason to try the next upstream, and with one upstream configured there is no next one, so the client is answered SERVFAIL. khaos will not deliver SERVFAIL and call it REFUSED, because a real library branches on the rcode and the branch is the finding. Run `dns.servfail` to measure the branch the target will actually take"
+
+ maskedTimeout = "Docker's embedded resolver answers SERVFAIL at a measured 4.001s rather than letting a query go unanswered, so a true socket timeout cannot reach a container on a user-defined bridge. Its `extIOTimeout` is 4 seconds and the ceiling is not configurable. khaos will not deliver an rcode and call it a timeout. Run `dns.slow_response` with a delay under 4s for a late answer, or `dns.servfail` for the definite failure the target would really get"
+
+ chainHops = 3
+)
+
+var ErrUnknownKind = errors.New("no DNS fault is registered under that name")
+
+// ErrMasked is the refusal the M8 exit gate is about. It is a distinct error
+// from an unknown fault, because the fault IS known, is catalogued, and is
+// refused for a measured reason rather than missing.
+var ErrMasked = errors.New(
+ "khaos refuses this DNS fault because Docker's embedded resolver would rewrite it",
+)
+
+var ErrDelayRange = errors.New(
+ "that delay is at or above the embedded resolver's ceiling, where this fault becomes a different one",
+)
+
+var ErrNoHost = errors.New("this DNS fault needs a name to act on")
+
+var ErrBadAddress = errors.New("that is not an IP address")
+
+// Trap is one armed fault, and it travels through the journal, so every field
+// has to be enough to re-describe the fault from a cold start.
+type Trap struct {
+ Kind Kind `json:"kind"`
+ Host string `json:"host"`
+
+ // Delay is SlowResponse's magnitude.
+ Delay time.Duration `json:"delay,omitempty"`
+
+ // Address is WrongAddress's answer.
+ Address string `json:"address,omitempty"`
+
+ // Inert is the negative control. An inert trap arms the resolver with
+ // nothing, so every observable this fault is confirmed by must disappear.
+ // Without it a confirmation that reads the trap RECORD rather than the
+ // SERVED answer passes against a resolver that is doing nothing, which is
+ // the bug the PKI tier's inert authority caught three times.
+ Inert bool `json:"inert,omitempty"`
+}
+
+// Kinds names every fault in the catalogue, including the two khaos refuses, so
+// a census over the catalogue covers them and cannot quietly lose one.
+func Kinds() []string {
+ return []string{
+ string(NXDomain),
+ string(ServFail),
+ string(SlowResponse),
+ string(WrongAddress),
+ string(CNAMEChain),
+ string(CNAMELoop),
+ string(Truncation),
+ string(TruncationNoTCP),
+ string(TTLZero),
+ string(TTLLong),
+ string(Refused),
+ string(Timeout),
+ }
+}
+
+// Deliverable reports whether khaos can actually produce this fault as asked
+// through Docker's embedded resolver. It is derived from one list so the
+// catalogue, the refusal and the gate's census cannot disagree about which
+// faults are masked.
+func Deliverable(kind Kind) bool {
+ return !slices.Contains(masked, kind)
+}
+
+var masked = []Kind{Refused, Timeout}
+
+// MaskedReason is the refusal text for a fault khaos will not approximate.
+func MaskedReason(kind Kind) string {
+ switch kind {
+ case Refused:
+ return maskedRefused
+ case Timeout:
+ return maskedTimeout
+ default:
+ return ""
+ }
+}
+
+// Known reports whether this is a catalogued fault at all, which is separate
+// from whether it can be delivered.
+func Known(kind Kind) bool {
+ return kind == PassThrough || slices.Contains(Kinds(), string(kind))
+}
+
+// Describe says what the fault does to the target, in the operator's words. It
+// is the only place the wording lives, so `khaos catalog`, `khaos plan` and the
+// run record cannot describe the same fault three different ways.
+func (t Trap) Describe(target string) string {
+ switch t.Kind {
+ case NXDomain:
+ return fmt.Sprintf(describeNXDomain, target)
+ case ServFail:
+ return fmt.Sprintf(describeServFail, target)
+ case SlowResponse:
+ return fmt.Sprintf(describeSlow, target, t.delay())
+ case WrongAddress:
+ return fmt.Sprintf(describeWrong, target, t.address())
+ case CNAMEChain:
+ return fmt.Sprintf(describeChain, target, chainHops)
+ case CNAMELoop:
+ return fmt.Sprintf(describeLoop, target)
+ case Truncation:
+ return fmt.Sprintf(describeTrunc, target)
+ case TruncationNoTCP:
+ return fmt.Sprintf(describeTruncNo, target)
+ case TTLZero:
+ return fmt.Sprintf(describeTTLZero, target)
+ case TTLLong:
+ return fmt.Sprintf(describeTTLLong, target, LongTTL)
+ case Refused:
+ return fmt.Sprintf(describeRefused, target)
+ case Timeout:
+ return fmt.Sprintf(describeTimeout, target)
+ case PassThrough:
+ return describeUnknown
+ default:
+ return describeUnknown
+ }
+}
+
+// Validate refuses a trap that cannot be armed, BEFORE it reaches the journal.
+// A record khaos cannot arm is one `khaos reclaim` would retry forever, and a
+// masked fault refused here is refused once with a reason rather than halfway
+// through a run.
+func (t Trap) Validate() error {
+ if err := t.ValidateShape(); err != nil {
+ return err
+ }
+
+ if !Deliverable(t.Kind) {
+ return fmt.Errorf(errMasked, ErrMasked, MaskedReason(t.Kind))
+ }
+
+ return nil
+}
+
+// ValidateShape checks everything except whether khaos will deliver the fault at
+// all, and the split exists because the two refusals belong at different moments.
+//
+// A delay above the ceiling is an operator typing a number, so it is refused
+// here, at build time. A MASKED fault is not a mistake: it is a catalogued
+// experiment whose answer is an explanation, so it has to build, plan and
+// describe like any other entry, and Validate refuses it only when khaos is
+// asked to arm one.
+func (t Trap) ValidateShape() error {
+ if !Known(t.Kind) {
+ return fmt.Errorf(
+ errUnknownKind,
+ ErrUnknownKind,
+ string(t.Kind),
+ joinKinds(),
+ )
+ }
+
+ if t.Kind == PassThrough {
+ return nil
+ }
+
+ if t.Host == "" {
+ return fmt.Errorf(errNoHost, ErrNoHost, string(t.Kind))
+ }
+
+ if t.Kind == SlowResponse && t.Delay > MaxDelay {
+ return fmt.Errorf(
+ errDelayRange,
+ ErrDelayRange,
+ string(t.Kind),
+ t.Delay,
+ MaxDelay,
+ )
+ }
+
+ if t.Kind == WrongAddress && t.Address != "" {
+ if err := validAddress(t.Address); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (t Trap) delay() time.Duration {
+ if t.Delay <= 0 {
+ return DefaultDelay
+ }
+ return t.Delay
+}
+
+func (t Trap) address() string {
+ if t.Address == "" {
+ return DefaultWrongAddress
+ }
+ return t.Address
+}
+
+// ttl is the lifetime this trap puts on an answer it synthesizes. Every kind
+// that answers at all goes through here, so a new kind cannot forget to carry
+// one and serve a zero TTL by accident, which would look exactly like TTLZero.
+func (t Trap) ttl() uint32 {
+ switch t.Kind {
+ case TTLZero:
+ return 0
+ case TTLLong:
+ return LongTTL
+ default:
+ return HonestTTL
+ }
+}
+
+func joinKinds() string {
+ return joinWith(Kinds(), ", ")
+}
+
+func sorted(values []string) []string {
+ out := slices.Clone(values)
+ slices.Sort(out)
+ return out
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap_test.go
new file mode 100644
index 00000000..9e11de49
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dnsrv/trap_test.go
@@ -0,0 +1,183 @@
+// ©AngelaMos | 2026
+// trap_test.go
+
+package dnsrv
+
+import (
+ "strings"
+ "testing"
+ "time"
+)
+
+// TestMaskedFaultsAreRefusedByName is the M8 exit gate in a unit test. The two
+// faults Docker's embedded resolver rewrites are catalogued rather than missing,
+// and asking for one has to produce the reason instead of a wrong measurement.
+func TestMaskedFaultsAreRefusedByName(t *testing.T) {
+ for _, kind := range []Kind{Refused, Timeout} {
+ t.Run(string(kind), func(t *testing.T) {
+ err := Trap{Kind: kind, Host: testHost}.Validate()
+ if err == nil {
+ t.Fatal("accepted a fault khaos cannot deliver")
+ }
+
+ if !is(err, ErrMasked) {
+ t.Fatalf("error %v, want %v", err, ErrMasked)
+ }
+
+ // A refusal that does not say what to do next is a dead end. Both
+ // reasons name the mechanism and a fault to run instead, and the
+ // gate asserts the same text, so this cannot decay into "refused".
+ reason := MaskedReason(kind)
+ for _, want := range []string{"embedded resolver", "dns."} {
+ if !strings.Contains(reason, want) {
+ t.Errorf("the refusal for %s never mentions %q: %s",
+ kind, want, reason)
+ }
+ }
+ })
+ }
+}
+
+// TestEveryCatalogueKindIsEitherDeliverableOrExplained is the census. A kind
+// that is neither servable nor refused-with-a-reason is one `khaos catalog`
+// would advertise and `khaos run` could not do anything with.
+func TestEveryCatalogueKindIsEitherDeliverableOrExplained(t *testing.T) {
+ for _, name := range Kinds() {
+ kind := Kind(name)
+
+ if Deliverable(kind) {
+ if reason := MaskedReason(kind); reason != "" {
+ t.Errorf("%s is deliverable and also carries a refusal: %s",
+ kind, reason)
+ }
+ continue
+ }
+
+ if MaskedReason(kind) == "" {
+ t.Errorf("%s is refused with no reason, so an operator is told no "+
+ "and not why", kind)
+ }
+ }
+}
+
+// TestEveryKindDescribesItself keeps `khaos catalog`, `khaos plan` and the run
+// record from describing the same fault three different ways, and catches a new
+// kind that forgets its wording and falls through to the generic string.
+func TestEveryKindDescribesItself(t *testing.T) {
+ for _, name := range Kinds() {
+ kind := Kind(name)
+
+ described := Trap{Kind: kind, Host: testHost}.Describe("the-target")
+
+ if described == describeUnknown {
+ t.Errorf("%s has no description of its own", kind)
+ }
+
+ if !strings.Contains(described, "the-target") {
+ t.Errorf("%s describes itself without naming the target: %s",
+ kind, described)
+ }
+ }
+}
+
+// TestDelayCeilingIsEnforced is the measured boundary, not a guessed one. At or
+// above 4.000s Docker's embedded resolver answers SERVFAIL itself at 4.001s and
+// the delay never reaches the client, so a larger magnitude would be the Timeout
+// fault wearing SlowResponse's name.
+func TestDelayCeilingIsEnforced(t *testing.T) {
+ cases := []struct {
+ delay time.Duration
+ accept bool
+ }{
+ {delay: 0, accept: true},
+ {delay: time.Second, accept: true},
+ {delay: 3900 * time.Millisecond, accept: true},
+ {delay: MaxDelay, accept: true},
+ {delay: 4 * time.Second, accept: false},
+ {delay: 4100 * time.Millisecond, accept: false},
+ {delay: 30 * time.Second, accept: false},
+ }
+
+ for _, test := range cases {
+ t.Run(test.delay.String(), func(t *testing.T) {
+ err := Trap{
+ Kind: SlowResponse,
+ Host: testHost,
+ Delay: test.delay,
+ }.Validate()
+
+ if test.accept && err != nil {
+ t.Fatalf("refused a deliverable delay: %v", err)
+ }
+
+ if !test.accept {
+ if err == nil {
+ t.Fatal("accepted a delay above the measured ceiling")
+ }
+ if !is(err, ErrDelayRange) {
+ t.Errorf("error %v, want %v", err, ErrDelayRange)
+ }
+ }
+ })
+ }
+}
+
+func TestValidateRefusesWhatCannotBeArmed(t *testing.T) {
+ cases := []struct {
+ name string
+ trap Trap
+ want error
+ }{
+ {
+ name: "unknown kind",
+ trap: Trap{Kind: "not-a-fault", Host: testHost},
+ want: ErrUnknownKind,
+ },
+ {
+ name: "no host",
+ trap: Trap{Kind: NXDomain},
+ want: ErrNoHost,
+ },
+ {
+ name: "redirect to something that is not an address",
+ trap: Trap{
+ Kind: WrongAddress,
+ Host: testHost,
+ Address: "not-an-address",
+ },
+ want: ErrBadAddress,
+ },
+ }
+
+ for _, test := range cases {
+ t.Run(test.name, func(t *testing.T) {
+ err := test.trap.Validate()
+ if err == nil {
+ t.Fatal("accepted a trap that cannot be armed")
+ }
+ if !is(err, test.want) {
+ t.Errorf("error %v, want %v", err, test.want)
+ }
+ })
+ }
+}
+
+// TestTTLIsCarriedByEveryAnsweringKind catches the shape where a new kind
+// forgets its TTL and serves zero by accident, which is indistinguishable from
+// the TTLZero fault having been armed.
+func TestTTLIsCarriedByEveryAnsweringKind(t *testing.T) {
+ for _, name := range Kinds() {
+ kind := Kind(name)
+
+ if kind == TTLZero {
+ continue
+ }
+
+ ttl := Trap{Kind: kind, Host: testHost}.ttl()
+
+ if ttl == 0 {
+ t.Errorf("%s serves a zero TTL, which reads exactly like %s",
+ kind, TTLZero)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go
index 242a6127..798ae89d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/doctor.go
@@ -37,6 +37,21 @@ const (
checkDaemon = "daemon"
checkTargets = "consenting targets"
+ // checkDNSTier is the one check whose subject is khaos's OWN binary rather
+ // than the operator's Docker. `khaos resolver up` imports that binary as a
+ // one-layer container image with no base, so a dynamically linked build
+ // cannot CREATE a resolver — and it finds out at run time, with an error
+ // about a loader path the operator never typed. `doctor` is where khaos is
+ // supposed to turn that into a named cause and a fix.
+ //
+ // It does NOT lose the family, and the first wording of this check said it
+ // did. Arming is independent of importing: a dynamically linked khaos drives
+ // a resolver that already exists perfectly well, including the range's own,
+ // which was measured by running `dns.nxdomain` from such a build and getting
+ // a real verdict. Saying otherwise would have sent an operator to rebuild
+ // over a limit they were not hitting.
+ checkDNSTier = "dns tier"
+
fixNoEndpoint = "install Docker, or set DOCKER_HOST to a reachable daemon"
fixDaemonLocal = "start the Docker daemon: systemctl --user start docker, or sudo systemctl start docker"
fixDaemonRemote = "check that %s is reachable and that the daemon there is listening"
@@ -56,7 +71,13 @@ const (
"Rootless Docker avoids this entirely and khaos prefers it, since access to %s is equivalent to root on this machine"
fallbackUser = "your user"
- checkCapacity = 4
+ detailDNSStatic = "khaos is statically linked, so `khaos resolver up` can build its own resolver image"
+ detailDNSDynamic = "khaos is dynamically linked, so `khaos resolver up` cannot build a resolver image; %d chaos resolver(s) already running are still usable"
+ detailResolvers = "%s, and %d chaos resolver(s) are running"
+
+ fixDNSDynamic = "rebuild with CGO_ENABLED=0 (the justfile and the release builds both do), or install the released binary. `khaos resolver up` imports khaos's own binary as a one-layer container image with no base, because an unprivileged process cannot bind port 53 and Docker's --dns carries an address with no port; a dynamically linked binary in a base-less image cannot exec. Running `dns.` experiments against a resolver that already exists, including the range's, does not need this"
+
+ checkCapacity = 5
)
func Diagnose(ctx context.Context) []Check {
@@ -135,7 +156,50 @@ func DiagnoseWith(
})
}
- return append(checks, targetsCheck(targets))
+ return append(checks, targetsCheck(targets), dnsTierCheck(ctx, cli))
+}
+
+// dnsTierCheck reports whether khaos can stand its chaos resolver up at all,
+// and how many it already has.
+//
+// The linkage half is checked here rather than discovered during a run, because
+// the failure is otherwise invisible until the worst moment: `khaos run dns.*`
+// imports an image, creates a container, starts it, and the container dies
+// looking for `/lib64/ld-linux-x86-64.so.2`. The operator sees an exec error
+// about a path they never typed. One line in `doctor` turns that into the build
+// flag that fixes it.
+func dnsTierCheck(ctx context.Context, cli *Client) Check {
+ // Counted first, because it changes what a dynamic build's refusal MEANS.
+ // A failure to list is not a failure of the tier: khaos can still stand a
+ // resolver up, it just cannot say what is already there, and reporting that
+ // as a broken tier would send an operator to rebuild a binary that is fine.
+ running := -1
+ if resolvers, err := cli.Resolvers(ctx); err == nil {
+ running = len(resolvers)
+ }
+
+ if err := SelfStatic(); err != nil {
+ return Check{
+ Name: checkDNSTier,
+ Status: StatusFail,
+ Detail: fmt.Sprintf(detailDNSDynamic, max(running, 0)),
+ Fix: fixDNSDynamic,
+ }
+ }
+
+ if running < 0 {
+ return Check{
+ Name: checkDNSTier,
+ Status: StatusPass,
+ Detail: detailDNSStatic,
+ }
+ }
+
+ return Check{
+ Name: checkDNSTier,
+ Status: StatusPass,
+ Detail: fmt.Sprintf(detailResolvers, detailDNSStatic, running),
+ }
}
func skipped(name string) Check {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go
index 34973643..d2151a3d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/endpoint.go
@@ -76,9 +76,8 @@ func ResolveEndpoint(
// ag-server. khaos would then inject on one machine while `docker compose`
// acted on another, which is the worst shape this resolver can take.
//
- // (The audit that raised this also claimed DOCKER_CONTEXT outranks
- // DOCKER_HOST. Measured on docker 29.1.3, it does not: DOCKER_HOST wins, and
- // the precedence above is correct.)
+ // DOCKER_HOST outranks DOCKER_CONTEXT, measured on docker 29.1.3, which is
+ // why it is checked first above.
if name := environ(EnvDockerContext); name == defaultContext {
return defaultEndpoint(environ)
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go
new file mode 100644
index 00000000..24dcbaac
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver.go
@@ -0,0 +1,554 @@
+// ©AngelaMos | 2026
+// resolver.go
+
+package dockerx
+
+import (
+ "archive/tar"
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "debug/elf"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "slices"
+ "strings"
+
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/network"
+ "github.com/moby/moby/client"
+)
+
+const (
+ // ResolverLabel marks a container khaos created to serve DNS, and
+ // ResolverNetworkLabel records which network it serves. Both exist so an
+ // orphan survives khaos's death and is reclaimable by label rather than by
+ // khaos remembering what it started, which is the same rule the netem
+ // sidecar tier is held to.
+ ResolverLabel = "khaos.resolver"
+ ResolverNetworkLabel = "khaos.resolver.network"
+ ResolverNonceLabel = "khaos.resolver.nonce"
+ ResolverValue = "true"
+
+ // ResolverImagePrefix tags the image khaos imports from its own binary. The
+ // tag carries a digest of that binary, so a rebuilt khaos imports a new
+ // image instead of silently running the previous build's resolver — which
+ // would be a resolver whose code does not match the khaos interpreting it.
+ ResolverImagePrefix = "khaos-dnsrv"
+
+ // ResolverEntrypoint is where khaos's own binary lands inside the image, and
+ // ResolverCommand is the hidden subcommand it is started with.
+ ResolverEntrypoint = "/khaos"
+ ResolverCommand = "serve-resolver"
+
+ // EnvResolverNonce is where the control nonce travels. See resolverArgs for
+ // why it is not a flag.
+ EnvResolverNonce = "KHAOS_RESOLVER_NONCE"
+
+ // resolverBinaryMode is executable and not writable. The image has exactly
+ // one file in it and nothing in the container can replace it.
+ resolverBinaryMode = 0o555
+
+ digestLength = 12
+ selfExe = "/proc/self/exe"
+)
+
+const (
+ errReadSelf = "read khaos's own binary at %s: %w"
+ errDynamicSelf = "%w: %s"
+ errTarSelf = "build the resolver image layer: %w"
+ errImportImage = "import the resolver image %s: %w"
+ errImportDrain = "import the resolver image %s: the daemon reported: %s"
+ errCreateResolve = "create the chaos resolver container on %s: %w"
+ errStartResolve = "start the chaos resolver container %s: %w"
+ errResolverAddr = "read the chaos resolver's address on %s: %w"
+ errListResolvers = "list khaos's resolver containers: %w"
+ errResolverExists = "%w: %s already serves DNS for %s"
+
+ // A dynamically linked khaos cannot BUILD a resolver image, and drives one
+ // that already exists perfectly well, so the refusal says which of the two
+ // it is.
+ dynamicSelfFix = "`khaos resolver up` imports khaos's own binary as a one-layer container image with no base, because an unprivileged process cannot bind port 53 and Docker's --dns carries an address with no port, so the binary has to be statically linked. This one is dynamically linked and would fail to exec with no shared libraries to find. Rebuild with CGO_ENABLED=0 (the justfile and the release builds both do), or install the released binary. Running `dns.` experiments against a resolver that already exists, including the range's, does not need this"
+
+ filterLabelKey = "label"
+)
+
+var ErrDynamicSelf = errors.New(
+ "this khaos binary is dynamically linked and cannot be its own container image",
+)
+
+var ErrResolverExists = errors.New(
+ "a khaos resolver is already serving that network",
+)
+
+// Resolver is a running chaos resolver container, as khaos addresses it.
+type Resolver struct {
+ ID string
+ Name string
+ Network string
+ Address string
+ Nonce string
+}
+
+// ResolverSpec is what khaos needs to stand one up.
+type ResolverSpec struct {
+ // Network is the target's network. The resolver joins it so the target can
+ // reach it by address with no published host port at all.
+ Network string
+
+ // Name is the container name, which carries the network so two resolvers on
+ // two networks cannot collide.
+ Name string
+
+ // Nonce authorises the resolver's control surface and is stored as a LABEL
+ // as well as passed as an argument, so khaos reads it back off the container
+ // rather than remembering what it chose.
+ Nonce string
+
+ // Zone is the names the resolver is authoritative for, each mapped to its
+ // honest address, passed as `name=address` arguments.
+ Zone map[string]string
+
+ // Control is the zone name no fault ever touches.
+ Control string
+}
+
+// SelfStatic refuses a khaos binary that cannot be its own container image.
+//
+// The check is on the ELF rather than on a build tag, because what matters is
+// the artifact: `go build` with cgo available links libc for `net`, and the
+// resulting binary in a base-less image dies looking for
+// `/lib64/ld-linux-x86-64.so.2`. Measured on this host before the tier was
+// built.
+func SelfStatic() error {
+ return staticELF(selfExe)
+}
+
+// staticELF is split out from SelfStatic so it can be given a two-sided
+// control. A guard that decides whether a whole command works needs an input it
+// MUST reject, and `/proc/self/exe` inside a test is whatever `go test` built,
+// which is not a value a test gets to choose.
+func staticELF(path string) error {
+ file, err := elf.Open(path)
+ if err != nil {
+ return fmt.Errorf(errReadSelf, path, err)
+ }
+ defer func() { _ = file.Close() }()
+
+ for _, prog := range file.Progs {
+ if prog.Type == elf.PT_INTERP {
+ return fmt.Errorf(errDynamicSelf, ErrDynamicSelf, dynamicSelfFix)
+ }
+ }
+
+ return nil
+}
+
+// ImportResolverImage builds a single-layer image out of khaos's own binary and
+// imports it, with no base image, no registry and no network.
+//
+// This is what keeps the DNS tier keyless. The alternative shapes were all
+// worse: a published image needs a registry and drifts out of step with the CLI
+// that drives it, and building one needs the source tree, which a curl-installed
+// khaos does not have. A one-file tar through the import endpoint needs nothing
+// that is not already on the machine.
+//
+// The image is khaos's ENTIRE binary, around 24 MB, so the resolver container
+// carries the whole CLI as its entrypoint. That is the price of there being no
+// second artifact to build, version and keep in step with the CLI driving it.
+func (c *Client) ImportResolverImage(ctx context.Context) (string, error) {
+ if err := SelfStatic(); err != nil {
+ return "", err
+ }
+
+ binary, err := os.ReadFile(selfExe)
+ if err != nil {
+ return "", fmt.Errorf(errReadSelf, selfExe, err)
+ }
+
+ digest := sha256.Sum256(binary)
+ tag := ResolverImagePrefix + ":" +
+ hex.EncodeToString(digest[:])[:digestLength]
+
+ // Tagged by content, so a khaos that has run before reuses its image and a
+ // rebuilt khaos cannot run the previous build's resolver.
+ if have, err := c.hasImage(ctx, tag); err == nil && have {
+ return tag, nil
+ }
+
+ layer, err := tarSingleFile(ResolverEntrypoint, binary)
+ if err != nil {
+ return "", fmt.Errorf(errTarSelf, err)
+ }
+
+ result, err := c.api.ImageImport(
+ ctx,
+ client.ImageImportSource{
+ Source: bytes.NewReader(layer),
+ SourceName: "-",
+ },
+ tag,
+ client.ImageImportOptions{
+ Changes: []string{
+ `ENTRYPOINT ["` + ResolverEntrypoint + `"]`,
+ },
+ },
+ )
+ if err != nil {
+ return "", fmt.Errorf(errImportImage, tag, err)
+ }
+ defer func() { _ = result.Close() }()
+
+ // The import endpoint streams progress and reports failures in the BODY
+ // rather than in the status, so an import that is never drained can fail
+ // invisibly and leave khaos creating a container from an image that does not
+ // exist.
+ body, err := io.ReadAll(result)
+ if err != nil {
+ return "", fmt.Errorf(errImportImage, tag, err)
+ }
+
+ if strings.Contains(string(body), `"error"`) {
+ return "", fmt.Errorf(errImportDrain, tag, firstLine(body))
+ }
+
+ // Every superseded resolver image is removed now, because nothing else ever
+ // would. The tag carries a digest of khaos's binary, so each rebuild of
+ // khaos creates a NEW image and leaves the previous one behind: measured at
+ // 24.4MB each, and khaos is rebuilt on every `just build`. On a machine that
+ // also serves live traffic, litter khaos creates and never collects is
+ // khaos's problem rather than the operator's.
+ //
+ // Failures are deliberately ignored. An image somebody else is using is
+ // refused by the daemon, which is the behaviour wanted, and a resolver that
+ // imported correctly must not fail because tidying up afterwards did not.
+ c.pruneResolverImages(ctx, tag)
+
+ return tag, nil
+}
+
+// pruneResolverImages removes khaos's own resolver images except the one just
+// imported.
+//
+// Three guards, because this deletes images. Only tags under
+// ResolverImagePrefix are candidates, so nothing khaos did not create can be
+// reached; the tag in use is excluded by name; and removal is not forced, so the
+// daemon refuses any image a container still references rather than khaos
+// deciding it is safe.
+func (c *Client) pruneResolverImages(ctx context.Context, keep string) {
+ result, err := c.api.ImageList(ctx, client.ImageListOptions{})
+ if err != nil {
+ return
+ }
+
+ for _, image := range result.Items {
+ for _, tag := range image.RepoTags {
+ if tag == keep ||
+ !strings.HasPrefix(tag, ResolverImagePrefix+":") {
+ continue
+ }
+
+ _, _ = c.api.ImageRemove(ctx, tag, client.ImageRemoveOptions{})
+ }
+ }
+}
+
+func (c *Client) hasImage(ctx context.Context, tag string) (bool, error) {
+ result, err := c.api.ImageList(ctx, client.ImageListOptions{})
+ if err != nil {
+ return false, err
+ }
+
+ for _, image := range result.Items {
+ if slices.Contains(image.RepoTags, tag) {
+ return true, nil
+ }
+ }
+
+ return false, nil
+}
+
+// RunResolver creates and starts a chaos resolver on a network and reads its
+// address back out of the daemon.
+//
+// The address is read back rather than chosen, because Docker refuses `--ip` on
+// a network whose subnet it picked: "user specified IP address is supported only
+// when connecting to networks with user configured subnets". That measurement is
+// also why arming is in band — a resolver that had to be recreated to be armed
+// would move the address the target was created to dial.
+func (c *Client) RunResolver(
+ ctx context.Context,
+ spec ResolverSpec,
+) (Resolver, error) {
+ if existing, err := c.FindResolver(ctx, spec.Network); err == nil {
+ return Resolver{}, fmt.Errorf(
+ errResolverExists,
+ ErrResolverExists,
+ existing.Name,
+ spec.Network,
+ )
+ }
+
+ image, err := c.ImportResolverImage(ctx)
+ if err != nil {
+ return Resolver{}, err
+ }
+
+ created, err := c.api.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Name: spec.Name,
+ Config: &container.Config{
+ Image: image,
+ Cmd: resolverArgs(spec),
+ Env: resolverEnv(spec),
+ Labels: map[string]string{
+ ResolverLabel: ResolverValue,
+ ResolverNetworkLabel: spec.Network,
+ ResolverNonceLabel: spec.Nonce,
+ },
+ },
+ HostConfig: &container.HostConfig{
+ // Nothing is published. The resolver listens on 53 inside its own
+ // network namespace, which is the entire reason khaos holds no
+ // capability: an unprivileged host process cannot bind 53 at all.
+ RestartPolicy: container.RestartPolicy{
+ Name: container.RestartPolicyDisabled,
+ },
+ },
+ NetworkingConfig: &network.NetworkingConfig{
+ EndpointsConfig: map[string]*network.EndpointSettings{
+ spec.Network: {},
+ },
+ },
+ })
+ if err != nil {
+ return Resolver{}, fmt.Errorf(errCreateResolve, spec.Network, err)
+ }
+
+ if _, err := c.api.ContainerStart(
+ ctx,
+ created.ID,
+ client.ContainerStartOptions{},
+ ); err != nil {
+ // A container that will not start is removed rather than left behind.
+ // The daemon accepts a DNS entry it will later refuse at start time, so
+ // this path is reachable by an operator mistake and not only by a bug.
+ _ = c.RemoveResolver(ctx, created.ID)
+ return Resolver{}, fmt.Errorf(errStartResolve, ShortID(created.ID), err)
+ }
+
+ address, err := c.NetworkAddress(ctx, created.ID, spec.Network)
+ if err != nil {
+ _ = c.RemoveResolver(ctx, created.ID)
+ return Resolver{}, fmt.Errorf(errResolverAddr, spec.Network, err)
+ }
+
+ return Resolver{
+ ID: created.ID,
+ Name: spec.Name,
+ Network: spec.Network,
+ Address: address,
+ Nonce: spec.Nonce,
+ }, nil
+}
+
+// FindResolver looks for a resolver khaos already has on a network, by LABEL.
+//
+// By label and not by name, because reclaim has to work from a cold start where
+// khaos remembers nothing, and because an operator who renamed one would
+// otherwise get a second resolver on the same network with the target pointed at
+// whichever one it was created against.
+func (c *Client) FindResolver(
+ ctx context.Context,
+ networkName string,
+) (Resolver, error) {
+ resolvers, err := c.Resolvers(ctx)
+ if err != nil {
+ return Resolver{}, err
+ }
+
+ for _, resolver := range resolvers {
+ if resolver.Network == networkName {
+ return resolver, nil
+ }
+ }
+
+ return Resolver{}, fmt.Errorf(errNoMatch, networkName)
+}
+
+// Resolvers lists every chaos resolver on this daemon, which is what `khaos
+// reclaim` sweeps and what `khaos doctor` reports.
+func (c *Client) Resolvers(ctx context.Context) ([]Resolver, error) {
+ // label=KEY=VALUE, not label=KEY. moby's MatchKVList splits on the first
+ // `=` and never compares the value when there is none, so the key-only
+ // filter matched every container carrying the label whatever it said —
+ // demonstrated with `com.docker.compose.project`, which returned 135
+ // containers across every distinct project value.
+ //
+ // That mattered because `RemoveResolver` FORCE-removes what this returns. A
+ // container labelled `khaos.resolver=false`, or labelled by somebody else
+ // entirely, was listed, adopted as khaos's own authority, and removable.
+ // The consent label has `checkConsentNotTrue` for exactly this reason;
+ // this tier took the loose filter and skipped the compensating check.
+ result, err := c.api.ContainerList(ctx, client.ContainerListOptions{
+ All: true,
+ Filters: make(client.Filters).Add(
+ filterLabelKey,
+ ResolverLabel+"="+ResolverValue,
+ ),
+ })
+ if err != nil {
+ return nil, fmt.Errorf(errListResolvers, err)
+ }
+
+ out := make([]Resolver, 0, len(result.Items))
+
+ for _, summary := range result.Items {
+ networkName := summary.Labels[ResolverNetworkLabel]
+
+ resolver := Resolver{
+ ID: summary.ID,
+ Name: containerName(summary.Names),
+ Network: networkName,
+ Nonce: summary.Labels[ResolverNonceLabel],
+ }
+
+ // The address comes from the daemon rather than from the summary,
+ // because a created-but-never-started resolver has none and has to
+ // report empty rather than a stale value from somewhere else.
+ if address, err := c.NetworkAddress(
+ ctx,
+ summary.ID,
+ networkName,
+ ); err == nil {
+ resolver.Address = address
+ }
+
+ out = append(out, resolver)
+ }
+
+ return out, nil
+}
+
+// RemoveResolver takes the container away. A missing container is success: the
+// resolver provably went with it, and reporting that as a failure would wedge a
+// journal record forever and, because `khaos reap` stops on a sweep error,
+// disable the dead man's switch for every other fault on the machine.
+func (c *Client) RemoveResolver(ctx context.Context, containerID string) error {
+ _, err := c.api.ContainerRemove(
+ ctx,
+ containerID,
+ client.ContainerRemoveOptions{
+ Force: true,
+ },
+ )
+
+ if err != nil && errors.Is(c.classify(err), ErrContainerGone) {
+ return nil
+ }
+
+ return err
+}
+
+// resolverArgs is the command line the container runs, and the NONCE is
+// deliberately not on it.
+//
+// Everything else is an argument, because `docker inspect` then shows exactly
+// what the resolver was told and that is how khaos reads its own configuration
+// back instead of remembering it. The nonce is the one field that is a secret,
+// and a command line is the leakiest place to put one: measured on this host,
+// `/proc/<pid>/cmdline` is mode 444 and readable by every local user, while
+// `/proc/<pid>/environ` is 400. Both are equally visible through `docker
+// inspect`, which already needs socket access that is root-equivalent anyway,
+// so the environment satisfies the read-back reason without the leak.
+//
+// What the leak buys an attacker is not academic: the nonce is the sole
+// authorisation for arming and disarming khaos's DNS faults, so any local user
+// could `POST /disarm` after khaos confirmed a fault and khaos would measure an
+// unfaulted window and report a verdict it never earned.
+func resolverArgs(spec ResolverSpec) []string {
+ args := []string{
+ ResolverCommand,
+ "--control", spec.Control,
+ }
+
+ for _, name := range sortedKeys(spec.Zone) {
+ args = append(args, "--zone", name+"="+spec.Zone[name])
+ }
+
+ return args
+}
+
+// resolverEnv carries the one field that must not appear in the host's process
+// table.
+func resolverEnv(spec ResolverSpec) []string {
+ return []string{EnvResolverNonce + "=" + spec.Nonce}
+}
+
+func sortedKeys(values map[string]string) []string {
+ out := make([]string, 0, len(values))
+ for key := range values {
+ out = append(out, key)
+ }
+ slices.Sort(out)
+ return out
+}
+
+func containerName(names []string) string {
+ if len(names) == 0 {
+ return ""
+ }
+ return strings.TrimPrefix(names[0], "/")
+}
+
+func firstLine(payload []byte) string {
+ text := strings.TrimSpace(string(payload))
+ if index := strings.IndexByte(text, '\n'); index > 0 {
+ return text[:index]
+ }
+ return text
+}
+
+// tarSingleFile builds the one-entry layer the import endpoint consumes.
+func tarSingleFile(name string, content []byte) ([]byte, error) {
+ var buffer bytes.Buffer
+
+ writer := tar.NewWriter(&buffer)
+
+ if err := writer.WriteHeader(&tar.Header{
+ Name: strings.TrimPrefix(name, "/"),
+ Mode: resolverBinaryMode,
+ Size: int64(len(content)),
+ }); err != nil {
+ return nil, err
+ }
+
+ if _, err := writer.Write(content); err != nil {
+ return nil, err
+ }
+
+ if err := writer.Close(); err != nil {
+ return nil, err
+ }
+
+ return buffer.Bytes(), nil
+}
+
+// ResolverControlName is the confirmation control khaos generates when the
+// operator did not name one. It is in `.invalid`, which RFC 2606 reserves, so
+// it can never collide with a real name the target needs and can never resolve
+// anywhere but khaos's own resolver.
+func ResolverControlName(networkName string) string {
+ return "control." + networkName + ".khaos.invalid."
+}
+
+// ResolverControlURL is the resolver's control surface on its own container
+// address. Nothing is published to the host; the host simply routes to
+// container addresses on a bridge network, which is what lets khaos arm its
+// resolver without a single host port.
+func ResolverControlURL(address string) string {
+ return fmt.Sprintf("http://%s:%d", address, dnsrv.ControlPort)
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go
new file mode 100644
index 00000000..54586d86
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/dockerx/resolver_test.go
@@ -0,0 +1,100 @@
+// ©AngelaMos | 2026
+// resolver_test.go
+
+package dockerx
+
+import (
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// TestStaticELFRejectsADynamicBinary is the two-sided control for the guard
+// that decides whether `khaos resolver up` can work at all.
+//
+// It matters because the failure it prevents is invisible until the worst
+// moment: khaos imports its own binary as a one-layer image with no base, the
+// container starts, and a dynamically linked binary dies looking for
+// `/lib64/ld-linux-x86-64.so.2` — an error about a loader path the operator
+// never typed, in the middle of an experiment.
+//
+// Both halves are built here rather than asserted against the test binary,
+// because `/proc/self/exe` under `go test` is whatever the toolchain happened
+// to produce and a guard checked against one linkage proves nothing about the
+// other.
+func TestStaticELFRejectsADynamicBinary(t *testing.T) {
+ dir := t.TempDir()
+
+ build := func(cgo string) string {
+ t.Helper()
+
+ out := filepath.Join(dir, "khaos-"+cgo)
+
+ cmd := exec.Command("go", "build", "-o", out, "../../cmd/khaos")
+ cmd.Env = append(os.Environ(), "CGO_ENABLED="+cgo)
+
+ if output, err := cmd.CombinedOutput(); err != nil {
+ t.Skipf(
+ "cannot build with CGO_ENABLED=%s: %v: %s",
+ cgo,
+ err,
+ output,
+ )
+ }
+
+ return out
+ }
+
+ static := build("0")
+
+ if err := staticELF(static); err != nil {
+ t.Errorf("a CGO_ENABLED=0 build was rejected: %v", err)
+ }
+
+ dynamic := build("1")
+
+ err := staticELF(dynamic)
+ if err == nil {
+ t.Fatal(
+ "a dynamically linked binary was accepted as its own container " +
+ "image, so the guard cannot detect the build it exists to refuse",
+ )
+ }
+
+ if !errors.Is(err, ErrDynamicSelf) {
+ t.Errorf("error %v, want %v", err, ErrDynamicSelf)
+ }
+
+ // The refusal has to be accurate about the SCOPE of the limit as well as
+ // name the fix. An earlier wording said the DNS tier does not work, which is
+ // wrong: only CREATING a resolver needs a static binary.
+ for _, want := range []string{"CGO_ENABLED=0", "resolver up", "already exists"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Errorf("the refusal never mentions %q: %v", want, err)
+ }
+ }
+}
+
+// TestStaticELFRefusesWhatItCannotRead keeps a missing or unreadable file from
+// reading as a static binary, which would let the import proceed and fail later.
+func TestStaticELFRefusesWhatItCannotRead(t *testing.T) {
+ if err := staticELF(filepath.Join(t.TempDir(), "absent")); err == nil {
+ t.Fatal("a file that does not exist passed as a static binary")
+ }
+
+ notELF := filepath.Join(t.TempDir(), "notelf")
+ if err := os.WriteFile(
+ notELF,
+ []byte("#!/bin/sh\necho hi\n"),
+ 0o600,
+ ); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+
+ if err := staticELF(notELF); err == nil {
+ t.Fatal("a shell script passed as a static binary")
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
index 4c982f52..4518892d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/catalog.go
@@ -11,6 +11,7 @@ import (
"strings"
"time"
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
"github.com/CarterPerez-dev/khaos/internal/idp"
"github.com/CarterPerez-dev/khaos/internal/inject"
"github.com/CarterPerez-dev/khaos/internal/inject/proxy"
@@ -23,8 +24,8 @@ const (
NameDependencyOutage = "network.dependency_outage"
- // A credential no correct verifier can accept: it is not a signed token in
- // any format, so a target that honours it is not verifying anything.
+ // Not a signed token in any format, so a target that honours it is not
+ // verifying anything.
DefaultInvalidToken = "khaos.invalid.credential.not.a.signed.token"
probeValidName = "valid-credential"
@@ -45,61 +46,50 @@ var ErrStrayParam = errors.New(
)
// Shape is the part of an experiment that depends on the operator's parameters.
-// Identity, polarity, fault kind and timing all come from the declaration, so a
-// builder cannot disagree with the catalogue about what it is or forget to carry
-// something universal through.
+// Identity, polarity, fault kind and timing come from the declaration instead, so
+// a builder cannot disagree with the catalogue about what it is.
type Shape struct {
Condition string
Target string
Probes probe.Pair
- // ProxyFault is set only by an experiment that injects through khaos's own
- // fault proxy. The experiment owns the magnitude because the operator declares
- // it as a parameter, and the CLI must not have to infer a fault from a kind.
+ // The experiment owns the magnitude because the operator declares it as a
+ // parameter, so the CLI never has to infer a fault from a kind.
ProxyFault *proxy.Fault
- // ProxyListen and ProxyUpstream are where the run binds the proxy and what
- // it forwards to. They come from declared parameters, so an operator can
- // point a proxy experiment at their own stack.
ProxyListen string
ProxyUpstream string
- // IdP and IdPTrap are set only by an experiment that arms khaos's own
- // identity provider, and the provider is built by the builder because its
- // probes mint out of it.
+ // Built by the builder because its probes mint out of it.
IdP *idp.Provider
IdPTrap *idp.Trap
- // Authority and PKITrap are set only by an experiment that arms khaos's own
- // certificate authority, which plays the dependency whose certificate is
- // under test.
Authority *pki.Authority
PKITrap *pki.Trap
- // AuthorityBase is the URL the TARGET uses to reach the authority's CRL and
- // OCSP endpoints, and AuthorityListen is where khaos binds them. They are
- // separate because the target addresses khaos by a name khaos cannot see
- // from its own listener, and deriving one from the other is the identity
- // provider's most expensive bug.
+ DNSTrap *dnsrv.Trap
+ ResolverNetwork string
+ DNSHost string
+
+ // AuthorityBase is the URL the TARGET reaches the CRL and OCSP endpoints by,
+ // AuthorityListen where khaos binds them. They are separate because the
+ // target addresses khaos by a name khaos cannot see from its own listener.
AuthorityBase string
AuthorityListen string
AuthorityTLS string
- // Defect names what khaos did to the credential, for the verdict's wording.
- Defect string
-
- // HeldCaveat qualifies a HELD verdict that covers less than it appears to.
+ Defect string
HeldCaveat string
- // Consulted reads how many times the target has reached the endpoint this
- // experiment faults, and ConsultedWhat names it. Together they are the
- // liveness guard for a fault whose landing the probe streams cannot see.
- Consulted func() int64
+ // Consulted reads how many times the target has reached the faulted endpoint,
+ // and ConsultedWhat names it; together they are the liveness guard for a fault
+ // whose landing the probe streams cannot see.
+ Consulted func() (int64, error)
ConsultedWhat string
- // Unsupportable names the check whose absence is itself the finding, which
- // turns a never-consulted endpoint into UNSUPPORTED rather than a void run.
Unsupportable string
+ CacheRequired string
+ DeliveredWhat string
// IdleFor asks the engine for the idleness discriminator.
IdleFor time.Duration
@@ -278,6 +268,11 @@ func Build(
Consulted: shape.Consulted,
ConsultedWhat: shape.ConsultedWhat,
Unsupportable: shape.Unsupportable,
+ CacheRequired: shape.CacheRequired,
+ DeliveredWhat: shape.DeliveredWhat,
+ DNSTrap: shape.DNSTrap,
+ ResolverNetwork: shape.ResolverNetwork,
+ DNSHost: shape.DNSHost,
IdleFor: shape.IdleFor,
Act: shape.Act,
ActDetail: shape.ActDetail,
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go
new file mode 100644
index 00000000..aa705822
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns.go
@@ -0,0 +1,161 @@
+// ©AngelaMos | 2026
+// dns.go
+
+package experiment
+
+import "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+
+// The `dns.` family. Each entry is a row of research 05 section 9's injectable
+// fault matrix, and the two khaos cannot deliver are entries as well, so that
+// asking for one gets the mechanism and a fault to run instead rather than
+// silence.
+//
+// What this family tests that no other one can: the dependency is HEALTHY and
+// the target cannot find it, or finds something else. A paused container and a
+// severed path both make a dependency unreachable, and a broken certificate
+// makes it unverifiable. Name resolution failing is a third branch, and research
+// 05 section 10 is the argument for why the fail-open lives there: a control
+// whose allowlist entry is a hostname has to resolve that name to make a
+// decision, which moves its trust boundary into DNS, which nobody operating it
+// controls.
+const (
+ NameDNSNXDomain = "dns.nxdomain"
+ NameDNSServFail = "dns.servfail"
+ NameDNSSlowResponse = "dns.slow_response"
+ NameDNSWrongAddress = "dns.wrong_address"
+ NameDNSCNAMEChain = "dns.cname_chain"
+ NameDNSCNAMELoop = "dns.cname_loop"
+ NameDNSTruncation = "dns.truncation"
+ NameDNSTruncationNoTCP = "dns.truncation_no_tcp"
+ NameDNSTTLZero = "dns.ttl_zero"
+ NameDNSTTLLong = "dns.ttl_long"
+ NameDNSRefused = "dns.refused"
+ NameDNSTimeout = "dns.timeout"
+)
+
+const (
+ summaryNXDomain = "answer the dependency's name with a definite NXDOMAIN and find out whether a control that cannot evaluate its own allowlist treats that as permission"
+
+ summaryServFail = "answer the dependency's name with a TEMPORARY failure, which clients retry where they would have given up on NXDOMAIN, and find out whether a transient resolution error becomes a decision or a retry storm"
+
+ summarySlowResponse = "hold the target's lookup of its dependency almost to the embedded resolver's 4 second ceiling and find out whether a slow name is treated as an absent one"
+
+ summaryWrongAddress = "answer the dependency's name with an address nobody approved and find out what the target's control does about it, which is the one fault where a hostname allowlist can actively authorise a host rather than merely stop working"
+
+ summaryCNAMEChain = "answer the dependency's name with a multi-hop CNAME ending somewhere else entirely, which asks whether name matching survives indirection out of the zone that was asked for"
+
+ summaryCNAMELoop = "answer the dependency's name with a CNAME cycle and find out what the target's own loop handling does with it"
+
+ summaryTruncation = "set the truncation bit on the dependency's lookup and answer completely only over TCP, and find out whether the target makes the RFC 5966 retry at all"
+
+ summaryTruncationNoTCP = "set the truncation bit and refuse to complete over TCP either, which is what a deny-all-TCP-53 egress rule looks like from inside, and find out whether the target can resolve anything that does not fit in a UDP packet"
+
+ summaryTTLZero = "answer the dependency's name correctly with a zero TTL so nothing may cache it, and measure whether the target re-resolves per request, which is both a DoS amplifier and the precondition for every other caching question"
+
+ summaryTTLLong = "answer the dependency's name correctly with a week-long TTL and measure whether the target honours it, which is how long a stale answer would keep being trusted after the dependency moved"
+
+ summaryRefused = "REFUSED through Docker's embedded resolver, which khaos refuses to approximate; the resolver rewrites it to SERVFAIL and a real library branches on the rcode"
+
+ summaryTimeout = "a dropped query through Docker's embedded resolver, which khaos refuses to approximate; the resolver answers SERVFAIL itself at a measured 4.001s and a client never sees a socket timeout"
+)
+
+// dnsCatalogue is the whole family in one list, so a census can walk it and the
+// catalogue, the refusals and the gate cannot disagree about what exists.
+var dnsCatalogue = []dnsSpec{
+ {
+ name: NameDNSNXDomain,
+ summary: summaryNXDomain,
+ fault: dnsrv.NXDomain,
+ },
+ {
+ name: NameDNSServFail,
+ summary: summaryServFail,
+ fault: dnsrv.ServFail,
+ },
+ {
+ name: NameDNSSlowResponse,
+ summary: summarySlowResponse,
+ fault: dnsrv.SlowResponse,
+ extra: []Param{dnsDelayParam},
+ },
+ {
+ name: NameDNSWrongAddress,
+ summary: summaryWrongAddress,
+ fault: dnsrv.WrongAddress,
+ extra: []Param{dnsAddressParam},
+ },
+ {
+ name: NameDNSCNAMEChain,
+ summary: summaryCNAMEChain,
+ fault: dnsrv.CNAMEChain,
+ extra: []Param{dnsAddressParam},
+ },
+ {
+ name: NameDNSCNAMELoop,
+ summary: summaryCNAMELoop,
+ fault: dnsrv.CNAMELoop,
+ },
+ {
+ name: NameDNSTruncation,
+ summary: summaryTruncation,
+ fault: dnsrv.Truncation,
+ },
+ {
+ name: NameDNSTruncationNoTCP,
+ summary: summaryTruncationNoTCP,
+ fault: dnsrv.TruncationNoTCP,
+ },
+ {
+ name: NameDNSTTLZero,
+ summary: summaryTTLZero,
+ fault: dnsrv.TTLZero,
+ needsCache: true,
+ },
+ {
+ name: NameDNSTTLLong,
+ summary: summaryTTLLong,
+ fault: dnsrv.TTLLong,
+ needsCache: true,
+ },
+ {
+ name: NameDNSRefused,
+ summary: summaryRefused,
+ fault: dnsrv.Refused,
+ },
+ {
+ name: NameDNSTimeout,
+ summary: summaryTimeout,
+ fault: dnsrv.Timeout,
+ },
+}
+
+func init() {
+ for _, spec := range dnsCatalogue {
+ catalog[spec.name] = dnsDefinition(spec)
+ }
+}
+
+// DNSNames is the family, in catalogue order, for a census that has to prove it
+// ran every one.
+func DNSNames() []string {
+ out := make([]string, 0, len(dnsCatalogue))
+ for _, spec := range dnsCatalogue {
+ out = append(out, spec.name)
+ }
+ return out
+}
+
+// DNSDeliverable splits the family into the faults khaos serves and the ones it
+// refuses by name, derived from the resolver's own list so the catalogue cannot
+// drift from what the resolver will actually accept.
+func DNSDeliverable() (deliverable, refused []string) {
+ for _, spec := range dnsCatalogue {
+ if dnsrv.Deliverable(spec.fault) {
+ deliverable = append(deliverable, spec.name)
+ continue
+ }
+ refused = append(refused, spec.name)
+ }
+
+ return deliverable, refused
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns_verdict_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns_verdict_test.go
new file mode 100644
index 00000000..c306fe6d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dns_verdict_test.go
@@ -0,0 +1,219 @@
+// ©AngelaMos | 2026
+// dns_verdict_test.go
+
+package experiment
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/probe"
+)
+
+const deliveredResolveWhat = "re-resolve its dependency's name"
+
+// faultWindows are the two shapes every test below starts from: one where the
+// invalid credential starts being accepted under the fault, and one where
+// nothing changes.
+func permittedFaultWindow() probe.Window {
+ return probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired*3)...),
+ Invalid: stream(
+ probe.Invalid,
+ append(
+ repeat(probe.Rejected, 1),
+ repeat(probe.Accepted, MinimumPaired*3)...,
+ )...,
+ ),
+ }
+}
+
+func heldFaultWindow() probe.Window {
+ return probe.Window{
+ Valid: stream(probe.Valid, repeat(probe.Accepted, MinimumPaired*3)...),
+ Invalid: stream(
+ probe.Invalid,
+ repeat(probe.Rejected, MinimumPaired*3)...,
+ ),
+ }
+}
+
+// TestAnExperimentWithoutThePauseSaysNothingAboutIt is the control for the guard
+// in diagnose.
+//
+// Without it every PERMITTED verdict in the three families that do not use the
+// idleness pause reported on a resumed window khaos never collected, which reads
+// as khaos having tried the pause and learned nothing from it.
+func TestAnExperimentWithoutThePauseSaysNothingAboutIt(t *testing.T) {
+ observation := Observation{
+ Baseline: healthyBaseline(),
+ Fault: permittedFaultWindow(),
+ Recovery: healthyRecovery(),
+ FaultConfirmed: true,
+ ConfirmMethod: "measured",
+ Delivered: 3,
+ DeliveredWhat: deliveredResolveWhat,
+ }
+
+ verdict := Judge(observation, dnsPolarity)
+
+ if verdict.Direction != Permitted {
+ t.Fatalf("direction %s, want %s", verdict.Direction, Permitted)
+ }
+
+ if verdict.Cause != Undiagnosed {
+ t.Errorf("cause %s, want %s", verdict.Cause, Undiagnosed)
+ }
+
+ for _, reason := range verdict.Reasons {
+ if strings.Contains(reason, "resumed window") {
+ t.Errorf(
+ "an experiment that never paused traffic reported on the pause: %q",
+ reason,
+ )
+ }
+ }
+}
+
+// TestAFaultTheTargetNeverMetIsAbortedNotHeld is the control for the delivery
+// guard, and the live run of this tier is what made it necessary.
+//
+// A DNS fault can be armed, confirmed at khaos's own resolver, and reach nothing
+// at all: Go's HTTP transport resolves a name once per kept-alive connection, so
+// a target holding a live connection never asks again. Measured on the range's
+// own gateway at two lookups across forty-six requests. Reporting that as HELD
+// is a clean survival khaos never earned.
+func TestAFaultTheTargetNeverMetIsAbortedNotHeld(t *testing.T) {
+ observation := Observation{
+ Baseline: healthyBaseline(),
+ Fault: heldFaultWindow(),
+ Recovery: healthyRecovery(),
+ FaultConfirmed: true,
+ ConfirmMethod: "measured",
+ Consulted: 2,
+ ConsultedWhat: ConsultsResolver,
+
+ Delivered: 0,
+ DeliveredWhat: deliveredResolveWhat,
+ }
+
+ verdict := Judge(observation, dnsPolarity)
+
+ if verdict.Epistemic != Aborted {
+ t.Fatalf(
+ "epistemic %s, want %s: a fault the target never met is not a survival",
+ verdict.Epistemic,
+ Aborted,
+ )
+ }
+
+ if verdict.Conclusive {
+ t.Error("a fault the target never met produced a conclusive verdict")
+ }
+
+ // The SAME observation with delivery has to reach the ordinary reading, or
+ // the guard is not reading the count it claims to and would abort every run.
+ observation.Delivered = MinimumPaired
+
+ delivered := Judge(observation, dnsPolarity)
+
+ if delivered.Epistemic == Aborted {
+ t.Errorf(
+ "a delivered fault was also reported as never met (%s), so the guard "+
+ "is not reading the delivery count",
+ strings.Join(delivered.Reasons, "; "),
+ )
+ }
+}
+
+// TestTheDeliveryGuardIsOptional keeps the new field from changing every other
+// family's verdict. An experiment that does not declare DeliveredWhat has no
+// delivery question, and a zero count must mean nothing there.
+func TestTheDeliveryGuardIsOptional(t *testing.T) {
+ observation := Observation{
+ Baseline: healthyBaseline(),
+ Fault: heldFaultWindow(),
+ Recovery: healthyRecovery(),
+ FaultConfirmed: true,
+ ConfirmMethod: "measured",
+ Delivered: 0,
+ }
+
+ verdict := Judge(observation, outagePolarity)
+
+ if verdict.Epistemic == Aborted {
+ t.Fatalf(
+ "an experiment that declares no delivery question was aborted for "+
+ "not answering it: %s",
+ strings.Join(verdict.Reasons, "; "),
+ )
+ }
+}
+
+// TestNoCacheIsUnsupportedNotSurvived is the control for CacheRequired. A target
+// that re-resolves on every request has no cache lifetime to honour or ignore,
+// so the two TTL faults have nothing to measure in it, and reporting that as a
+// survival claims the target withstood something it never experienced.
+func TestNoCacheIsUnsupportedNotSurvived(t *testing.T) {
+ baseline := healthyBaseline()
+
+ observation := Observation{
+ Baseline: baseline,
+ Fault: heldFaultWindow(),
+ Recovery: healthyRecovery(),
+ FaultConfirmed: true,
+ ConfirmMethod: "measured",
+ ConsultedWhat: ConsultsResolver,
+ CacheRequired: cacheRequiredTTL,
+
+ // Two whole lookups inside a baseline measured in seconds, against the
+ // ten-minute lifetime khaos serves. A target honouring that resolves
+ // once for the entire window.
+ Consulted: queriesPerLookup * cacheLookupGrace,
+
+ Delivered: 1,
+ DeliveredWhat: deliveredResolveWhat,
+ }
+
+ verdict := Judge(observation, dnsPolarity)
+
+ if verdict.Epistemic != Unsupported {
+ t.Fatalf("epistemic %s, want %s", verdict.Epistemic, Unsupported)
+ }
+
+ // A target that DID cache has to reach the ordinary reading, or the
+ // threshold is rejecting every run rather than the uncached ones.
+ observation.Consulted = 1
+
+ cached := Judge(observation, dnsPolarity)
+
+ if cached.Epistemic == Unsupported {
+ t.Errorf(
+ "a target that resolved once across the whole baseline was reported "+
+ "as having no cache: %s",
+ strings.Join(cached.Reasons, "; "),
+ )
+ }
+
+ // The BOUNDARY, both sides, because the whole defect was an off-by-four in
+ // this comparison and a threshold with no control at its edge is a number
+ // nobody checked.
+ threshold := int64(queriesPerLookup * cacheLookupGrace)
+
+ observation.Consulted = threshold - 1
+ if Judge(observation, dnsPolarity).Epistemic == Unsupported {
+ t.Errorf(
+ "one query below the threshold was reported as no cache at all (%d of %d)",
+ threshold-1,
+ threshold,
+ )
+ }
+
+ observation.Consulted = threshold
+ if Judge(observation, dnsPolarity).Epistemic != Unsupported {
+ t.Errorf(
+ "exactly at the threshold was not reported as uncached (%d)",
+ threshold,
+ )
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go
new file mode 100644
index 00000000..cc5ef193
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/dnstier.go
@@ -0,0 +1,301 @@
+// ©AngelaMos | 2026
+// dnstier.go
+
+package experiment
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+ "github.com/CarterPerez-dev/khaos/internal/inject"
+)
+
+const (
+ FamilyDNS = "dns"
+
+ // ParamDNSHost is the dependency's NAME, and it is the parameter that makes
+ // this family work at all.
+ //
+ // It has to be a name khaos's resolver is authoritative for, and a name the
+ // target's own embedded resolver cannot answer. A container name or network
+ // alias is answered locally out of the embedded resolver's own table and
+ // never forwarded, so a fault aimed at one would be armed, never seen, and
+ // reported as a clean survival.
+ ParamDNSHost = "dns-host"
+
+ // ParamResolverNetwork is the Docker network khaos's resolver serves, which
+ // must be the target's own. khaos finds the resolver there by label rather
+ // than being told where it is.
+ ParamResolverNetwork = "resolver-network"
+
+ // ParamDNSAddress is the address the redirect faults answer with, and
+ // ParamDNSDelay is how long the slow fault holds a lookup.
+ ParamDNSAddress = "dns-address"
+ ParamDNSDelay = "dns-delay"
+
+ usageDNSHost = "the dependency's DNS name, which khaos's resolver must be authoritative for; it cannot be a container name or network alias, because the target's own embedded resolver answers those locally and never forwards them"
+ usageResolverNetwork = "the Docker network khaos's chaos resolver serves, which must be the target's own network"
+ usageDNSAddress = "the address the redirect faults answer with; point it at a service that behaves differently from the real dependency and the fault becomes a fail-open demonstration rather than an outage"
+ usageDNSDelay = "how long to hold the target's lookup; the ceiling is 3.999s, because Docker's embedded resolver answers SERVFAIL itself at a measured 4.001s and a larger delay would be a different fault"
+
+ dnsCondition = "%s resolving its dependency through a resolver khaos has made wrong: %s"
+
+ dnsResponse = "continuing to refuse credentials it would have refused anyway, rather than letting them through because it could not find its dependency"
+
+ // ConsultsResolver is the endpoint the liveness guard names, as an operator
+ // reads it in a verdict.
+ ConsultsResolver = "khaos's chaos resolver"
+
+ // cacheRequiredTTL is what the TTL faults declare they need. A target with
+ // no cached answer has no lifetime to honour or ignore, so the TTL khaos
+ // serves governs nothing it does.
+ cacheRequiredTTL = "the lifetime of a cached DNS answer"
+
+ // caveatRedirectUnseen is what khaos says instead of claiming the target
+ // verified the host it reached. khaos has no instrument at the far end of an
+ // operator-chosen address.
+ caveatRedirectUnseen = "khaos served the redirect and measured that the target re-resolved, and it did NOT observe which host the target then connected to: the redirect address is an operator parameter and khaos has nothing listening at the far end of it. So this HELD means the target's control was not disturbed, and it does not establish that the target verified who it reached. If the address you redirected to answers at all, check its own logs for the target's requests before reading this as a pass"
+
+ // deliveredResolve is what the target has to DO for a DNS fault to reach it,
+ // phrased as the operator reads it in a verdict.
+ deliveredResolve = "re-resolve its dependency's name"
+)
+
+// dnsPolarity is PERMITTED alone, and the two directions it leaves out are
+// deliberate.
+//
+// DENIED is the correct outcome with an availability cost. A target that cannot
+// resolve its identity provider and therefore refuses everything has failed
+// closed; research 05 section 10 calls that bad in a different way -- DNS
+// becomes a single point of failure and anyone who can break it can deny the
+// service -- but it is not a security defect and khaos will not report it as
+// one.
+//
+// HELD is not a defect either, and that is the opposite of the `tls.` family's
+// polarity for a reason worth stating. There, carrying on unchanged means the
+// target USED a certificate khaos deliberately broke. Here, carrying on
+// unchanged means the target's control was not disturbed by its dependency
+// moving or vanishing, which is what a correct control does. The case where
+// carrying on hides something -- the target connected to the address khaos
+// redirected it to and that address approved everything -- does not surface as
+// HELD at all: it surfaces as the invalid credential being accepted, which is
+// PERMITTED.
+var dnsPolarity = Polarity{
+ Defects: []Direction{Permitted},
+ Rationale: "a control that stops rejecting invalid credentials because it could not resolve its dependency has moved its trust boundary into DNS, which nobody operating it controls; refusing to serve while continuing to enforce is the correct outcome with an availability cost, not a defect",
+}
+
+func dnsParams(extra ...Param) []Param {
+ return append([]Param{
+ {Name: ParamTarget, Usage: usageTarget, Required: true},
+ {Name: ParamURL, Usage: usageURL, Required: true},
+ {Name: ParamToken, Usage: usageToken, Required: true},
+ {Name: ParamDNSHost, Usage: usageDNSHost, Required: true},
+ {
+ Name: ParamResolverNetwork,
+ Usage: usageResolverNetwork,
+ Required: true,
+ },
+ {
+ Name: ParamInvalidToken,
+ Usage: usageInvalidToken,
+ Default: DefaultInvalidToken,
+ },
+ }, extra...)
+}
+
+var dnsAddressParam = Param{
+ Name: ParamDNSAddress,
+ Usage: usageDNSAddress,
+ Default: dnsrv.DefaultWrongAddress,
+}
+
+var dnsDelayParam = Param{
+ Name: ParamDNSDelay,
+ Usage: usageDNSDelay,
+ Default: dnsrv.DefaultDelay.String(),
+}
+
+// dnsSpec is one DNS experiment's declaration. Everything not derived from a
+// parameter lives here, so a builder cannot restate its own name or forget its
+// polarity.
+type dnsSpec struct {
+ name string
+ summary string
+ fault dnsrv.Kind
+
+ // needsCache marks the two experiments whose question only applies to a
+ // target that caches. It turns a target re-resolving per request into
+ // UNSUPPORTED rather than a clean survival it never earned.
+ needsCache bool
+
+ extra []Param
+}
+
+// dnsDefinition turns one declaration into a catalogue entry. Every DNS
+// experiment goes through here, so none of them can forget the liveness guard
+// that makes its verdict interpretable.
+func dnsDefinition(spec dnsSpec) Definition {
+ return Definition{
+ Name: spec.name,
+ Family: FamilyDNS,
+ Summary: spec.summary,
+ Hypothesis: Hypothesis{Condition: outageGeneric, Response: dnsResponse},
+ Polarity: dnsPolarity,
+ FaultKind: inject.DNSKind(spec.fault),
+ Params: dnsParams(spec.extra...),
+ Build: buildDNS(spec),
+ }
+}
+
+func buildDNS(spec dnsSpec) func(*Params) (Shape, error) {
+ return func(params *Params) (Shape, error) {
+ target := params.Get(ParamTarget)
+ host := params.Get(ParamDNSHost)
+
+ trap := dnsrv.Trap{Kind: spec.fault, Host: host}
+
+ if err := readDNSMagnitude(params, spec, &trap); err != nil {
+ return Shape{}, err
+ }
+
+ // SHAPE only. A delay above the measured ceiling is refused here,
+ // because that is an operator typing a number and they should hear it
+ // immediately. Whether khaos will DELIVER the fault is not checked here:
+ // the two masked faults are catalogued in order to be explained, so they
+ // build, plan and describe like any other entry and are refused when
+ // khaos is asked to arm one.
+ if err := trap.ValidateShape(); err != nil {
+ return Shape{}, err
+ }
+
+ shape := Shape{
+ Condition: fmt.Sprintf(
+ dnsCondition,
+ target,
+ trap.Describe(target),
+ ),
+ Target: target,
+ Probes: probePair(params),
+ DNSTrap: &trap,
+ ResolverNetwork: params.Get(ParamResolverNetwork),
+ DNSHost: host,
+ Defect: spec.summary,
+
+ // The proof-of-arrival control, and it is not optional. Research 05
+ // section 12 states the reason: without it khaos cannot tell "the
+ // control failed closed" from "the fault never reached the target",
+ // and the second is the likely one, because `--dns` is fixed at
+ // container creation and Docker offers no way to change it on a
+ // running container. A target that was not recreated against khaos's
+ // resolver never sends it a single query.
+ ConsultedWhat: ConsultsResolver,
+
+ // The caveat on a HELD verdict for the two REDIRECT faults, and it
+ // is load-bearing rather than hedging.
+ //
+ // khaos measures that it served the wrong address and that the
+ // target re-resolved. It does NOT measure which host the target
+ // then connected to, because the redirect address is an operator
+ // parameter and khaos has no instrument at the far end of it.
+ //
+ // That gap decides the polarity. Against the default documentation
+ // address, with nothing listening, HELD honestly means the target
+ // held through an unreachable dependency, which is correct
+ // behaviour. Against an address that ANSWERS and mirrors the real
+ // dependency's decisions, the same HELD means the target moved its
+ // trust to a host nobody approved and khaos cleared it -- which is
+ // the `tls.` family's rule one domain over, where carrying on
+ // unchanged means the target USED the thing khaos broke.
+ //
+ // A blanket defect declaration would be wrong for the first case
+ // and silence would be wrong for the second, so khaos states what
+ // it did not observe.
+ HeldCaveat: heldCaveatFor(spec),
+
+ // The DELIVERY guard, which no other tier needs. Every other
+ // fault acts on a path the target cannot avoid using; a DNS
+ // answer is one it can simply not ask for again, and a target
+ // holding a kept-alive connection does exactly that.
+ //
+ // The TTL pair is EXEMPT and that is not an oversight. For every
+ // other fault, a target that did not re-resolve never met the
+ // fault. For these two, not re-resolving is the measurement: the
+ // experiment is about how long a cached answer is honoured, so a
+ // target that re-asks less is a target that cached more. Applying
+ // the guard there aborted the only run the experiment exists for,
+ // and its own remediation ("run this against a service that
+ // re-resolves") contradicted cacheNotObserved's four lines
+ // earlier in the same verdict.
+ DeliveredWhat: deliveredFor(spec),
+ }
+
+ if spec.needsCache {
+ shape.CacheRequired = cacheRequiredTTL
+ }
+
+ return shape, nil
+ }
+}
+
+// readDNSMagnitude reads whichever parameter this experiment declared. Params
+// refuses a read of a name the experiment never declared, so this cannot read a
+// flag that is not there.
+func readDNSMagnitude(
+ params *Params,
+ spec dnsSpec,
+ trap *dnsrv.Trap,
+) error {
+ for _, param := range spec.extra {
+ switch param.Name {
+ case ParamDNSAddress:
+ trap.Address = params.Get(ParamDNSAddress)
+
+ case ParamDNSDelay:
+ delay, err := params.Duration(ParamDNSDelay)
+ if err != nil {
+ return err
+ }
+
+ trap.Delay = delay
+
+ default:
+ // A parameter this tier does not read, which is legal: the shared
+ // set carries several.
+ }
+ }
+
+ return nil
+}
+
+// heldCaveatFor returns the qualification a HELD verdict needs, or none.
+//
+// Only the two faults that MOVE the dependency's address carry one, because
+// they are the only two where carrying on unchanged might mean the target
+// reached somewhere khaos cannot see.
+func heldCaveatFor(spec dnsSpec) string {
+ if spec.fault != dnsrv.WrongAddress && spec.fault != dnsrv.CNAMEChain {
+ return ""
+ }
+
+ return caveatRedirectUnseen
+}
+
+// deliveredFor returns the delivery question this experiment asks, or none.
+//
+// It is derived from the same `needsCache` declaration that turns on the cache
+// reading, so the two cannot disagree about which faults are about caching.
+func deliveredFor(spec dnsSpec) string {
+ if spec.needsCache {
+ return ""
+ }
+
+ return deliveredResolve
+}
+
+// DNSDelayCeiling is the measured bound an operator sees in a refusal, exported
+// so the CLI and the gate name the same number the resolver enforces.
+func DNSDelayCeiling() time.Duration {
+ return dnsrv.MaxDelay
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
index 349fa628..92cb50e3 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/engine.go
@@ -115,7 +115,7 @@ func (e Engine) measureBaseline(
) Observation {
e.emit(StageBaseline, detailBaseline)
- consultedBefore := e.consulted(spec)
+ before, beforeErr := e.consulted(spec)
observation.Baseline = e.sample(
ctx,
@@ -124,9 +124,21 @@ func (e Engine) measureBaseline(
timing.SampleInterval,
)
+ after, afterErr := e.consulted(spec)
+
observation.ConsultedWhat = spec.ConsultedWhat
observation.Unsupportable = spec.Unsupportable
- observation.Consulted = e.consulted(spec) - consultedBefore
+ observation.CacheRequired = spec.CacheRequired
+ observation.Consulted = after - before
+
+ // EITHER end failing voids the count, and a negative delta is impossible
+ // from a monotonic counter so it can only mean khaos misread one end. Both
+ // are recorded rather than silently absorbed, because the guards downstream
+ // read zero as a statement about the TARGET.
+ if beforeErr != nil || afterErr != nil || observation.Consulted < 0 {
+ observation.CounterUnreadable = errors.Join(beforeErr, afterErr)
+ observation.Consulted = 0
+ }
return observation
}
@@ -291,7 +303,40 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
return e.abandon(ctx, spec, result, observation, actErr)
}
+ observation = e.measureFault(ctx, spec, timing, observation)
+
+ observation = e.discriminate(ctx, spec, observation, timing)
+
+ observation, revertErr := e.restore(ctx, spec, timing, observation)
+
+ if ctx.Err() != nil {
+ observation = markInterrupted(observation)
+ }
+
+ return e.conclude(result, observation, spec.Polarity, revertErr)
+}
+
+// measureFault samples under the fault and records whether the target ever MET
+// it, which is a different question from whether khaos armed it.
+//
+// The baseline's count asks whether the target uses the thing khaos faulted at
+// all; this asks whether it went back to it WHILE the fault was armed. A DNS
+// fault is the case that forced the distinction: it can be armed, confirmed at
+// khaos's own resolver, and reach nothing, because Go's HTTP transport resolves
+// a name once per kept-alive connection. Measured on the range's own gateway at
+// two lookups across forty-six requests. Every other tier collapses the two
+// questions safely, because a severed path or a broken certificate sits on a
+// route the target cannot avoid using.
+func (e Engine) measureFault(
+ ctx context.Context,
+ spec Experiment,
+ timing Timing,
+ observation Observation,
+) Observation {
e.emit(StageObserve, detailObserve)
+
+ before, beforeErr := e.consulted(spec)
+
observation.Fault = e.sample(
ctx,
spec.Probes,
@@ -299,15 +344,21 @@ func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
timing.SampleInterval,
)
- observation = e.discriminate(ctx, spec, observation, timing)
+ after, afterErr := e.consulted(spec)
- observation, revertErr := e.restore(ctx, spec, timing, observation)
+ observation.Delivered = after - before
+ observation.DeliveredWhat = spec.DeliveredWhat
- if ctx.Err() != nil {
- observation = markInterrupted(observation)
+ if beforeErr != nil || afterErr != nil || observation.Delivered < 0 {
+ observation.CounterUnreadable = errors.Join(
+ observation.CounterUnreadable,
+ beforeErr,
+ afterErr,
+ )
+ observation.Delivered = 0
}
- return e.conclude(result, observation, spec.Polarity, revertErr)
+ return observation
}
// discriminate is the idleness phase: stop sending traffic for longer than any
@@ -382,9 +433,16 @@ func (e Engine) conclude(
// consulted reads the experiment's endpoint counter, or zero when it declares
// none.
-func (e Engine) consulted(spec Experiment) int64 {
+// consulted reads the experiment's counter, and a FAILURE to read is reported
+// rather than folded into zero.
+//
+// Zero is the sentinel three separate guards branch on, so a swallowed error
+// becomes "the target never did it" and khaos blames the target for khaos's own
+// unreachable instrument. An experiment with no counter is a different thing
+// again and returns zero with no error, because it declared no question.
+func (e Engine) consulted(spec Experiment) (int64, error) {
if spec.Consulted == nil {
- return 0
+ return 0, nil
}
return spec.Consulted()
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
index 7e09fddb..1d1507b1 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/experiment.go
@@ -10,6 +10,7 @@ import (
"strings"
"time"
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
"github.com/CarterPerez-dev/khaos/internal/idp"
"github.com/CarterPerez-dev/khaos/internal/inject/proxy"
"github.com/CarterPerez-dev/khaos/internal/pki"
@@ -21,31 +22,14 @@ const (
DefaultSampleInterval = 500 * time.Millisecond
DefaultBaselineFor = 5 * time.Second
- // Long enough to clear MinimumToClaimClosed at the default sample interval,
- // with room for the jitter. At ten seconds a healthy target produced about
- // twenty paired samples against a threshold of thirty, so every default run
- // carried `Conclusive=false` and a thin-evidence warning — and exited 0
- // anyway. Raising the window is the honest lever: it buys confidence with
- // time rather than by probing the target harder.
+ // Long enough to clear MinimumToClaimClosed at the default sample interval.
DefaultFaultFor = 20 * time.Second
DefaultRecoveryFor = 5 * time.Second
)
-// Polarity declares which way breaking is a defect for the control under test.
-// Failing open is not universally a defect: an authentication check that stops
-// enforcing is catastrophic, while a rate-limit counter that stops counting
-// when its datastore is unreachable is a trade plenty of teams make on purpose.
-// Defects is a SET, not one direction, and the single-direction version was a
-// live bug. `identity.key_rotation` declares DENIED its defect, so PERMITTED
-// fell to the else branch and khaos printed "this experiment declares PERMITTED
-// to be acceptable for this control" — which nobody declared, and which is not
-// true of any control khaos ships. The run then exited 0, because the exit code
-// keys off Defect alone, so a target that failed wide open passed a CI build.
-//
-// The axis still means what the design contract says: failing open is not
-// universally a defect, and a rate limiter that stops counting when its
-// datastore is unreachable is a trade teams make on purpose. That is why this is
-// a declared set rather than a hardcoded rule.
+// Polarity declares which directions of breakage are defects for this control.
+// It is a set, not one direction: failing open is not universally a defect, so
+// each control declares its own.
type Polarity struct {
Defects []Direction
Rationale string
@@ -118,84 +102,73 @@ type Experiment struct {
Probes probe.Pair
Timing Timing
- // ProxyFault is non-nil only for an experiment that injects through khaos's
- // own fault proxy, and it is what tells the run to stand one up at all.
+ // A non-nil ProxyFault is what tells the run to stand a fault proxy up at all.
ProxyFault *proxy.Fault
- // ProxyListen is the host address the run binds the proxy on, and
- // ProxyUpstream is the dependency it forwards to.
+ // ProxyListen is where the run binds the proxy, ProxyUpstream what it
+ // forwards to.
ProxyListen string
ProxyUpstream string
- // IdP and IdPTrap are non-nil only for an experiment that arms khaos's own
- // identity provider. The provider is BUILT here rather than by the run,
- // because the probes have to mint credentials out of it and minting needs no
- // listener: the run only has to bind it where the target is already looking.
+ // The provider is built by the catalogue rather than the run, because the
+ // probes mint credentials out of it and minting needs no listener.
IdP *idp.Provider
IdPTrap *idp.Trap
- // Authority and PKITrap are non-nil only for an experiment that arms
- // khaos's own certificate authority. AuthorityBase is the URL the TARGET
- // reaches its revocation endpoints by, which is pinned rather than derived
- // from wherever khaos binds.
+ // AuthorityBase is the URL the TARGET reaches the revocation endpoints by,
+ // which is pinned rather than derived from wherever khaos binds.
Authority *pki.Authority
PKITrap *pki.Trap
AuthorityBase string
AuthorityListen string
AuthorityTLS string
- // Defect names what khaos did to the credential the finding is about, in
- // words, so the verdict can state the actual observation rather than a
- // constant that is only true of one experiment.
+ // There is no resolver object here, only the fault and where to find one: an
+ // unprivileged process cannot bind port 53, so the resolver runs in a
+ // container on the target's network and the run attaches a client to it by
+ // label. DNSHost is the name under fault.
+ DNSTrap *dnsrv.Trap
+ ResolverNetwork string
+ DNSHost string
+
+ // Defect names what khaos did to the credential, in words, for the verdict.
Defect string
// HeldCaveat qualifies a HELD verdict where the absence of a finding covers
// less than it appears to.
HeldCaveat string
- // Consulted and ConsultedWhat are the liveness guard for the experiments
- // that fault an ENDPOINT rather than a credential.
- //
- // Three of them break khaos's introspection endpoint, and their only
- // enforcement evidence was that the invalid control got refused. A target
- // doing purely local JWT validation refuses a non-JWT bearer token itself
- // and never calls introspection at all, so the baseline passed, the trap
- // confirmed at khaos's own provider, and the fault window looked exactly
- // like the baseline: `passed / HELD`, conclusive, over a fault the target
- // never experienced.
- //
- // khaos already had the instrument and was not using it. Research 07's
- // requirement R5 names a fetch counter as a thing no third-party provider
- // will give you, and khaos built one and then read it nowhere.
- Consulted func() int64
+ // Consulted and ConsultedWhat are the liveness guard for an experiment that
+ // faults an ENDPOINT rather than a credential, where the probe streams cannot
+ // see whether the fault ever landed.
+ Consulted func() (int64, error)
ConsultedWhat string
- // Unsupportable names the check whose ABSENCE is a finding in its own
- // right, and turns a never-consulted endpoint into UNSUPPORTED instead of
- // BASELINE-NOT-MET.
- //
- // The two readings of "the target never touched the endpoint" are opposite
- // and both are sometimes right. For the introspection experiments it means
- // khaos faulted something the target does not use, so the run is void and
- // baseline-not-met is honest. For the revocation experiments it is THE
- // ANSWER: no mainstream TLS stack checks revocation by default, Go's does
- // not check it at all, and a target that never asked has no revocation
- // check that could have failed.
- //
- // Only an experiment that declares this gets the second reading, because
- // nothing in a request count distinguishes the two situations.
+ // Unsupportable names the check whose ABSENCE is itself the finding, which
+ // turns a never-consulted endpoint into UNSUPPORTED instead of a void run.
+ // Nothing in a request count distinguishes the two readings, so only an
+ // experiment that declares this gets the second one: no mainstream TLS stack
+ // checks revocation by default and Go's does not check it at all, so a target
+ // that never asked has no revocation check that could have failed.
Unsupportable string
- // IdleFor turns on the idleness discriminator. It must be longer than the
- // target's own decision-cache lifetime, which khaos cannot discover, so the
- // experiment declares it as a parameter and the verdict names the pause it
- // actually used.
+ // CacheRequired names the behaviour this experiment can only measure in a
+ // target that caches, which turns a target re-resolving per request into
+ // UNSUPPORTED rather than a clean survival.
+ CacheRequired string
+
+ // DeliveredWhat names what the target would have had to DO for the fault to
+ // reach it, which turns an armed-and-confirmed fault the target never met
+ // into an aborted run rather than a clean survival.
+ DeliveredWhat string
+
+ // IdleFor must exceed the target's own decision-cache lifetime, which khaos
+ // cannot discover, so the experiment declares it.
IdleFor time.Duration
// Act runs once, after the fault is confirmed and before anything is measured
- // under it. It exists for the one thing no amount of fault injection can
- // express: making a change at an authority the target can no longer reach.
- // ActDetail is what the operator is told khaos is about to do.
+ // under it, for the one thing fault injection cannot express: making a change
+ // at an authority the target can no longer reach.
Act func(context.Context) error
ActDetail string
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go
index fa44371c..29865b3a 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/idptier.go
@@ -429,11 +429,21 @@ func (s idpSpec) wantsIdleness() bool {
// counterFor maps the declared endpoint onto the provider's own measurement of
// it. The two endpoint names are constants rather than free text so a typo is a
// compile error and not a silently absent guard.
-func counterFor(provider *idp.Provider, endpoint string) func() int64 {
+// The error is always nil, for the same reason revocationCounter's is: this
+// counter lives in khaos's own process and there is no read that can fail. The
+// signature carries one because the DNS tier's counter lives in another
+// container, where a failed read used to be indistinguishable from a target
+// that never asked.
+func counterFor(
+ provider *idp.Provider,
+ endpoint string,
+) func() (int64, error) {
+ read := provider.Introspections
if endpoint == ConsultsJWKS {
- return provider.JWKSFetches
+ read = provider.JWKSFetches
}
- return provider.Introspections
+
+ return func() (int64, error) { return read(), nil }
}
func claimShiftMagnitude(params *Params, trap *idp.Trap) error {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/tlstier.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/tlstier.go
index 4f81515b..f93b7587 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/tlstier.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/tlstier.go
@@ -275,13 +275,18 @@ func readMagnitude(
// the authority, because a count that included them would make "the target
// checked revocation" true for every run and the UNSUPPORTED verdict would
// never fire.
+// The error is always nil, and that is the point of the wrapper rather than a
+// shortcut. This counter lives in khaos's own process, so there is no read that
+// can fail; the DNS tier's counter lives in another container and can, which is
+// why the signature carries one at all.
func revocationCounter(
authority *pki.Authority,
endpoint string,
-) func() int64 {
+) func() (int64, error) {
+ read := authority.CRLFetches
if endpoint == ConsultsOCSP {
- return authority.OCSPQueries
+ read = authority.OCSPQueries
}
- return authority.CRLFetches
+ return func() (int64, error) { return read(), nil }
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
index ee6759a1..bb67b19f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/experiment/verdict.go
@@ -92,10 +92,10 @@ const (
// experiment "should report an observed lower bound on the stale window and
// label it as a bound, never as 'caches forever'".
//
- // The old label asserted the unbounded claim in two words while the reason
- // beside it hedged, so a five-minute JWKS cache — the default in most
- // libraries, and bounded, and defensible — was reported as having no expiry
- // at all on the strength of a sixteen-second observation.
+ // A label asserting an unbounded cache cannot be earned from a window of
+ // seconds: any cache whose lifetime exceeds the observation looks identical
+ // to one that never expires, and a bounded cache is a defensible design
+ // rather than a defect.
StalePastPause Cause = "STALE-PAST-PAUSE"
)
@@ -132,8 +132,21 @@ const (
MinimumToProveOpen = 1
MinimumToClaimClosed = 30
MinimumPaired = 5
- RecoveryRunLength = 2
- crashFraction = 0.5
+
+ // queriesPerLookup is how many queries one hostname lookup puts on khaos's
+ // resolver, measured rather than assumed: Go fires A and AAAA in parallel
+ // per name, and under a negative fault the forced `search` domain doubles
+ // the names tried. Four is the figure the 16.003s dropped-query measurement
+ // in research 05 section 14.6 is built from.
+ queriesPerLookup = 2
+
+ // cacheLookupGrace is how many whole lookups a target is allowed inside the
+ // baseline before khaos will say it holds no cached answer. One would be
+ // exact; two leaves room for a retry or a second name without khaos
+ // claiming more than it saw.
+ cacheLookupGrace = 2
+ RecoveryRunLength = 2
+ crashFraction = 0.5
)
const (
@@ -172,6 +185,9 @@ const (
reasonTTLExtension = "the credential survived %s of STEADY traffic and then stopped being accepted after a %s pause, and no fixed lifetime behaves that way: every failed check was re-arming whatever the target had cached, a decision or a key set, so the lifetime was bounding IDLENESS rather than staleness. The fix is to stop writing to the cache in the error branch"
reasonLapsedOnPause = "the credential stopped being accepted after the %s pause, but the steady-traffic window before it was only %s, so an ordinary fixed expiry between those two durations produces this exact result and khaos will not call it re-arming. Re-run with --fault-for at least %s to separate them"
reasonStalePastPause = "the credential was still accepted after traffic was paused for %s, so whatever the target cached did not lapse across that pause. khaos cannot tell a cache with no expiry from one whose lifetime simply exceeds %s, and it will not claim the first from an observation that only establishes the second; raise --idle-for to raise the bound"
+ reasonNeverDelivered = "the fault was armed and confirmed, and khaos did not observe the target meeting it: it did not %s once during the fault window, so nothing khaos changed there can be shown to have reached it. khaos does NOT establish why. A kept-alive connection produces this shape -- Go's HTTP transport resolves a name once when it opens a connection and never again while that connection lives, measured at two lookups across forty-six requests -- and so would the target's own resolver cache, an /etc/hosts entry, or resolution over DoH, none of which khaos ruled out. What to check, in that order: the target's connection lifetime, its own DNS cache, /etc/hosts, and whether it resolves over plain DNS at all"
+ reasonCounterUnreadable = "khaos could not read its own count of the target's use of %s, so it cannot tell a target that never went there from an instrument it failed to read: %v. Every guard built on that count makes a statement about the TARGET, and khaos will not make one from a number it does not have"
+ reasonNoCache = "the target holds no cached answer, so %s governs nothing it does: it asked %s %d times during a baseline of %d requests, while khaos was serving a ten-minute lifetime for that name. A target honouring it resolves once for the whole window. There is no cache lifetime to have been honoured or ignored, so its behaviour under this fault is not a defect and khaos makes no security claim here. Go's resolver has no answer cache and ignores TTL entirely, which khaos measured rather than assumed from the target's language"
reasonNeverConsulted = "the target never used %s during the baseline, so the fault this experiment applies there is one it cannot experience; a target that validates credentials locally never calls it, and khaos would otherwise report a clean survival of a fault that never reached it"
reasonNoResumedEvidence = "the resumed window produced %d usable samples in %s and never a refusal, so khaos saw nothing that could tell a cache being re-armed from one that outlasts the pause"
reasonStaleWindow = "the credential khaos invalidated was accepted over a span of at least %s, which is the lower bound on the exposure and not a measurement of the cache lifetime"
@@ -222,14 +238,49 @@ type Observation struct {
// Consulted is how many times the target reached the faulted endpoint during
// the baseline, and ConsultedWhat names it. Zero means the fault khaos is
// about to apply is one the target cannot experience.
- Consulted int64 `json:"consulted,omitempty"`
- ConsultedWhat string `json:"consulted_what,omitempty"`
+ Consulted int64 `json:"consulted,omitempty"`
+
+ // CounterUnreadable is khaos failing to read its OWN instrument, which is a
+ // different thing from the target not having used it and must never be
+ // reported as the second.
+ //
+ // Three guards branch on a count of zero -- checkNotImplemented,
+ // cacheNotObserved and endpointNeverConsulted -- and each of them makes a
+ // statement about the TARGET. A swallowed read error reaching them produces
+ // a paragraph blaming the target's connection pool for khaos's own timeout.
+ CounterUnreadable error `json:"-"`
+ ConsultedWhat string `json:"consulted_what,omitempty"`
// Unsupportable names the check whose absence makes a never-consulted
// endpoint UNSUPPORTED rather than a void run. Only the experiments that
// probe something no mainstream TLS stack does by default set it.
Unsupportable string `json:"unsupportable,omitempty"`
+ // Delivered is how many times the target reached the faulted thing DURING
+ // the fault window, and DeliveredWhat names what it would have had to do.
+ // Zero means the fault was armed, confirmed, and never experienced.
+ Delivered int64 `json:"delivered,omitempty"`
+ DeliveredWhat string `json:"delivered_what,omitempty"`
+
+ // CacheRequired names the behaviour an experiment can only measure in a
+ // target that CACHES, and it is the INVERSE of Unsupportable rather than a
+ // variation on it.
+ //
+ // Unsupportable reads a count of ZERO as the answer: the target never
+ // performed the check, so there was none to fail. This reads a count of ONE
+ // PER PROBE as the answer: the target re-resolved every single time, so it
+ // holds no cached answer and the lifetime this experiment manipulates
+ // governs nothing it does.
+ //
+ // It is declared rather than derived because the same observation is a
+ // finding for one experiment and noise for another. A target that
+ // re-resolves per request is exactly correct for every other DNS fault, and
+ // only the two TTL faults have nothing left to measure in it. Go has no
+ // answer cache and ignores TTL entirely, proven against a 604800 TTL, so
+ // against a Go target this is the honest verdict every time -- and khaos
+ // MEASURES it rather than inferring it from a language it was never told.
+ CacheRequired string `json:"cache_required,omitempty"`
+
// MintedByKhaos records that the valid credential came from khaos's own
// identity provider rather than from the operator. It changes only the
// wording of one refusal, and that wording is the difference between telling
@@ -296,6 +347,23 @@ func epistemic(observation Observation) (Verdict, bool) {
return verdict, false
}
+ // BEFORE every guard that branches on the count, because all of them read a
+ // zero as a statement about the target and khaos cannot make that statement
+ // from an instrument it could not read.
+ if observation.CounterUnreadable != nil {
+ verdict.Epistemic = ProbeError
+ verdict.Reasons = append(
+ verdict.Reasons,
+ sprintf(
+ reasonCounterUnreadable,
+ observation.ConsultedWhat,
+ observation.CounterUnreadable,
+ ),
+ )
+
+ return verdict, false
+ }
+
// BEFORE endpointNeverConsulted, because the two read the same observation
// in opposite directions and only the experiment knows which applies. An
// endpoint the target never touched is a void run for a fault aimed at
@@ -308,6 +376,17 @@ func epistemic(observation Observation) (Verdict, bool) {
return verdict, false
}
+ // AFTER checkNotImplemented and BEFORE endpointNeverConsulted, because all
+ // three read the same count and the order encodes which reading wins. A
+ // target that asked ZERO times is a void run; one that asked EVERY time has
+ // no cache; one in between is the ordinary case this experiment can measure.
+ if reason, uncached := cacheNotObserved(observation); uncached {
+ verdict.Epistemic = Unsupported
+ verdict.Reasons = append(verdict.Reasons, reason)
+
+ return verdict, false
+ }
+
if reason, never := endpointNeverConsulted(observation); never {
verdict.Epistemic = BaselineNotMet
verdict.Reasons = append(verdict.Reasons, reason)
@@ -320,6 +399,25 @@ func epistemic(observation Observation) (Verdict, bool) {
return verdict, false
}
+ // AFTER confirmation, which is now true of the CODE and not only of an
+ // invariant in another file. The comment claimed this ordering while the
+ // check sat above the confirmation test; it was unreachable out of order
+ // through Engine.Run, because DeliveredWhat is only set inside measureFault,
+ // but Judge is exported and the verdict tests call it directly, so the
+ // sentence was one refactor away from lying.
+ //
+ // The two claims are genuinely different. Confirmed means khaos really armed
+ // the fault; delivered means the target really met it. Every other tier
+ // collapses them safely, because a severed path or a broken certificate sits
+ // on a route the target cannot avoid. A DNS answer is one it can simply not
+ // ask for again.
+ if reason, undelivered := faultNeverDelivered(observation); undelivered {
+ verdict.Epistemic = Aborted
+ verdict.Reasons = append(verdict.Reasons, reason)
+
+ return verdict, false
+ }
+
// A failed mid-run action is NOT a clean run. The credential khaos measured
// was never revoked, so every acceptance of it is correct behaviour and the
// finding this experiment exists to produce was unreachable.
@@ -616,6 +714,22 @@ func revokedHonoured(observation Observation) int {
// rejected. The only question left is whether it STILL accepts one once traffic
// has stopped for longer than any plausible cache lifetime.
func diagnose(observation Observation, reasons []string) (Cause, []string) {
+ // An experiment that never ran the pause has nothing to say on this axis,
+ // and saying nothing is the correct answer.
+ //
+ // Without this guard every PERMITTED verdict in the families that do not use
+ // the discriminator -- `network.`, `tls.` and `dns.` -- carried "the resumed
+ // window produced 0 usable samples in 0s and never a refusal, so khaos saw
+ // nothing that could tell a cache being re-armed from one that outlasts the
+ // pause". Every clause of that is true of the data and the sentence as a
+ // whole is not: it describes a measurement khaos declined to attempt as one
+ // it attempted and learned nothing from. The causal axis exists to answer
+ // WHY a credential is still honoured, and the idleness pause is the only
+ // instrument that answers it.
+ if observation.IdleFor <= 0 {
+ return Undiagnosed, reasons
+ }
+
// Deliberately NOT gated on Resumed.Paired(). That counts only the valid and
// invalid streams, so a resumed window full of decisions about the REVOKED
// credential — the stream that actually leaked — read as no evidence at all
@@ -773,6 +887,71 @@ func checkNotImplemented(observation Observation) (string, bool) {
), true
}
+// faultNeverDelivered reports a fault that was armed and confirmed and that the
+// target never met.
+//
+// It is ABORTED rather than HELD, and that distinction is the whole value of it.
+// HELD would say the target withstood the fault; aborted says khaos could not
+// put the fault in front of it. The second is true and the first is a clean
+// survival khaos never earned, which is the single failure mode this project
+// treats as a blocker every time it appears.
+func faultNeverDelivered(observation Observation) (string, bool) {
+ if observation.DeliveredWhat == "" || observation.Delivered > 0 {
+ return "", false
+ }
+
+ return sprintf(reasonNeverDelivered, observation.DeliveredWhat), true
+}
+
+// cacheNotObserved reports that the target holds no cached answer, for an
+// experiment whose whole subject is how long a cached answer is honoured.
+//
+// The threshold is one query per PAIRED BASELINE SAMPLE, which is deliberately
+// generous toward "no cache": a target that caches for the TTL khaos serves
+// asks once for the entire baseline, so the two populations are an order of
+// magnitude apart rather than adjacent. A target somewhere in between -- a
+// short cache, a pool that re-resolves occasionally -- falls through to the
+// ordinary reading, which is correct, because the experiment can then observe
+// the difference the TTL makes.
+func cacheNotObserved(observation Observation) (string, bool) {
+ if observation.CacheRequired == "" {
+ return "", false
+ }
+
+ // Measured against the TTL khaos itself served, not against how many
+ // requests khaos made.
+ //
+ // khaos cannot scale this by its own traffic, because the target's DNS
+ // lookups do not scale with it: the range's gateway caches its DECISIONS
+ // for two seconds, so it calls its dependency fewer times than khaos probes
+ // it, and a DNS lookup happens only when it does call. Any denominator
+ // built from khaos's request count therefore measures the target's decision
+ // cache rather than its DNS cache.
+ //
+ // What khaos does control is the lifetime it served. The baseline runs for
+ // seconds against an honest TTL of HonestTTL, so a target honouring that
+ // TTL resolves ONCE for the whole window. Go fires A and AAAA in parallel
+ // per name, so one lookup is queriesPerLookup queries; more than that means
+ // it resolved repeatedly inside a window orders of magnitude shorter than
+ // the lifetime it was given, which is the definition of not holding the
+ // answer.
+ //
+ // The margin is deliberately generous in the direction of NOT making this
+ // claim: at twice one lookup, the target has to have resolved at least
+ // twice before khaos says it caches nothing.
+ if observation.Consulted < queriesPerLookup*cacheLookupGrace {
+ return "", false
+ }
+
+ return sprintf(
+ reasonNoCache,
+ observation.CacheRequired,
+ observation.ConsultedWhat,
+ observation.Consulted,
+ int64(observation.Baseline.Requests()),
+ ), true
+}
+
// endpointNeverConsulted is the liveness guard for a fault applied to an
// endpoint rather than to a credential. Without it, three experiments report a
// clean survival of a fault the target never experienced.
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/dnsfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/dnsfault.go
new file mode 100644
index 00000000..bbb65bbc
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/dnsfault.go
@@ -0,0 +1,259 @@
+// ©AngelaMos | 2026
+// dnsfault.go
+
+package inject
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+ "github.com/CarterPerez-dev/khaos/internal/dockerx"
+ "github.com/CarterPerez-dev/khaos/internal/journal"
+)
+
+const (
+ KindDNSPrefix = "dns-"
+
+ errNoResolver = "%w: %s needs khaos's chaos resolver, and this run has none"
+
+ describeDNS = "%s, at khaos's own chaos resolver, which serves DNS inside its own container with nothing published to the host"
+)
+
+var ErrNoResolver = errors.New(
+ "this mutation needs khaos's chaos resolver, which is not running",
+)
+
+// DNSKind names the mutation that arms one fault on khaos's chaos resolver. The
+// fault kinds and the mutation kinds are one-to-one, so `khaos catalog` names
+// the actual defect being provoked rather than a generic "DNS" entry.
+func DNSKind(kind dnsrv.Kind) string {
+ return KindDNSPrefix + string(kind)
+}
+
+// dnsMutator arms a fault on khaos's chaos resolver.
+//
+// This is a FIFTH tier, and the reason it is its own tier rather than a fourth
+// authority is measured rather than architectural taste. Every other authority
+// khaos plays — the identity provider, the certificate authority — is a
+// goroutine in khaos's own process, bound on a high host port the target was
+// told to dial. A DNS resolver cannot be: `--dns` carries an address and no
+// port, the daemon rejects one at container start, and an unprivileged process
+// gets `bind: permission denied` on port 53 at every address including
+// loopback. So this authority lives in a container, which is the only place a
+// process can hold port 53 without khaos holding a capability.
+//
+// The fault it injects is also unlike the others. The dependency is healthy and
+// reachable; the target cannot FIND it, or finds something else. That is a third
+// branch of the target's code, distinct from both an unreachable dependency and
+// one whose certificate will not verify.
+type dnsMutator struct {
+ kind dnsrv.Kind
+}
+
+func init() {
+ for _, name := range dnsrv.Kinds() {
+ kind := dnsrv.Kind(name)
+ registry[DNSKind(kind)] = dnsMutator{kind: kind}
+ }
+}
+
+func (m dnsMutator) Describe(env Env, target dockerx.Target) string {
+ trap := env.DNSTrap
+ if trap.Kind != m.kind {
+ trap = dnsrv.Trap{Kind: m.kind}
+ }
+
+ return fmt.Sprintf(describeDNS, trap.Describe(target.Name))
+}
+
+// dnsPayload is what the journal carries. It records how to reach the resolver
+// as well as the fault, because the revert has to work from a COLD START where
+// khaos remembers nothing — and unlike every other in-process tier, this one's
+// authority outlives the khaos process that armed it. A fault left armed by a
+// killed khaos is a target whose dependency stays unresolvable, so the record
+// has to be enough to undo it without the run that made it.
+type dnsPayload struct {
+ Trap dnsrv.Trap `json:"trap"`
+ Network string `json:"network"`
+ Address string `json:"address"`
+ Control string `json:"control"`
+}
+
+func (m dnsMutator) Plan(
+ _ context.Context,
+ env Env,
+ _ dockerx.Target,
+) (json.RawMessage, error) {
+ if env.Resolver == nil {
+ return nil, fmt.Errorf(errNoResolver, ErrNoResolver, DNSKind(m.kind))
+ }
+
+ if env.DNSTrap.Kind != m.kind {
+ return nil, fmt.Errorf(
+ errWrongTrapKind,
+ ErrTrapMismatch,
+ env.DNSTrap.Kind,
+ m.kind,
+ )
+ }
+
+ trap := env.DNSTrap
+
+ if err := trap.Validate(); err != nil {
+ return nil, err
+ }
+
+ return journal.EncodePayload(dnsPayload{
+ Trap: trap,
+ Network: env.ResolverNetwork,
+ Address: env.Resolver.DNS,
+ Control: env.Resolver.Control,
+ })
+}
+
+func (m dnsMutator) Apply(
+ ctx context.Context,
+ env Env,
+ _ dockerx.Target,
+ raw json.RawMessage,
+) error {
+ if env.Resolver == nil {
+ return fmt.Errorf(errNoResolver, ErrNoResolver, DNSKind(m.kind))
+ }
+
+ var payload dnsPayload
+ if err := json.Unmarshal(raw, &payload); err != nil {
+ return err
+ }
+
+ _, err := env.Resolver.Arm(ctx, payload.Trap)
+
+ return err
+}
+
+// Revert disarms, which restores khaos's own honest answer so the recovery
+// window is measured through a WORKING resolver rather than through no
+// resolution at all.
+//
+// A cold start rebuilds the client from the daemon rather than giving up, and
+// that is the difference between this tier and the in-process ones. An identity
+// provider or a certificate authority dies with the khaos process that held it,
+// so the fault provably went with it and there is nothing to undo. A resolver
+// container does not die: it keeps serving NXDOMAIN for the target's dependency
+// until something disarms it. The nonce is read off the container's own label,
+// so the revert needs nothing khaos remembered.
+func (m dnsMutator) Revert(
+ ctx context.Context,
+ env Env,
+ record journal.Record,
+) error {
+ client := env.Resolver
+
+ if client == nil {
+ var payload dnsPayload
+ if len(record.Payload) > 0 {
+ if err := record.DecodePayload(&payload); err != nil {
+ return err
+ }
+ }
+
+ client = reconnectResolver(ctx, env, payload)
+ }
+
+ // Nothing left to disarm, which is SUCCESS rather than a failure to report.
+ // The resolver the record named is gone, so it took the fault with it, and
+ // reporting that as an error would wedge the record forever and, because
+ // `khaos reap` stops on a sweep error, disable the dead man's switch for
+ // every other fault on the machine.
+ if client == nil {
+ return nil
+ }
+
+ _, err := client.Disarm(ctx)
+
+ return err
+}
+
+// reconnectResolver finds the resolver a journal record names and builds a
+// client for it, or nil if there is none to find.
+//
+// It returns no error ON PURPOSE, and the single return is the honest signature
+// rather than a convenience. Every way this can fail to produce a client means
+// the same thing: the resolver that held the fault is not there, so the fault is
+// not there either. A record with no network, a daemon khaos cannot reach from a
+// cold start, a container somebody already removed -- each of those is a revert
+// that has nothing left to do, and the caller treats nil as done.
+func reconnectResolver(
+ ctx context.Context,
+ env Env,
+ payload dnsPayload,
+) *dnsrv.Client {
+ if env.Docker == nil || payload.Network == "" {
+ return nil
+ }
+
+ resolver, err := env.Docker.FindResolver(ctx, payload.Network)
+ if err != nil {
+ return nil
+ }
+
+ return dnsrv.NewClient(
+ resolver.Address,
+ dockerx.ResolverControlURL(resolver.Address),
+ resolver.Nonce,
+ payload.Control,
+ )
+}
+
+// Confirm delegates to the resolver client's own read-back, which asks the
+// resolver the same question the target asks and reports what came off the
+// wire.
+//
+// It deliberately does not read the armed state back. The control surface
+// answers /arm with the resolver's state, which would be the easy thing to
+// check and would confirm a fault against a resolver whose serving path is
+// broken. The PKI tier's inert-authority control caught fifteen traps doing
+// exactly that, every one of them reading the trap RECORD instead of what was
+// served.
+func (m dnsMutator) Confirm(
+ _ context.Context,
+ env Env,
+ target dockerx.Target,
+) (string, bool, error) {
+ if env.Resolver == nil {
+ return "", false, fmt.Errorf(
+ errNoResolver,
+ ErrNoResolver,
+ DNSKind(m.kind),
+ )
+ }
+
+ trap := env.DNSTrap
+ if trap.Kind != m.kind {
+ trap = dnsrv.Trap{Kind: m.kind, Host: target.Name}
+ }
+
+ return env.Resolver.Observe(trap)
+}
+
+// NeedsResolver reports whether this mutation arms khaos's chaos resolver, so
+// the run knows to find one before the baseline. It is derived from the registry
+// rather than from the kind's spelling, for the same reason NeedsProxy is.
+func NeedsResolver(kind string) bool {
+ mutator, ok := registry[kind]
+ if !ok {
+ return false
+ }
+
+ _, isDNS := mutator.(dnsMutator)
+
+ return isDNS
+}
+
+// Prefix declares this tier's kinds to the registry. The chaos resolver outlives
+// the khaos process, but it is still per-RUN state: `khaos inject` neither
+// stands one up nor knows which network it would serve.
+func (dnsMutator) Prefix() string { return KindDNSPrefix }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/dnsfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/dnsfault_test.go
new file mode 100644
index 00000000..4488047d
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/dnsfault_test.go
@@ -0,0 +1,92 @@
+// ©AngelaMos | 2026
+// dnsfault_test.go
+
+package inject
+
+import (
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
+)
+
+// daemonKinds is the CLOSED set of mutations `khaos inject` may offer: the ones
+// whose whole subject is a container the daemon can be told about.
+//
+// It is pinned by NAME on purpose, and the reason is a defect. Deriving the
+// exclusion from the registry is necessary but not sufficient: a sixth tier that
+// forgets to declare itself per-run is simply absent from the derived prefix
+// list, the IFF assertion still holds, and `khaos inject` quietly advertises a
+// command that cannot succeed. That is exactly how thirty-six `pki-*` kinds
+// shipped. A literal allowlist fails loudly and names them, and this set is
+// closed by its own definition rather than by convention.
+var daemonKinds = []string{KindPause, KindDisconnect}
+
+func TestDirectKindsIsExactlyTheDaemonKinds(t *testing.T) {
+ direct := DirectKinds()
+
+ for _, kind := range direct {
+ if !slices.Contains(daemonKinds, kind) {
+ t.Errorf(
+ "`khaos inject` offers %s, which is not a mutation the daemon "+
+ "can be told about; either it belongs in daemonKinds or its "+
+ "tier never declared Prefix()",
+ kind,
+ )
+ }
+ }
+
+ for _, kind := range daemonKinds {
+ if !slices.Contains(direct, kind) {
+ t.Errorf("`khaos inject` no longer offers %s", kind)
+ }
+ }
+}
+
+// TestEveryDNSFaultIsRegistered is the census. A fault in the resolver's
+// catalogue with no mutation is one `khaos catalog` could advertise and the
+// engine could not apply, and the masked faults are included deliberately: they
+// have to be reachable in order to be refused by name.
+func TestEveryDNSFaultIsRegistered(t *testing.T) {
+ for _, name := range dnsrv.Kinds() {
+ kind := DNSKind(dnsrv.Kind(name))
+
+ // Lookup must RESOLVE it: the engine gets its mutator from there for
+ // every tier. Only Lookup's error message is restricted to the direct
+ // kinds, so that a typo is answered with the kinds an operator can
+ // actually ask for.
+ if _, err := Lookup(kind); err != nil {
+ t.Errorf("the engine cannot resolve %s: %v", kind, err)
+ }
+
+ if _, ok := registry[kind]; !ok {
+ t.Errorf("no mutation is registered for DNS fault %s", name)
+ }
+
+ if !NeedsResolver(kind) {
+ t.Errorf("%s does not ask the run for a resolver", kind)
+ }
+
+ if IsDirect(kind) {
+ t.Errorf("`khaos inject` offers %s, which needs a resolver it "+
+ "never starts", kind)
+ }
+ }
+}
+
+// TestNeedsResolverIsDerivedFromTheRegistry guards against the shape that gave
+// the proxy and identity tiers their own bugs: a predicate that keys off the
+// kind's spelling agrees with itself about a kind that does not exist.
+func TestNeedsResolverIsDerivedFromTheRegistry(t *testing.T) {
+ if NeedsResolver(KindDNSPrefix + "not-a-real-fault") {
+ t.Error("NeedsResolver answered yes for a kind no mutation is " +
+ "registered under, so it is reading the name rather than the registry")
+ }
+
+ for _, kind := range Kinds() {
+ if strings.HasPrefix(kind, KindDNSPrefix) != NeedsResolver(kind) {
+ t.Errorf("%s: NeedsResolver reports %v", kind, NeedsResolver(kind))
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go
index 9c0be650..7aa67199 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/idpfault.go
@@ -159,3 +159,6 @@ func (m idpMutator) Confirm(
return env.IdP.Observe(ctx, m.kind)
}
+
+// Prefix declares this tier's kinds to the registry.
+func (idpMutator) Prefix() string { return KindIdPPrefix }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/pkifault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/pkifault.go
index 5e5ee06a..701f1fd5 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/pkifault.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/pkifault.go
@@ -161,3 +161,8 @@ func (m pkiMutator) Confirm(
return env.Authority.Observe(ctx, m.kind)
}
+
+// Prefix declares this tier's kinds to the registry, which keeps `khaos inject`
+// from offering them: the certificate authority is a goroutine inside a run and
+// `khaos inject` starts none.
+func (pkiMutator) Prefix() string { return KindPKIPrefix }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
index 35a145ab..5c03cb35 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxy/proxy.go
@@ -407,13 +407,12 @@ func (p *Proxy) sever(client net.Conn, fault Fault, generation uint64) bool {
// the error to its write and its own read returns EOF. That is the EOF
// fault's observable, which would collapse two catalogue entries into one.
//
- // An earlier pass deleted this wait as inert because both reset controls
- // stayed green over 20 runs without it. Those controls run through
- // net/http, which reports a reset at connect and a reset at read as the
- // same error, so they could not see the difference and alibied the arm.
- // Measured at the socket instead, 20 trials per arm: without the wait,
- // `write=ECONNRESET read=EOF` 13 times and a reset at connect the other 7,
- // never a reset at the read; with it, `write=nil read=ECONNRESET` 20 of 20.
+ // Measured at the socket, 20 trials per arm: without this wait,
+ // `write=ECONNRESET read=EOF` 13 times and a reset at connect the other
+ // 7, never a reset at the read; with it, `write=nil read=ECONNRESET` 20
+ // of 20. net/http cannot see the difference, reporting a reset at connect
+ // and one at read as the same error, so it must be measured at the
+ // socket.
settle(client)
if tcp, ok := client.(*net.TCPConn); ok {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go
index 16542887..9a2c56c6 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault.go
@@ -150,3 +150,7 @@ func (m proxyMutator) Confirm(
return env.Proxy.Observe(ctx, m.kind)
}
+
+// Prefix declares this tier's kinds to the registry, which is how `khaos
+// inject` knows not to offer a fault that only exists inside a run.
+func (proxyMutator) Prefix() string { return KindProxyPrefix }
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
index e00314da..318d771f 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/proxyfault_test.go
@@ -274,8 +274,22 @@ func TestRevertingWithoutAProxyIsResolvedRatherThanFailed(t *testing.T) {
// that something is excluded lets one tier's exclusion alibi the other's: this
// test passed the whole time the identity tier was unreachable from DirectKinds
// by accident rather than by the registry knowing about it.
+// The prefix list is now DERIVED from the registry rather than written here,
+// and that change has a defect behind it. The hand-written list named the proxy
+// and identity tiers; M7 added the certificate authority and extended neither
+// the type switch nor this list, so `khaos inject` advertised thirty-six
+// `pki-*` kinds that each failed with "needs khaos's certificate authority, and
+// this run did not start one" — and this test stayed green for the whole
+// milestone. A per-prefix assertion over a list nobody maintains is an
+// aggregate assertion wearing better clothes.
func TestInProcessKindsAreNotOfferedForDirectInjection(t *testing.T) {
- prefixes := []string{KindProxyPrefix, KindIdPPrefix}
+ prefixes := PerRunPrefixes()
+
+ if len(prefixes) == 0 {
+ t.Fatal(
+ "no tier declares a per-run prefix, so this test asserts nothing",
+ )
+ }
for _, kind := range Kinds() {
wantDirect := true
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
index 645c8d3a..8601a6aa 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/inject/registry.go
@@ -11,6 +11,7 @@ import (
"slices"
"strings"
+ "github.com/CarterPerez-dev/khaos/internal/dnsrv"
"github.com/CarterPerez-dev/khaos/internal/dockerx"
"github.com/CarterPerez-dev/khaos/internal/idp"
"github.com/CarterPerez-dev/khaos/internal/inject/proxy"
@@ -59,6 +60,19 @@ type Env struct {
// nothing but the target.
Authority *pki.Authority
PKITrap pki.Trap
+
+ // Resolver, ResolverNetwork and DNSTrap are the FIFTH injection tier, and
+ // the only one whose authority is not in khaos's own process.
+ //
+ // It could not be. `--dns` carries an address with no port, the daemon
+ // refuses one at container start, and an unprivileged process cannot bind
+ // port 53 on any address. So the resolver runs in a container on the
+ // target's network with nothing published, and khaos holds a CLIENT for it
+ // rather than the resolver itself. ResolverNetwork travels with it so a
+ // cold-start revert can find the container again by label.
+ Resolver *dnsrv.Client
+ ResolverNetwork string
+ DNSTrap dnsrv.Trap
}
// Plan discovers everything the revert will need and returns it so the caller
@@ -125,17 +139,48 @@ func DirectKinds() []string {
return kinds
}
+// perRun is implemented by every tier whose authority exists only while the run
+// that owns it does, so `khaos inject` can decline to offer a mutation that
+// cannot succeed outside a run.
+//
+// An interface rather than a type switch: a new tier declares itself by
+// implementing one method, where a switch has to be remembered. The direct set
+// is additionally pinned by name in the tests, because a tier that forgets this
+// method is simply absent from anything derived from it.
+type perRun interface {
+ Prefix() string
+}
+
// inProcess reports whether a mutation acts on state that lives inside the run
// rather than in the daemon. It is derived from the mutator's TYPE rather than
// from the kind's spelling, so a mutation cannot be given an in-process-shaped
// name and then quietly be offered under `khaos inject`, or the reverse.
func inProcess(mutator Mutator) bool {
- switch mutator.(type) {
- case proxyMutator, idpMutator:
- return true
- default:
- return false
+ _, ok := mutator.(perRun)
+ return ok
+}
+
+// PerRunPrefixes is every prefix belonging to a tier that lives inside a run,
+// read out of the registry. A test that needs the list derives it from here
+// instead of restating it, which is the specific thing that failed before.
+func PerRunPrefixes() []string {
+ seen := map[string]bool{}
+
+ for _, mutator := range registry {
+ tier, ok := mutator.(perRun)
+ if !ok {
+ continue
+ }
+ seen[tier.Prefix()] = true
}
+
+ prefixes := make([]string, 0, len(seen))
+ for prefix := range seen {
+ prefixes = append(prefixes, prefix)
+ }
+ slices.Sort(prefixes)
+
+ return prefixes
}
func IsDirect(kind string) bool {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/issuers.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/issuers.go
index 9da87ef3..c9d7e477 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/issuers.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/issuers.go
@@ -42,9 +42,11 @@ func (a *Authority) buildConstrained(from, until time.Time) error {
// buildEKULimited is a CA authorised for client authentication only. Go checks
// extended key usage NESTED DOWN THE CHAIN, so a perfectly ordinary serverAuth
-// leaf under this CA trips InvalidReason 9,
-// CANotAuthorizedForExtKeyUsage. The leaf itself is blameless, which is what
-// makes this distinct from the leaf-level EKU fault.
+// leaf under this CA is refused with InvalidReason 4, IncompatibleUsage — the
+// same reason as the leaf-level fault, because `crypto/x509` declares
+// CANotAuthorizedForExtKeyUsage and never constructs it. The leaf itself is
+// blameless, which is what makes the two faults distinct despite the identical
+// error.
func (a *Authority) buildEKULimited(from, until time.Time) error {
template, err := caTemplate(nameEKULimited, from, until)
if err != nil {
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/trap.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/trap.go
index f517f0ca..d77c8c2a 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/trap.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/pki/trap.go
@@ -88,9 +88,11 @@ const (
KeyUsageWithoutDigitalSignature Kind = "key-usage-no-digital-signature"
// EKUClientAuthOnly and EKUConstrainedIntermediate are the extended key
- // usage faults, Reason 4 and Reason 9. Go checks EKU nested down the chain
- // but does not check the basic KeyUsage bits on a leaf, which is why the
- // kind above exists beside these two. Rows F14 and F15.
+ // usage faults, and BOTH come back as Reason 4: `crypto/x509` declares
+ // CANotAuthorizedForExtKeyUsage and never constructs it, so no chain can
+ // elicit it. Go checks EKU nested down the chain but does not check the
+ // basic KeyUsage bits on a leaf, which is why the kind above exists beside
+ // these two. Rows F14 and F15.
EKUClientAuthOnly Kind = "eku-client-auth-only"
EKUConstrainedIntermediate Kind = "eku-constrained-intermediate"
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
index 639f327f..fa9311af 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/probe/sample.go
@@ -217,6 +217,22 @@ func (w Window) Len() int {
return w.Paired()
}
+// Requests is how many probe requests this window actually issued, across every
+// stream. It is the denominator for any per-REQUEST ratio, and Paired is not:
+// one paired sample is at least two requests, because the pair fires both
+// streams per iteration.
+//
+// The distinction had a cost. A guard comparing a count of the target's DNS
+// queries against Paired() was comparing queries to half the requests, and with
+// Go's A-plus-AAAA fan-out on top it tripped at one lookup per four requests
+// while its own message claimed "a fresh lookup per request".
+func (w Window) Requests() int {
+ return w.Valid.Countable() +
+ w.Invalid.Countable() +
+ w.Revoked.Countable() +
+ w.Forged.Countable()
+}
+
func (w *Window) Record(outcome Outcome) {
switch outcome.Credential {
case Invalid:
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
index 188038a0..d263cc7d 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/rangelab/rangelab.go
@@ -37,6 +37,34 @@ const (
// uses.
TLSGatewayName = "khaos-range-gateway-tls"
+ // DNSGatewayName is the FIFTH gateway, and the only one that reaches its
+ // dependency BY NAME through khaos's chaos resolver. RogueName is the
+ // service a redirect fault points it at, and ResolverName is the resolver
+ // itself.
+ DNSGatewayName = "khaos-range-gateway-dns"
+ RogueName = "khaos-range-rogue"
+ ResolverName = "khaos-range-resolver"
+
+ // The addresses the DNS tier PINS, and they are pinned because they have to
+ // be. `dns:` on the gateway and the resolver's zone are both fixed at
+ // container creation, so neither can be filled in from an address khaos
+ // discovered afterwards. The range network declares its own subnet so Docker
+ // will accept them.
+ ResolverAddress = "10.244.244.5"
+ DependencyAddress = "10.244.244.10"
+ RogueAddress = "10.244.244.20"
+
+ // DNSDependencyName is the name the DNS gateway asks for. It is NOT a
+ // container name or a network alias, and that is measured rather than
+ // stylistic: the target's own embedded resolver answers those out of its own
+ // table and never forwards them, so a fault aimed at one would never be
+ // seen.
+ DNSDependencyName = "dependency.khaos.test."
+
+ // DNSControlName is the name khaos's resolver answers that no fault ever
+ // touches, used as the confirmation oracle's positive control.
+ DNSControlName = "control.khaos.invalid."
+
IDPName = "khaos-range-idp"
LabelRange = "khaos.range"
FailModeOpen = "open"
@@ -54,6 +82,14 @@ const (
EnvTLSPort = "KHAOS_RANGE_TLS_PORT"
+ // EnvDNSPort is the DNS gateway's published port, and EnvResolverNonce
+ // authorises its resolver's control surface. The nonce goes in the compose
+ // environment because the resolver's command line is fixed at container
+ // creation, and khaos reads it BACK off the container's label rather than
+ // remembering what it chose.
+ EnvDNSPort = "KHAOS_RANGE_DNS_PORT"
+ EnvResolverNonce = "KHAOS_RANGE_RESOLVER_NONCE"
+
// The two ports khaos binds in its OWN process for the certificate tier.
// They go into the compose file because the gateway's environment is fixed
// at container creation and it has to know where to look.
@@ -114,6 +150,12 @@ const (
stderrLines = 4
teardownPort = 1
+ // teardownNonce satisfies the compose file's required variable during a
+ // teardown, where no resolver is being created and the value is never used.
+ // The alternative is `range down` failing on a missing variable, which would
+ // leave the range running with no way to remove it.
+ teardownNonce = "teardown"
+
EnvComposeDir = "KHAOS_RANGE_DIR"
EnvUpstream = "KHAOS_RANGE_UPSTREAM"
@@ -126,6 +168,8 @@ const (
errBadIssuer = "%w: %s trusts the issuer %q, which khaos cannot read as a host and port (%v)"
errNoVerifierGateway = "%w: this experiment arms khaos's own identity provider, and the running range has no %s; bring the range down and up again to create it"
errNoTLSGateway = "%w: this experiment arms khaos's own certificate authority, and the running range has no %s; bring the range down and up again to create it"
+ errNoDNSGateway = "%w: this experiment arms khaos's chaos resolver, and the running range has no %s; bring the range down and up again to create it"
+ errNoRangeResolver = "%w: this experiment arms khaos's chaos resolver, and the running range has no %s; bring the range down and up again to create it"
)
// Port is the direct gateway's published port. ProxiedPort is the second
@@ -157,6 +201,12 @@ type Lab struct {
AuthorityPort int
DependencyPort int
+ // DNSPort is the DNS gateway's published port. ResolverNonce is held only
+ // so `range up` can put it in the compose environment; once the range is
+ // running khaos reads it back off the resolver container's own label.
+ DNSPort int
+ ResolverNonce string
+
// Dependency, AuthorityBase and CertHost are read back from the TLS
// gateway's own environment, never remembered from what `range up` chose.
// The container is the only authority on where it will actually look and
@@ -195,6 +245,8 @@ func Locate(ports Ports) (Lab, error) {
TLSPort: ports.TLS,
AuthorityPort: ports.Authority,
DependencyPort: ports.Dependency,
+
+ DNSPort: ports.DNS,
}, nil
}
@@ -213,6 +265,11 @@ type Ports struct {
TLS int
Authority int
Dependency int
+
+ // DNS is the DNS gateway's own published port. The resolver publishes
+ // NOTHING: it listens on 53 inside its own network namespace, which is the
+ // entire reason that tier needs no capability.
+ DNS int
}
// FreePorts picks every host port the range needs in one go, all distinct. Each
@@ -247,6 +304,7 @@ func FreePorts() (Ports, error) {
&chosen.TLS,
&chosen.Authority,
&chosen.Dependency,
+ &chosen.DNS,
} {
port, err := pick()
if err != nil {
@@ -391,6 +449,8 @@ func (l Lab) Up(
EnvTLSPort+"="+strconv.Itoa(l.TLSPort),
EnvAuthorityPort+"="+strconv.Itoa(l.AuthorityPort),
EnvDependencyPort+"="+strconv.Itoa(l.DependencyPort),
+ EnvDNSPort+"="+strconv.Itoa(l.DNSPort),
+ EnvResolverNonce+"="+l.ResolverNonce,
EnvFailMode+"="+failMode,
EnvCacheTTL+"="+cacheTTL,
EnvTLSMode+"="+tlsMode,
@@ -424,6 +484,8 @@ func (l Lab) Down(ctx context.Context, out io.Writer) error {
EnvVerifierPort+"="+strconv.Itoa(teardownPort),
EnvIdPPort+"="+strconv.Itoa(teardownPort),
EnvTLSPort+"="+strconv.Itoa(teardownPort),
+ EnvDNSPort+"="+strconv.Itoa(teardownPort),
+ EnvResolverNonce+"="+teardownNonce,
EnvAuthorityPort+"="+strconv.Itoa(teardownPort),
EnvDependencyPort+"="+strconv.Itoa(teardownPort),
)
@@ -670,9 +732,37 @@ func Discover(ctx context.Context, reader PortReader) (Lab, error) {
return Lab{}, err
}
+ if err = lab.readDNSTier(ctx, reader, running); err != nil {
+ return Lab{}, err
+ }
+
return lab, nil
}
+// readDNSTier discovers the DNS gateway's published port, on the same terms as
+// the other three gateways: absent is not fatal, because every experiment that
+// does not arm khaos's chaos resolver has no use for it, and failing the whole
+// discovery over it would break all of them.
+func (l *Lab) readDNSTier(
+ ctx context.Context,
+ reader PortReader,
+ running map[string]dockerx.Target,
+) error {
+ dnsGateway, ok := running[DNSGatewayName]
+ if !ok {
+ return nil
+ }
+
+ port, err := reader.PublishedPort(ctx, dnsGateway.ID, GatewayInternalPort)
+ if err != nil {
+ return err
+ }
+
+ l.DNSPort = port
+
+ return nil
+}
+
// readAuthorityWiring reads the three strings the TLS gateway was configured
// with, out of its own environment, for the same reason issuerOf does: the
// container is the only authority on what it will dial, where it will fetch
@@ -787,6 +877,36 @@ func (l Lab) TLSGatewayURL() string {
return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.TLSPort))
}
+// RequireDNSTier refuses a DNS experiment against a range that has no DNS
+// gateway, BY NAME, before anything is probed.
+//
+// A missing gateway otherwise surfaces as a published port of zero, so khaos
+// probes `http://127.0.0.1:0/protected`, nothing answers, and the run blames the
+// target for not enforcing when khaos was pointing at nowhere.
+func (l Lab) RequireDNSTier() error {
+ if l.DNSPort == 0 {
+ return fmt.Errorf(
+ errNoDNSGateway,
+ ErrRangeIncomplete,
+ DNSGatewayName,
+ )
+ }
+
+ return nil
+}
+
+func (l Lab) DNSGatewayURL() string {
+ return "http://" + net.JoinHostPort(loopback, strconv.Itoa(l.DNSPort))
+}
+
+func (l Lab) DNSProtectedURL() string {
+ return l.DNSGatewayURL() + protectedPath
+}
+
+func (l Lab) RogueURL() string {
+ return "http://" + net.JoinHostPort(RogueAddress, "8080")
+}
+
func (l Lab) TLSProtectedURL() string {
return l.TLSGatewayURL() + protectedPath
}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/internal/tlsclient/revocation.go b/PROJECTS/intermediate/chaos-engineering-tool/internal/tlsclient/revocation.go
index 0c107eae..8cc0fe4b 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/internal/tlsclient/revocation.go
+++ b/PROJECTS/intermediate/chaos-engineering-tool/internal/tlsclient/revocation.go
@@ -97,11 +97,10 @@ func (c *Client) fromStaple(
// Believed without being parsed. Garbage bytes are accepted here, which
// is the whole of the StapleGarbage fault.
//
- // The question is marked ANSWERED. A client that trusts the staple does
- // not then go and ask the responder as well, and the first version fell
- // through to doing exactly that: it accepted the staple, queried the
- // endpoint anyway, and refused on the real answer. Three traps looked
- // caught by a mode that by construction cannot catch them.
+ // The question is marked ANSWERED and the function returns here. A client
+ // that trusts the staple must not also query the responder: it would then
+ // refuse on the real answer, and three traps would look caught by a mode
+ // that by construction cannot catch them.
decision.AskedOCSP = true
return reasonAccepted, true
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/justfile b/PROJECTS/intermediate/chaos-engineering-tool/justfile
index ac8aee22..aa5785e7 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/justfile
+++ b/PROJECTS/intermediate/chaos-engineering-tool/justfile
@@ -8,8 +8,15 @@ binary := "khaos"
default:
@just --list
+# CGO_ENABLED=0 is REQUIRED, not a preference. khaos's DNS tier imports khaos's
+# own binary as a one-layer container image with no base, because an
+# unprivileged process cannot bind port 53 and Docker's --dns carries an address
+# with no port. A dynamically linked binary in a base-less image cannot exec:
+# there is no /lib64/ld-linux-x86-64.so.2 for it to find. khaos refuses the DNS
+# tier by name when it detects a PT_INTERP segment in itself, so a build without
+# this flag loses one whole fault family.
build:
- go build -o {{binary}} ./cmd/{{binary}}
+ CGO_ENABLED=0 go build -o {{binary}} ./cmd/{{binary}}
run *args:
go run ./cmd/{{binary}} {{args}}
@@ -40,7 +47,7 @@ tidy-check:
check: lint tidy-check test colors
-gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate
+gate: check race smoke survive range-gate proxy-gate identity-gate family-gate tls-gate dns-gate
colors: build
./scripts/colors.sh
@@ -66,5 +73,8 @@ family-gate: build
tls-gate: build
./scripts/tls.sh
+dns-gate: build
+ ./scripts/dns.sh
+
clean:
rm -f {{binary}}
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
index e1437d92..54e3f181 100644
--- a/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
+++ b/PROJECTS/intermediate/chaos-engineering-tool/range/compose.yml
@@ -14,7 +14,14 @@ services:
labels:
khaos.target: "true"
khaos.range: "true"
- networks: [range]
+ # The address is PINNED, and only on this network, because khaos's chaos
+ # resolver has to be authoritative for a name that maps to it and the zone
+ # is passed as a literal at container creation. Docker refuses a static
+ # address on a network whose subnet it chose, which is why the range network
+ # declares its own below.
+ networks:
+ range:
+ ipv4_address: 10.244.244.10
healthcheck:
test: ["CMD", "/usr/local/bin/khaos-range", "healthcheck"]
interval: 2s
@@ -161,6 +168,133 @@ services:
networks: [range]
restart: "no"
+
+ # The service a DNS redirect fault points the target at, and the reason
+ # `dns.wrong_address` is a fail-open demonstration rather than an outage.
+ # It approves every credential, including ones khaos never issued, which is
+ # the shape of an attacker who has won a name. Research 05 section 10: a
+ # hostname allowlist does not merely stop working when the name moves, it
+ # actively authorises the new host.
+ rogue:
+ container_name: khaos-range-rogue
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["rogue"]
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ networks:
+ range:
+ ipv4_address: 10.244.244.20
+ restart: "no"
+
+ # khaos's chaos resolver, as a range service rather than a container khaos
+ # creates.
+ #
+ # Outside the range khaos imports its own binary as a one-layer image and
+ # starts it, because an operator's khaos has no source tree. Here the image
+ # already exists and the resolver is the same `internal/dnsrv` package behind
+ # a different entry point, so there is nothing to import and the compose file
+ # is the honest place for it. The labels are what `khaos.FindResolver` matches
+ # on, so khaos finds this one exactly as it finds one it started itself.
+ resolver:
+ container_name: khaos-range-resolver
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command:
+ - "resolver"
+ - "--control"
+ - "control.khaos.invalid."
+ - "--zone"
+ - "dependency.khaos.test.=10.244.244.10"
+ - "--zone"
+ - "control.khaos.invalid.=10.244.244.10"
+ # The nonce is an ENVIRONMENT variable and not an argument. A container's
+ # command line appears in the HOST process table: measured,
+ # /proc/<pid>/cmdline is mode 444 and world-readable while
+ # /proc/<pid>/environ is 400. The nonce is the only authorisation for
+ # arming and disarming khaos's DNS faults, so a local user reading it could
+ # disarm a fault mid-run and khaos would measure an unfaulted window.
+ #
+ # The matching LABEL is not the same exposure: labels come from the daemon
+ # API, which already needs socket access that is root-equivalent, and khaos
+ # has to read the nonce back from somewhere on a cold start.
+ environment:
+ KHAOS_RESOLVER_NONCE: "${KHAOS_RANGE_RESOLVER_NONCE:?khaos chooses this nonce; run `khaos range up`}"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ khaos.resolver: "true"
+ khaos.resolver.network: "khaos-range"
+ khaos.resolver.nonce: "${KHAOS_RANGE_RESOLVER_NONCE:?khaos chooses this nonce; run `khaos range up`}"
+ networks:
+ range:
+ ipv4_address: 10.244.244.5
+ restart: "no"
+
+ # The FIFTH gateway. Each one exists so a single injection tier is measured on
+ # a path nothing else contaminates, and this one is the only gateway that
+ # reaches its dependency BY NAME through khaos's resolver.
+ #
+ # It runs the same `gateway` role as the first two, with no new code, and that
+ # is the point: a failed name lookup reaches the same fail-open or fail-closed
+ # branch a failed request does, so the fixture under test is the one khaos
+ # already ships rather than one written to be caught.
+ #
+ # `dns:` is a literal because it has to be. It is fixed at container creation
+ # and there is no API to change it afterwards, so the resolver's address
+ # cannot be discovered first and filled in here.
+ gateway-dns:
+ container_name: khaos-range-gateway-dns
+ build:
+ context: ..
+ dockerfile: range/Dockerfile
+ image: khaos-range:latest
+ command: ["gateway"]
+ dns: ["10.244.244.5"]
+ environment:
+ KHAOS_RANGE_UPSTREAM: "http://dependency.khaos.test:8080"
+ KHAOS_RANGE_FAIL_MODE: "${KHAOS_RANGE_FAIL_MODE:-closed}"
+ KHAOS_RANGE_CACHE_TTL: "${KHAOS_RANGE_CACHE_TTL:-2s}"
+ # Re-resolve the dependency's name on EVERY request, which only this
+ # gateway does. Go's http.Transport keeps connections alive, so a pooled
+ # client resolves once and never again: measured on this fixture, 46
+ # probes produced TWO queries to khaos's resolver, both at the first
+ # request, and every DNS fault armed afterwards was inert. That is a real
+ # property of the fault class rather than a fixture defect, and khaos
+ # reports it when it measures it -- but a fixture that can never
+ # experience the fault cannot be what an exit gate asserts against.
+ KHAOS_RANGE_RERESOLVE: "true"
+ labels:
+ khaos.target: "true"
+ khaos.range: "true"
+ ports:
+ - "127.0.0.1:${KHAOS_RANGE_DNS_PORT:?khaos chooses this port; run `khaos range up`}:8080"
+ depends_on:
+ - idp
+ - resolver
+ networks: [range]
+ restart: "no"
+
networks:
range:
name: khaos-range
+ # An EXPLICIT subnet, and it is load-bearing rather than tidy. Docker refuses
+ # a static container address on a network whose subnet it chose ("user
+ # specified IP address is supported only when connecting to networks with
+ # user configured subnets"), and the DNS tier needs two: the resolver's,
+ # because the gateway's `dns:` is fixed at creation and cannot be filled in
+ # from a discovered value, and the dependency's, because the resolver's zone
+ # is passed as a literal at the same moment.
+ #
+ # The range is namespace-isolated by construction, so this is a structural
+ # literal like the project name rather than a variable to remember to
+ # change. If it ever collides with another network on the host, `range up`
+ # fails with the daemon's own message and the fix is to edit this one line.
+ ipam:
+ config:
+ - subnet: 10.244.244.0/24
diff --git a/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
new file mode 100755
index 00000000..b327cf76
--- /dev/null
+++ b/PROJECTS/intermediate/chaos-engineering-tool/scripts/dns.sh
@@ -0,0 +1,534 @@
+#!/usr/bin/env bash
+# ©AngelaMos | 2026
+# dns.sh
+
+set -euo pipefail
+
+# M8's exit gate. Four things have to be true of the `dns.` family, and every
+# one of them is about khaos declining to say more than it knows.
+#
+# 1. The MASKED faults are refused by name, not approximated. Docker's
+# embedded resolver rewrites REFUSED to SERVFAIL and answers SERVFAIL
+# itself at a measured 4.001s rather than letting a query go unanswered,
+# so neither can reach a container on a user-defined bridge. Delivering
+# SERVFAIL while claiming REFUSED would be worse than refusing: a real
+# library branches on the rcode and the branch is the finding.
+#
+# 2. The delay ceiling is enforced from the same measurement. At or above
+# 4.000s a slow answer silently becomes the timeout fault, so khaos
+# refuses the magnitude rather than serving a different experiment.
+#
+# 3. TRUNCATION SHIPS, which is the correction this milestone is built on.
+# Research 05 recorded it as absorbed entirely by the embedded resolver;
+# it is not. `tc=1` reaches the client in every truncating shape and the
+# client's RFC 5966 TCP retry arrives at khaos's resolver. The original
+# measurement used `dig` without `+ignore`, so dig performed the fallback
+# itself and reported the clean answer -- the instrument absorbed the
+# fault and the daemon was blamed.
+#
+# 4. A fault the target never MET is aborted, not survived. A DNS fault can
+# be armed, confirmed at khaos's own resolver, and reach nothing: Go's
+# HTTP transport resolves a name once per kept-alive connection, measured
+# on this range's own gateway at two lookups across forty-six requests.
+# Every other tier collapses "armed" and "experienced" safely, because a
+# severed path or a broken certificate sits on a route the target cannot
+# avoid. A DNS answer is one it can simply not ask for again.
+
+BIN="${BIN:-$PWD/khaos}"
+
+KHAOS_STATE_DIR="$(mktemp -d)"
+export KHAOS_STATE_DIR
+
+# The fault window has to clear MinimumToClaimClosed, which is 30 paired
+# samples. khaos refuses to call a HELD run conclusive below that, and this
+# gate's truncation assertions are about a HELD verdict's exit code: at 8s and
+# 400ms it got 19 samples, the verdict was honestly marked inconclusive, and it
+# exited 4 instead of 0. That was the gate asking khaos to claim more than its
+# own sample floor allows, which is the same way `tls.sh` failed its first run.
+TIMING=(--baseline-for 5s --fault-for 20s --recover-for 3s --sample-every 300ms)
+
+failures=0
+report=""
+last_code=0
+
+cleanup() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ rm -rf "$KHAOS_STATE_DIR"
+}
+trap cleanup EXIT INT TERM HUP
+
+pass() { echo "ok $1"; }
+fail() { echo "FAIL $1"; failures=$((failures + 1)); }
+
+# The gateway's fail mode is fixed at container creation, deliberately, so a
+# gate spanning both directions has to restart the range. A gateway that could
+# be reconfigured mid-run would let khaos change the target it is measuring.
+start_range() {
+ "$BIN" range down --no-color >/dev/null 2>&1 || true
+ if ! "$BIN" range up --fail-mode "$1" --no-color --quiet >/dev/null 2>&1; then
+ fail "range up --fail-mode $1"
+ return 1
+ fi
+ return 0
+}
+
+# Sets the globals `report` and `last_code`. Never call it through a command
+# substitution: that runs it in a subshell and the report is lost while the exit
+# code still comes back, which looks like a passing gate with no evidence.
+# Appended to by run_experiment, so the census at the bottom proves what the gate
+# RAN rather than comparing one hand-written list to another.
+ran=()
+
+# rogue_asked reads the range rogue provider's own request counter, which is what
+# separates a redirect finding from an outage finding. khaos cannot read it: the
+# redirect address is an operator parameter and khaos has nothing at the far end
+# of it. The host can, because it routes to container addresses on a bridge.
+rogue_asked() {
+ curl -s --max-time 5 "http://10.244.244.20:8080/khaos-rogue-count" 2>/dev/null |
+ sed -E 's/.*"asked":([0-9]+).*/\1/' | grep -E '^[0-9]+$' || echo 0
+}
+
+run_experiment() {
+ local experiment="$1"
+ shift
+ ran+=("$experiment")
+ set +e
+ report="$("$BIN" run "$experiment" --no-color "${TIMING[@]}" "$@" 2>&1)"
+ last_code=$?
+ set -e
+}
+
+assert_verdict() {
+ local label="$1" want="$2"
+ if grep -qE "^ verdict .*${want}" <<<"$report"; then
+ pass "${label} verdict is ${want}"
+ else
+ fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
+ fi
+}
+
+assert_exit() {
+ local label="$1" want="$2"
+ if [[ "$last_code" == "$want" ]]; then
+ pass "${label} exit code ${want}"
+ else
+ fail "${label} exit code ${last_code}, want ${want}"
+ fi
+}
+
+assert_confirmed() {
+ local label="$1"
+ if grep -qE '^ plan confirm fault confirmed by ' <<<"$report"; then
+ pass "${label} confirmed by a measured observable"
+ else
+ fail "${label} reported no confirmation, so the run measured a fault it never proved landed"
+ fi
+}
+
+assert_report_has() {
+ local label="$1" want="$2"
+ if grep -qF "$want" <<<"$report"; then
+ pass "${label}"
+ else
+ fail "${label}: the report never says ${want}"
+ fi
+}
+
+assert_report_lacks() {
+ local label="$1" want="$2"
+ if grep -qF "$want" <<<"$report"; then
+ fail "${label}: the report says ${want}"
+ else
+ pass "${label}"
+ fi
+}
+
+# The observable, not the verdict. This is the half of the gate that separates
+# faults a target cannot tell apart: through the embedded resolver SERVFAIL,
+# REFUSED and a dropped query all reach a Go target as `server misbehaving`
+# with IsTemporary set, so khaos has to name which one it served.
+observable_of() {
+ grep -E '^ plan confirm fault confirmed by ' <<<"$report" |
+ sed -E 's/^.*fault confirmed by //'
+}
+
+echo "== the masked faults are refused by name =="
+
+# The range HAS to be up for this section, and the first version of this script
+# said the opposite: "No range needed: the refusal has to arrive before khaos
+# touches anything, and asserting it with no range running proves it does."
+#
+# Both halves were false. With no range, `khaos run dns.refused` dies in
+# fillFromRange with "no running khaos range was found" and exit 1 — which the
+# old `[[ $last_code == 0 ]] && fail || pass` read as a successful refusal,
+# while the grep for the mechanism would have gone red. It passed only because
+# the range happened to be up from earlier manual runs. Measured: with the range
+# genuinely down, `grep -c "embedded resolver"` is 0.
+#
+# And the refusal is deliberately NOT early. The design doc says so: a masked
+# fault builds, plans and describes like any other entry and is refused when
+# khaos is asked to ARM one, because a catalogue entry that cannot be built
+# broke five separate censuses that walk the catalogue.
+if start_range closed; then
+ for masked in dns.refused dns.timeout; do
+ run_experiment "$masked"
+
+ if [[ "$last_code" == 0 ]]; then
+ fail "${masked} succeeded, so khaos delivered something else and called it this"
+ else
+ pass "${masked} is refused"
+ fi
+
+ # The MECHANISM, which is the assertion that distinguishes khaos's own
+ # refusal from any other non-zero exit. Without it, a missing range, a
+ # missing resolver and a half-specified invocation all read as "khaos
+ # refused this fault by name".
+ assert_report_has \
+ "${masked} names the mechanism rather than just saying no" \
+ "embedded resolver"
+
+ # The remediation, anchored on the fault khaos actually tells you to run
+ # instead. `grep -F "dns."` was inert: it matches the run header, the
+ # plan lines and the experiment's own name.
+ case "$masked" in
+ dns.refused)
+ assert_report_has \
+ "${masked} says to run dns.servfail instead" \
+ "Run \`dns.servfail\`"
+ ;;
+ dns.timeout)
+ assert_report_has \
+ "${masked} says to run dns.slow_response instead" \
+ "Run \`dns.slow_response\`"
+ ;;
+ esac
+ done
+
+ # The ceiling is a measurement, not a preference: at or above 4.000s the
+ # embedded resolver answers SERVFAIL itself at 4.001s and the delay never
+ # reaches the client, so the operator would get the timeout fault under the
+ # slow fault's name.
+ run_experiment dns.slow_response --dns-delay 30s
+
+ if [[ "$last_code" == 0 ]]; then
+ fail "a 30s delay was accepted, which the embedded resolver turns into a different fault"
+ else
+ pass "a delay above the measured ceiling is refused"
+ fi
+
+ assert_report_has "the refusal names the ceiling" "3.999s"
+
+ # And 3.9s is DELIVERED, not merely accepted. The old assertion was
+ # `[[ $last_code == 1 ]] && fail || pass`, which passes on exit 0, 2 and 4
+ # alike — its real content was "no generic error happened". Worse, nothing
+ # in the gate ever delivered a 3.9s delay to anything, so the one number the
+ # research swept to resolve its open question was unexercised.
+ run_experiment dns.slow_response --dns-delay 3900ms
+
+ assert_confirmed "a 3.9s delay"
+ assert_report_has \
+ "3.9s is delivered, one millisecond under the measured ceiling" \
+ "3.9s"
+ assert_report_lacks \
+ "3.9s is not refused as above the ceiling" \
+ "3.999s"
+fi
+
+echo "== against a gateway that fails OPEN =="
+
+if start_range open; then
+ # The whole point of the family. The dependency is HEALTHY and reachable
+ # throughout; only the gateway's ability to find it changes, and it responds
+ # by honouring a credential it had already rejected.
+ for fault in nxdomain servfail slow_response wrong_address \
+ cname_chain cname_loop truncation_no_tcp; do
+ run_experiment "dns.${fault}"
+ assert_verdict "${fault} vs fail-open" "deviated / PERMITTED"
+ assert_exit "${fault} vs fail-open" 2
+ assert_confirmed "${fault}"
+ done
+
+ # The REDIRECT faults have to be findings rather than outages, and this is
+ # the assertion that separates them. Pointing a redirect at an unroutable
+ # address only reproduces what three other tiers already do to an
+ # unreachable dependency; pointed at the range's rogue provider, which
+ # approves everything, the gateway authorises an attacker who won a name.
+ # Asserted on the CONFIRM line, not on the report as a whole. `describeWrong`
+ # already puts the address in the plan's condition line, so a bare grep was
+ # satisfied by the address khaos was ASKED to serve whether or not the
+ # observable ever named what it actually served.
+ run_experiment dns.wrong_address
+
+ if grep -E '^ plan confirm fault confirmed by ' <<<"$report" |
+ grep -qF "10.244.244.20"; then
+ pass "the redirect's own observable names the address it served"
+ else
+ fail "the confirmation does not name the served address: $(observable_of)"
+ fi
+
+ # And it has to say what it REPLACED, which is the honest differential. The
+ # first version compared the served address against itself and read
+ # "with [X] instead of X".
+ if grep -E '^ plan confirm fault confirmed by ' <<<"$report" |
+ grep -qF "instead of 10.244.244.10"; then
+ pass "the redirect names the honest address it replaced"
+ else
+ fail "the confirmation does not name the honest address: $(observable_of)"
+ fi
+
+ # THE INSTRUMENT THAT SEPARATES A REDIRECT FROM AN OUTAGE, read from the
+ # fixture because khaos cannot read it: the redirect address is an operator
+ # parameter and khaos has nothing at the far end of it.
+ #
+ # Without this, `deviated / PERMITTED` on a redirect is produced identically
+ # by "the gateway asked the rogue and the rogue approved" and "the gateway
+ # could not reach the redirect at all and failed open" — and the second is
+ # what three other tiers already test.
+ asked="$(rogue_asked)"
+ if [[ "$asked" -gt 0 ]]; then
+ pass "the redirect was FOLLOWED: the rogue was asked ${asked} time(s)"
+ else
+ fail "the rogue was never asked, so this PERMITTED is an unreachable-dependency finding rather than a redirect finding"
+ fi
+
+ # And khaos must NOT claim the half it did not measure.
+ assert_report_lacks \
+ "khaos does not claim the target verified who it reached" \
+ "verifies who it reached"
+
+ # The two CNAME faults must not be confirmed by each other's answer. A loop
+ # also has three hops, so a hop count alone confirmed the loop's reply as a
+ # chain until the isolation control caught it.
+ run_experiment dns.cname_chain
+ chain_observable="$(observable_of)"
+ run_experiment dns.cname_loop
+ loop_observable="$(observable_of)"
+
+ if [[ "$chain_observable" == "$loop_observable" ]]; then
+ fail "the chain and the loop produced the same observable, so neither is disambiguated"
+ else
+ pass "the chain and the loop produce distinct observables"
+ fi
+
+ if [[ "$loop_observable" == *"cycle"* ]]; then
+ pass "the loop's observable names the cycle"
+ else
+ fail "the loop's observable does not mention a cycle: ${loop_observable}"
+ fi
+
+ # TRUNCATION IS DELIVERABLE, and this is the assertion the research
+ # correction rests on. The gateway sees `tc`, re-asks over TCP, gets the
+ # complete answer and carries on: the control HELD, which is the correct
+ # outcome and must not be called a defect.
+ run_experiment dns.truncation
+ assert_verdict "truncation vs fail-open" "passed / HELD"
+ assert_exit "truncation vs fail-open" 0
+ assert_confirmed "truncation"
+
+ assert_report_has \
+ "truncation proves the TCP half as well as the UDP half" \
+ "completely over TCP"
+
+ # Its sibling must be the OPPOSITE. Both set the bit over UDP, so UDP alone
+ # cannot tell them apart and the difference is the whole finding: one tests
+ # whether the client makes the retry, the other whether the retry is allowed
+ # off the network.
+ run_experiment dns.truncation_no_tcp
+ assert_report_has \
+ "the no-TCP variant proves no transport completes" \
+ "over UDP and over TCP"
+
+ # The TTL faults report UNSUPPORTED, measured rather than assumed. This
+ # gateway re-resolves per request, so it holds no cached answer and the
+ # lifetime khaos serves governs nothing it does. Go has no answer cache and
+ # ignores TTL entirely, and khaos COUNTS that rather than inferring it from
+ # a language it was never told.
+ for fault in ttl_zero ttl_long; do
+ run_experiment "dns.${fault}"
+ assert_verdict "${fault} vs an uncached target" "unsupported"
+ assert_exit "${fault} vs an uncached target" 4
+
+ # Anchored on the LIFETIME khaos served, which is the measurement the
+ # verdict is entitled to make. The target's lookups do not scale with
+ # khaos's traffic -- the gateway caches decisions, so it calls its
+ # dependency less often than khaos probes it -- so a per-request ratio
+ # would be measuring the wrong cache.
+ assert_report_has \
+ "${fault} says it measured the absence of a cache" \
+ "A target honouring it resolves once for the whole window"
+
+ assert_report_lacks \
+ "${fault} does not claim the target survived" \
+ "no bypass was observed"
+ done
+
+ # The causal axis belongs to the idleness pause, and no DNS experiment runs
+ # one. Reporting on a resumed window khaos never collected reads as khaos
+ # having tried the pause and learned nothing, which is a claim about a
+ # measurement it declined to attempt.
+ run_experiment dns.nxdomain
+ assert_report_lacks \
+ "a family that never pauses traffic says nothing about the pause" \
+ "resumed window"
+fi
+
+echo "== against a gateway that fails CLOSED =="
+
+if start_range closed; then
+ # DENIED is the correct outcome with an availability cost, and this family
+ # declares it acceptable, so it must not be called a defect. Research 05
+ # section 10 calls fail-closed bad in a DIFFERENT way: DNS becomes a single
+ # point of failure and anyone who can break it can deny the service. That is
+ # not a security defect and khaos will not report it as one.
+ # The faults that make the dependency UNFINDABLE. A gateway that cannot
+ # resolve its identity provider refuses everything: failed closed, correct,
+ # with an availability cost this family declares acceptable.
+ for fault in nxdomain servfail; do
+ run_experiment "dns.${fault}"
+ assert_verdict "${fault} vs fail-closed" "deviated / DENIED"
+ assert_exit "${fault} vs fail-closed" 0
+ assert_confirmed "${fault}"
+ done
+
+ # THE REDIRECT FAULTS ARE DIFFERENT, AND THIS IS THE FAMILY'S SHARPEST
+ # RESULT: fail-closed is no defence against them.
+ #
+ # Failing closed only helps when the dependency is unreachable. A redirect
+ # to a host that ANSWERS never makes the gateway fail at all — it succeeds,
+ # against the wrong authority, and the rogue approves everything. So a
+ # gateway explicitly configured to fail closed serves invalid credentials
+ # exactly as the fail-open one does, and both report PERMITTED.
+ #
+ # Research 05 section 10 states it as the inversion: the control does not
+ # merely stop working when the name moves, it grants access to whoever took
+ # the name.
+ for fault in wrong_address cname_chain; do
+ run_experiment "dns.${fault}"
+ assert_verdict "${fault} vs fail-closed" "deviated / PERMITTED"
+ assert_exit "${fault} vs fail-closed" 2
+ assert_confirmed "${fault}"
+ done
+
+ # And the instrument that makes that claim rather than inferring it: the
+ # gateway did not fail to reach anything, it reached the rogue.
+ if [[ "$(rogue_asked)" -gt 0 ]]; then
+ pass "fail-closed did not protect the gateway: it reached the rogue and the rogue approved"
+ else
+ fail "the rogue was never asked, so this PERMITTED is not a redirect finding"
+ fi
+
+ # The correct client still falls back to TCP, so the fault it CAN handle
+ # leaves it serving. Both halves of this gate assert the same fault in
+ # opposite fixtures, which is what stops a verdict being right by accident.
+ run_experiment dns.truncation
+ assert_verdict "truncation vs fail-closed" "passed / HELD"
+ assert_exit "truncation vs fail-closed" 0
+fi
+
+echo "== claim 4: a fault the target never MET is aborted, not survived =="
+
+# The script's own header names this as one of four things that have to be true,
+# and the first version never asserted it: no `aborted`, no "never met", and no
+# target that fails to re-resolve. Three of four claims were asserted.
+#
+# The gateway under test re-resolves per request deliberately, so the way to
+# produce a non-delivering target is to point the experiment at a gateway that
+# does NOT: the DIRECT gateway reaches its dependency by container name, which
+# its own embedded resolver answers locally and never forwards to khaos. A fault
+# armed at khaos's resolver therefore reaches it zero times.
+if start_range closed; then
+ # A REAL credential, issued the way the lab issues one. A bogus token makes
+ # the baseline fail with "the valid credential was never accepted", which is
+ # also an inconclusive verdict — so asserting on "any inconclusive verdict"
+ # would have passed for entirely the wrong reason. The assertion below
+ # anchors on the delivery reason's own text for the same reason.
+ direct_url="$(NO_COLOR=1 "$BIN" range status 2>/dev/null |
+ grep -oE 'docker-native tier +http://127\.0\.0\.1:[0-9]+/protected' |
+ grep -oE 'http://127\.0\.0\.1:[0-9]+/protected' | head -1)"
+ gateway="${direct_url%/protected}"
+ token="$(curl -s --max-time 5 -X POST "${gateway}/token" |
+ sed -E 's/.*"token":"([^"]+)".*/\1/')"
+
+ if [[ -z "$direct_url" || -z "$token" || "$token" == *'{'* ]]; then
+ fail "could not issue a credential against the direct gateway, so claim 4 is unasserted"
+ else
+ set +e
+ report="$("$BIN" run dns.nxdomain --no-color "${TIMING[@]}" \
+ --target khaos-range-gateway \
+ --url "$direct_url" \
+ --token "$token" \
+ --dns-host dependency.khaos.test. \
+ --resolver-network khaos-range 2>&1)"
+ last_code=$?
+ set -e
+ ran+=("dns.nxdomain")
+
+ # The DIRECT gateway reaches its dependency by CONTAINER NAME, which its
+ # own embedded resolver answers out of its own table and never forwards.
+ # So a fault armed at khaos's resolver reaches it zero times, however
+ # correctly khaos armed it.
+ #
+ # Anchored on the reason's own wording, not merely on an inconclusive
+ # verdict, because several unrelated problems also produce one.
+ if grep -qF "did not re-resolve its dependency's name" <<<"$report"; then
+ pass "a fault the target never met is reported as never met"
+ elif grep -qF "never used khaos's chaos resolver" <<<"$report"; then
+ pass "a target that never used khaos's resolver voids the run by name"
+ else
+ fail "a target that cannot meet the fault produced: $(grep -E '^ verdict|^ why' <<<"$report" | head -2)"
+ fi
+
+ # What is FORBIDDEN is the clean survival, in either form.
+ if grep -qE '^ verdict .*passed' <<<"$report"; then
+ fail "a fault the target never met was reported as PASSED"
+ else
+ pass "no clean survival was claimed over an undelivered fault"
+ fi
+
+ if [[ "$last_code" == 0 ]]; then
+ fail "an undelivered fault exited 0"
+ else
+ pass "an undelivered fault does not exit 0 (${last_code})"
+ fi
+ fi
+fi
+
+echo "== the census: every catalogued fault was actually run =="
+
+# A catalogued experiment this gate never runs is a failure, for the same reason
+# the identity family's census exists: a family that advertises a fault nothing
+# verifies is the thing this project exists not to do.
+catalogued="$(NO_COLOR=1 "$BIN" catalog 2>/dev/null |
+ grep -oE '^ dns\.[a-z_]+' | tr -d ' ')"
+
+
+missing=0
+for name in $catalogued; do
+ found=0
+ for executed in "${ran[@]}"; do
+ [[ "$executed" == "$name" ]] && found=1
+ done
+ if [[ "$found" == 0 ]]; then
+ fail "the catalogue advertises ${name} and this gate never RAN it"
+ missing=$((missing + 1))
+ fi
+done
+
+if [[ "$missing" == 0 ]]; then
+ pass "every catalogued dns. fault was actually run by this gate (${#ran[@]} runs)"
+fi
+
+# And the reverse: a name this gate ran that is NOT catalogued means the gate is
+# asserting against something that no longer exists.
+for executed in "${ran[@]}"; do
+ if ! grep -qx "$executed" <<<"$catalogued"; then
+ fail "${executed} was run by this gate and is not in the catalogue"
+ fi
+done
+
+if [[ "$failures" -gt 0 ]]; then
+ echo "dns: ${failures} assertion(s) failed"
+ exit 1
+fi
+
+echo "dns: all assertions passed"