Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
#!/usr/bin/env bash
# ©AngelaMos | 2026
# credential.sh
set -euo pipefail
# This is the first family whose authority behaves CORRECTLY. Every other tier
# exists because no correct dependency will emit the defect - no correct
# provider mints `alg: none`, no correct CA signs an expired leaf. Here the
# engines are right and the operator's model is wrong: authentication is
# evaluated once, at connection establishment, and never again. So khaos drives
# three stock engines rather than replacing them, and the gate's job is to prove
# the pair inverts.
#
# 1. THE PAIR. `db_rotate` returns `deviated / PERMITTED` exit 2 on every
# engine, because the session khaos held keeps answering. The same fault
# plus one statement, `db_rotate_terminate`, returns `passed / HELD` exit 0
# on the same engine. One statement is the entire difference between a
# change of admission policy and a revocation.
#
# 3. EVERY OBSERVABLE IS THE ENGINE'S OWN MESSAGE. The confirmation quotes
# what the engine said, per engine, and the four faults produce DISTINCT
# strings. A shared constant describing the technique measures nothing,
# which `scripts/proxy.sh` learned at the cost of two implementations.
#
# 4. THE VERSION COMES FROM THE SERVER. On this host `postgres:18` resolved to
# 18.3 for one measurement and 18.6 hours later, and `mariadb:11.4` moved a
# patch release the same way. A run record naming a tag names nothing.
#
# 5. NOTHING IS LEFT BEHIND. khaos mints a role per run and the engine must
# hold none of them afterwards. The revert RESTORES rather than removes,
# because dropping the role would make khaos's own cleanup read as a target
# that never came back - measured, as a degraded recovery on a run where
# nothing about the engine had degraded.
#
# 6. THE CLASSIFIER IS A DIFFERENTIAL, not a table khaos agrees with itself
# about. `internal/dbx`'s live tests run against all three engines from
# here, including the confirmation oracle's inert control, because a
# self-consistent client agrees with its own mistake.
BIN="${BIN:-$PWD/khaos}"
KHAOS_STATE_DIR="$(mktemp -d)"
export KHAOS_STATE_DIR
# The HELD half of the pair has to clear MinimumToClaimClosed, which is 30
# paired samples: khaos refuses to call a HELD run conclusive below that and
# exits 4 instead of 0. `tls.sh` and `dns.sh` both failed their first run on
# exactly this, so the window is sized for it rather than discovered again.
TIMING=(--baseline-for 4s --fault-for 20s --recover-for 3s --sample-every 300ms)
ENGINES=(postgres mysql mariadb)
declare -A CONTAINER=(
[postgres]=khaos-range-postgres
[mysql]=khaos-range-mysql
[mariadb]=khaos-range-mariadb
)
declare -A ADMIN_VAR=(
[postgres]=POSTGRES_PASSWORD
[mysql]=MYSQL_ROOT_PASSWORD
[mariadb]=MARIADB_ROOT_PASSWORD
)
declare -A ADMIN_ROLE=(
[postgres]=postgres
[mysql]=root
[mariadb]=root
)
declare -A ENGINE_PORT=(
[postgres]=5432
[mysql]=3306
[mariadb]=3306
)
failures=0
report=""
last_code=0
ran=()
active_pid=""
cleanup() {
if [[ -n "$active_pid" ]]; then
kill -9 "$active_pid" 2>/dev/null || true
wait "$active_pid" 2>/dev/null || true
fi
"$BIN" reclaim --no-color >/dev/null 2>&1 || true
"$BIN" range down --no-color >/dev/null 2>&1 || true
rm -rf "$KHAOS_STATE_DIR"
}
trap cleanup EXIT INT TERM HUP
pass() { echo "ok $1"; }
fail() { echo "FAIL $1"; failures=$((failures + 1)); }
# Never call through a command substitution: that runs it in a subshell and the
# report is lost while the exit code still comes back, which looks like a
# passing gate with no evidence.
run_experiment() {
local experiment="$1"
shift
ran+=("$experiment")
run_unmeasured "$experiment" "$@"
}
# run_unmeasured is for a run that is NOT a measurement of the experiment: a
# refusal, or a read of something khaos printed before the baseline. It does not
# enter the census, because a refused run that counted would alibi an experiment
# this gate never actually measured.
#
# The first version of this script achieved that by clearing the census array
# after such a run, which wiped the record of every earlier one and reported
# that the gate had run nothing at all.
run_unmeasured() {
local experiment="$1"
shift
set +e
report="$("$BIN" run "$experiment" --no-color "${TIMING[@]}" "$@" 2>&1)"
last_code=$?
set -e
}
assert_verdict() {
local label="$1" want="$2"
if grep -qE "^ verdict .*${want}" <<<"$report"; then
pass "${label} verdict is ${want}"
else
fail "${label} expected ${want}, got: $(grep -E '^ verdict' <<<"$report" || echo '<none>')"
fi
}
# exit_for_verdict maps an expected verdict onto khaos's exit contract.
#
# It takes the experiment's DECLARED DEFECT DIRECTIONS as its second argument,
# and that is not decoration: the exit code is a function of (verdict,
# polarity), never of the verdict alone. `passed / HELD` is 0 for most of the
# catalogue and 2 for the pooled family, where poolPolarity declares HELD the
# defect -- a lifetime that converts a rotation into a revocation is the thing
# those experiments hunt. An earlier version of this helper read the verdict
# string by itself and reported four pooled assertions as exit 0 when khaos
# correctly returned 2.
#
# The directions are passed per section rather than looked up, so an inverted
# polarity is visible at the call site instead of buried in a helper.
exit_for_verdict() {
local verdict="$1" defects="${2:-PERMITTED}" direction
direction="${verdict##*/ }"
if [[ " ${defects} " == *" ${direction} "* ]]; then
echo 2
return
fi
case "$verdict" in
"passed / "*) echo 0 ;;
"deviated / "*) echo 3 ;;
*)
echo "exit_for_verdict: no code for the verdict '$verdict'" >&2
echo 99
;;
esac
}
assert_exit() {
local label="$1" want="$2"
if [[ "$last_code" == "$want" ]]; then
pass "${label} exit code ${want}"
else
fail "${label} exit code ${last_code}, want ${want}"
fi
}
assert_confirmed() {
local label="$1"
if grep -qE '^ plan confirm fault confirmed by ' <<<"$report"; then
pass "${label} confirmed by a measured observable"
else
fail "${label} reported no confirmation, so the run measured a fault it never proved landed"
fi
}
assert_report_has() {
local label="$1" want="$2"
if grep -qF "$want" <<<"$report"; then
pass "${label}"
else
fail "${label}: the report never says ${want}"
fi
}
assert_report_lacks() {
local label="$1" want="$2"
if grep -qF "$want" <<<"$report"; then
fail "${label}: the report says ${want}"
else
pass "${label}"
fi
}
assert_control_held() {
local label="$1"
# The subject is a parameter because this one helper now serves two kinds of
# target. For ten experiments it is an engine checking a password; for the
# service-to-service one it is a caller authenticating a credential, and
# that distinction is the whole reason the fixture is measurable at all.
local subject="${2:-the engine checks passwords}"
assert_report_lacks \
"${label} established that ${subject} at all" \
"the invalid credential was never rejected"
}
# EVERY helper below that greps ends in `|| true`, and that is not defensive
# noise: with `set -e`, a grep inside a command substitution that legitimately
# matches nothing kills the script, and `grep -c` does it even when printing a
# perfectly good `0`. The symptom is a gate that exits 1 with no FAIL line at
# all, which reads like a mystery rather than like a bug. It has fired three
# times in this one file: on an unmatched confirmation line, on a count of
# leftovers that was correctly zero, and on an absent environment variable.
observable_of() {
grep -E '^ plan confirm fault confirmed by ' <<<"$report" |
sed -E 's/^.*fault confirmed by //' || true
}
admin_password() {
docker inspect "${CONTAINER[$1]}" \
--format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null |
grep "^${ADMIN_VAR[$1]}=" | cut -d= -f2- || true
}
engine_address() {
docker inspect "${CONTAINER[$1]}" \
--format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \
2>/dev/null
}
# leftover_roles counts roles khaos minted and did not remove. Read over
# loopback INSIDE the container on purpose: that is the `trust` path research
# 07 section 6.1 warns about, which makes it useless as a probe and exactly
# right as the gate's own administrative read.
leftover_roles() {
local engine="$1"
case "$engine" in
postgres)
docker exec "${CONTAINER[$engine]}" psql -U postgres -tAc \
"SELECT count(*) FROM pg_roles WHERE rolname LIKE 'khaos_cred_%'" \
2>/dev/null | tail -1 | tr -d '[:space:]'
;;
*)
docker exec "${CONTAINER[$engine]}" sh -c \
"exec \"\$(command -v mariadb || command -v mysql)\" -uroot -p\"\$${ADMIN_VAR[$engine]}\" -N -B -e \"SELECT count(*) FROM mysql.user WHERE user LIKE 'khaos\\_cred\\_%'\"" \
2>/dev/null | tail -1 | tr -d '[:space:]'
;;
esac
}
kill_credential_at() {
local engine="$1" stage="$2" log attempt reached=0 had_role=0 baseline=4s
if [[ "$stage" == baseline ]]; then
baseline=60s
fi
log="${KHAOS_STATE_DIR}/kill-${engine}-${stage}.log"
"$BIN" run credential.db_rotate --target "${CONTAINER[$engine]}" \
--engine "$engine" --baseline-for "$baseline" --fault-for 2s \
--recover-for 60s --sample-every 300ms --no-color \
>"$log" 2>&1 &
active_pid=$!
for ((attempt = 0; attempt < 60; attempt++)); do
if grep -qE "plan[[:space:]]+${stage}[[:space:]]" "$log"; then
reached=1
break
fi
kill -0 "$active_pid" 2>/dev/null || break
sleep 1
done
if [[ "$reached" != 1 ]]; then
fail "${engine} did not reach ${stage} before interruption"
cat "$log"
elif [[ "$(leftover_roles "$engine")" != 1 ]]; then
fail "${engine} ${stage} did not hold exactly one temporary role"
else
had_role=1
fi
kill -9 "$active_pid" 2>/dev/null || true
wait "$active_pid" 2>/dev/null || true
active_pid=""
if "$BIN" reclaim --no-color >>"$log" 2>&1 && \
[[ "$(leftover_roles "$engine")" == 0 && "$had_role" == 1 ]]; then
pass "${engine} kill during ${stage} left a role that cold reclaim removed"
else
fail "${engine} cold cleanup after ${stage} left a role or failed"
cat "$log"
fi
}
# The pool configuration is fixed at container creation, deliberately, so a
# gate spanning all three has to restart the range. A fixture that could be
# reconfigured mid-run would let khaos change the target it is measuring.
held_containers() {
docker ps -a --format '{{.Names}}' |
grep -c -- '-khaos-held' | tr -d '[:space:]' || true
}
env_of() {
docker inspect "$1" --format '{{range .Config.Env}}{{println .}}{{end}}' \
2>/dev/null | grep -E "^$2=" | tail -1 || true
}
start_range() {
local pool="${1:-unbounded}"
local secret="${2:-unchecked}"
"$BIN" range down --no-color >/dev/null 2>&1 || true
# The OUTPUT, not just the fact. A bare "range up failed" is what a
# transient startup error looks like in a two-hour gate run, and it
# leaves nothing to diagnose from.
local started
if ! started="$("$BIN" range up --with-db --pool-mode "$pool" \
--secret-mode "$secret" --no-color --quiet 2>&1)"; then
fail "range up --with-db --pool-mode $pool --secret-mode $secret: $(tail -3 <<<"$started")"
printf '%s\n' "$started" >&2
return 1
fi
return 0
}
secret_leftovers() {
docker ps -a --filter "label=khaos.mutator=true" --format '{{.Names}}' |
wc -l | tr -d '[:space:]'
}
held_originals() {
docker exec khaos-range-gateway-secret \
sh -c 'ls /run/khaos-secrets/ 2>/dev/null' 2>/dev/null |
grep -c 'khaos-held' | tr -d '[:space:]' || true
}
echo "== the stateful tier is refused by name when it is not running =="
# The refusal has to arrive with a range UP and the database tier DOWN, which is
# the state an operator reaches by running `khaos range up` and then asking for
# a credential experiment. With no range at all the run dies in fillFromRange
# with a different message, and a gate that accepted either would pass against
# the wrong one.
"$BIN" range down --no-color >/dev/null 2>&1 || true
if "$BIN" range up --no-color --quiet >/dev/null 2>&1; then
run_unmeasured credential.db_rotate
assert_exit "a credential experiment with no stateful tier" 1
assert_report_has \
"the refusal names the missing tier" \
"the running range has none"
assert_report_has \
"the refusal names the command that fixes it" \
"khaos range up --with-db"
assert_report_has \
"the refusal says why khaos does not fake it" \
"the finding IS what correct ones do"
else
fail "range up (without the stateful tier)"
fi
echo "== THE PAIR: rotation is not revocation, and termination is =="
if start_range unbounded; then
for engine in "${ENGINES[@]}"; do
container="${CONTAINER[$engine]}"
run_experiment credential.db_rotate \
--target "$container" --engine "$engine"
assert_verdict "${engine} db_rotate" "deviated / PERMITTED"
assert_exit "${engine} db_rotate" 2
assert_confirmed "${engine} db_rotate"
assert_control_held "${engine} db_rotate"
assert_report_has \
"${engine} db_rotate names what khaos did to the credential" \
"by rotating its password at the engine and terminating nothing"
assert_report_has \
"${engine} db_rotate reports the exposure as a lower bound" \
"which is the lower bound on the exposure"
rotate_observable="$(observable_of)"
run_experiment credential.db_rotate_terminate \
--target "$container" --engine "$engine"
assert_verdict "${engine} db_rotate_terminate" "passed / HELD"
assert_exit "${engine} db_rotate_terminate" 0
assert_confirmed "${engine} db_rotate_terminate"
assert_control_held "${engine} db_rotate_terminate"
assert_report_has \
"${engine} db_rotate_terminate says what HELD means here" \
"closed by the engine, which is what makes termination"
# The inversion, read off the FINDING rather than off the verdict
# label: the rotation half reports the invalidated credential being
# honoured, and the half that also terminated must report no such
# thing. Two assertions on two reports, which is what makes the pair
# the exit gate rather than two unrelated runs.
assert_report_lacks \
"${engine} nothing was honoured once the sessions were terminated" \
"was still accepted"
if [[ -n "$rotate_observable" ]]; then
pass "${engine} the rotation produced an observable to compare"
else
fail "${engine} the rotation produced no observable to compare"
fi
done
fi
echo "== the pooled fixture: the fail-open is the DEFAULT configuration =="
# Three pool configurations, and the verdict has to follow the configuration
# rather than the fault. The fault is identical in all three runs.
#
# unbounded the shape a Go service has when nobody called
# SetConnMaxLifetime. HELD is the defect: the service carried on.
# bounded the knob that converts a rotation into a revocation. DENIED is
# correct, and it arrives as UNAVAILABILITY rather than a refusal,
# which is why the experiment declares what failing closed means.
# held the same lifetime as bounded, plus one connection checked out
# and never returned. The cleaner only walks the IDLE list, so the
declare -A WANT_VERDICT=(
[unbounded]="passed / HELD"
[bounded]="deviated / DENIED"
[held]="passed / HELD"
)
# What khaos DID is carried by the HELD CAVEAT for these two, not by the defect
# phrase: that phrase is part of a leak reason and a leak is exactly what a
# HELD finding does not have. Asserting the defect phrase here failed six times
# against a product that was reporting correctly.
declare -A WANT_CAVEAT=(
[credential.db_pool_stale]="khaos rotated the credential and the service carried on"
[credential.db_pending_window]="the liveness endpoint answered throughout on a connection that authenticated before the rotation"
)
for pool in unbounded bounded held; do
if ! start_range "$pool"; then
continue
fi
for experiment in credential.db_pool_stale credential.db_pending_window; do
run_experiment "$experiment"
label="${experiment#credential.} on a ${pool} pool"
assert_verdict "$label" "${WANT_VERDICT[$pool]}"
assert_exit "$label" \
"$(exit_for_verdict "${WANT_VERDICT[$pool]}" "HELD PERMITTED")"
assert_confirmed "$label"
assert_control_held "$label"
# A HELD verdict has to say what khaos did and what HELD means here.
# A DENIED one carries no caveat at all, deliberately: the caveat is
# the explanation of a survival.
if [[ "${WANT_VERDICT[$pool]}" == "passed / HELD" ]]; then
assert_report_has \
"${label} says what khaos did and what surviving it means" \
"${WANT_CAVEAT[$experiment]}"
else
assert_report_lacks \
"${label} does not explain a survival that did not happen" \
"${WANT_CAVEAT[$experiment]}"
fi
# The sharpest row, asserted per experiment rather than after the loop,
# where $report held the LAST experiment's output: a lifetime IS set in
# held mode and the service still carried on.
if [[ "$pool" == "held" &&
"$experiment" == "credential.db_pool_stale" ]]; then
assert_report_has \
"a checked-out connection is not bounded by the lifetime" \
"a connection that is checked out rather than idle is not bounded by it at all"
fi
# The correct configuration has to lose access rather than merely slow
# down, or DENIED would be an artefact of the probe timing out.
if [[ "$pool" == "bounded" ]]; then
assert_report_has \
"${label} stopped being able to serve" \
"the valid credential stopped being accepted"
fi
done
done
echo "== the secret file: one fault of four is a finding, and the other three say why =="
declare -A WANT_SECRET_UNCHECKED=(
[credential.secret_file_empty]="deviated / PERMITTED"
[credential.secret_file_truncated]="deviated / DENIED"
[credential.secret_file_missing]="deviated / DENIED"
[credential.secret_file_unreadable]="deviated / DENIED"
)
if start_range unbounded unchecked; then
declare -A SECRET_OBSERVABLE=()
for experiment in "${!WANT_SECRET_UNCHECKED[@]}"; do
run_experiment "$experiment"
label="${experiment#credential.} against a verifier that does not check the length"
assert_verdict "$label" "${WANT_SECRET_UNCHECKED[$experiment]}"
assert_exit "$label" "$(exit_for_verdict "${WANT_SECRET_UNCHECKED[$experiment]}")"
assert_confirmed "$label"
assert_control_held "$label"
SECRET_OBSERVABLE[$experiment]="$(observable_of)"
done
# The confirmation is read off the TARGET's own filesystem, so each fault
# has to name what the target now has rather than what khaos asked for.
assert_report_has \
"the empty-secret confirmation names the target's own copy" \
"is now zero bytes, and os.ReadFile returns no error on one"
distinct=$(printf '%s\n' "${SECRET_OBSERVABLE[@]}" | sort -u | wc -l)
if [[ "$distinct" == "${#SECRET_OBSERVABLE[@]}" ]]; then
pass "all four secret faults produce DISTINCT observables"
else
fail "the four secret faults produced ${distinct} distinct observables, so some of them are not evidence"
fi
# khaos's helper is short-lived by construction, and an orphan would be a
# container holding two file-permission capabilities over the operator's
# own volume.
left="$(secret_leftovers)"
if [[ "$left" == "0" ]]; then
pass "no mutator container is left behind"
else
fail "${left} mutator container(s) left behind"
fi
# The original is kept BESIDE the secret, inside the target's own mount, so
# khaos never carries it out - and the revert has to remove it again.
held="$(held_originals)"
if [[ "$held" == "0" ]]; then
pass "the original khaos kept beside the secret is gone after the revert"
else
fail "khaos left ${held} copy of the original inside the target's mount"
fi
fi
echo "== the length check is the fix, and khaos says so =="
if start_range unbounded validated; then
run_experiment credential.secret_file_empty
assert_verdict \
"secret_file_empty against a verifier that checks the length" \
"deviated / DENIED"
assert_exit \
"secret_file_empty against a verifier that checks the length" 3
assert_confirmed \
"secret_file_empty against a verifier that checks the length"
assert_report_lacks \
"no credential was honoured once the length was checked" \
"was accepted"
fi
echo "== a secret khaos cannot reach reversibly is refused by name =="
if start_range unbounded unchecked; then
# A path on no mount at all. A helper cannot reach it, and a cold-started
# revert could not put it back, so khaos declines rather than breaking
# something it could not undo.
run_unmeasured credential.secret_file_empty \
--target khaos-range-gateway-secret \
--url "http://127.0.0.1:1/protected" \
--token nothing \
--secret-path /etc/hostname
if [[ "$last_code" != 0 ]]; then
pass "a secret on no mount at all is refused"
else
fail "khaos broke a file it could not have restored"
fi
assert_report_has \
"the refusal names the reason and the fix" \
"Deliver the secret on a volume or a bind-mounted directory"
# A tmpfs is private to the container's own namespace, so a helper mounting
# the same spec gets an EMPTY filesystem: khaos would write a secret into
# nowhere and report a fault that landed.
run_unmeasured credential.secret_file_empty \
--target khaos-range-postgres \
--url "http://127.0.0.1:1/protected" \
--token nothing \
--secret-path /var/lib/postgresql/khaos-probe
if [[ "$last_code" != 0 ]]; then
pass "a secret on a tmpfs mount is refused"
else
fail "khaos accepted a secret on a tmpfs, which a helper cannot reach"
fi
assert_report_has \
"the tmpfs refusal names why a helper cannot reach it" \
"private to the container"
fi
echo "== the environment: the one fault class that REPLACES the container =="
# A container's environment is fixed at creation. `ContainerUpdate` accepts an
# Env field, returns HTTP 200 with no warning, and changes nothing - measured -
# so these two replace the container, and the mechanism was chosen to keep the
# ORIGINAL rather than to rebuild it: a recreate from an inspection loses the
# anonymous volume with its data stranded, every prior log line, and the
# container id invariant 8 re-verifies against. khaos renames the original
# aside and renames it back on the way out.
#
# The two faults are INDISTINGUISHABLE to a service that reads with os.Getenv,
# which is the whole teaching point, so both are expected to read the same
# against each fixture mode.
declare -A WANT_ENV=(
[unchecked]="deviated / PERMITTED"
[validated]="deviated / DENIED"
)
for secret in unchecked validated; do
if ! start_range unbounded "$secret"; then
continue
fi
for experiment in credential.env_empty credential.env_unset; do
run_experiment "$experiment"
label="${experiment#credential.} against a ${secret} verifier"
assert_verdict "$label" "${WANT_ENV[$secret]}"
assert_exit "$label" "$(exit_for_verdict "${WANT_ENV[$secret]}")"
assert_confirmed "$label"
assert_control_held "$label"
done
# The original has to be back, running, under its own name - and nothing
# held aside. A revert that renamed it back and forgot to start it left the
# next run reporting the range as missing the fixture entirely.
if docker ps --filter name=khaos-range-gateway-env --filter status=running \
--format '{{.Names}}' | grep -q 'khaos-range-gateway-env$'; then
pass "the original container is back and running after the revert"
else
fail "the original container did not come back running"
fi
held="$(held_containers)"
if [[ "$held" == "0" ]]; then
pass "no container is left renamed aside"
else
fail "${held} container(s) left renamed aside"
fi
# The variable is back with its ORIGINAL value, not an empty one: the
# revert restores the container rather than patching the environment.
restored="$(env_of khaos-range-gateway-env KHAOS_RANGE_SECRET_VALUE)"
if [[ -n "$restored" && "$restored" != "KHAOS_RANGE_SECRET_VALUE=" ]]; then
pass "the variable is back with its original value"
else
fail "the variable came back as ${restored:-<absent>}"
fi
done
echo "== an environment fault khaos cannot deliver is refused by name =="
if start_range unbounded unchecked; then
port="$(docker port khaos-range-gateway-env 8080 2>/dev/null |
head -1 | cut -d: -f2)"
token="$(curl -s --max-time 5 -X POST \
"http://127.0.0.1:${port}/secret-token" 2>/dev/null |
sed -E 's/.*"token":"([^"]+)".*/\1/')"
# A variable the IMAGE defines as well. The daemon merges the image's
# environment into any key a create request omits, so a replacement made
# without it gets the image's value back and khaos would report a fault the
# daemon had undone for it.
run_unmeasured credential.env_unset \
--target khaos-range-gateway-env \
--url "http://127.0.0.1:${port}/protected" \
--token "$token" \
--variable PATH
if [[ "$last_code" != 0 ]]; then
pass "a variable the image defines is refused"
else
fail "khaos reported an unset the daemon would have undone"
fi
assert_report_has \
"the refusal names the mechanism and the fault to use instead" \
"gets the image's value back"
# A variable that is not there at all. Nothing to take away, and the
# refusal names what IS there so the operator can see the typo.
run_unmeasured credential.env_unset \
--target khaos-range-gateway-env \
--url "http://127.0.0.1:${port}/protected" \
--token "$token" \
--variable KHAOS_NOT_A_VARIABLE
if [[ "$last_code" != 0 ]]; then
pass "a variable the target does not carry is refused"
else
fail "khaos took away a variable that was never there"
fi
assert_report_has \
"the refusal names the variables the target DOES carry" \
"It carries:"
fi
echo "== the pooled faults put BOTH containers to the refusal predicates =="
# khaos measures one container and performs the rotation at another, so the
# engine's consent is not implied by the target's. A rotation at a container
# that never consented would be khaos mutating something nobody approved.
if start_range unbounded; then
run_unmeasured credential.db_pool_stale \
--target khaos-range-gateway-db \
--engine-target khaos-range-idp \
--engine postgres \
--subject-role appuser \
--subject-password-var KHAOS_RANGE_DB_PASSWORD \
--ready-url "http://127.0.0.1:1/ready" \
--url "http://127.0.0.1:1/protected" \
--token nothing
if [[ "$last_code" != 0 ]]; then
pass "an engine container with no administrative credential is refused"
else
fail "khaos accepted an engine container it could not read a credential from"
fi
fi
echo "== the two other controls an operator reaches for do not revoke either =="
for engine in "${ENGINES[@]}"; do
container="${CONTAINER[$engine]}"
run_experiment credential.db_account_lock \
--target "$container" --engine "$engine"
assert_verdict "${engine} db_account_lock" "deviated / PERMITTED"
assert_exit "${engine} db_account_lock" 2
assert_confirmed "${engine} db_account_lock"
assert_control_held "${engine} db_account_lock"
assert_report_has \
"${engine} db_account_lock confirmation quotes the engine on the lock" \
"because the account is locked"
lock_observable="$(observable_of)"
run_experiment credential.db_credential_expiry \
--target "$container" --engine "$engine"
assert_verdict "${engine} db_credential_expiry" "deviated / PERMITTED"
assert_exit "${engine} db_credential_expiry" 2
assert_confirmed "${engine} db_credential_expiry"
assert_control_held "${engine} db_credential_expiry"
assert_report_has \
"${engine} db_credential_expiry confirmation names the validity" \
"because the credential's validity has passed"
expiry_observable="$(observable_of)"
# Distinct observables, per engine. The lock and the expiry both refuse a
# correct password at a new connection, so a confirmation that described the
# technique rather than reading the engine back would be identical for both.
if [[ "$lock_observable" != "$expiry_observable" ]]; then
pass "${engine} the lock and the expiry produce DISTINCT observables"
else
fail "${engine} the lock and the expiry produced the same observable, so neither is evidence"
fi
done
echo "== the engine version comes from the server, not from the image tag =="
for engine in "${ENGINES[@]}"; do
run_unmeasured credential.db_rotate \
--target "${CONTAINER[$engine]}" --engine "$engine"
if grep -qE '^ plan credential tier open on .*which reports "' <<<"$report"; then
reported="$(grep -oE 'which reports "[^"]+"' <<<"$report")"
case "$engine" in
postgres) expect='PostgreSQL 18\.[0-9]+' ;;
mysql) expect='8\.4\.[0-9]+' ;;
mariadb) expect='11\.4\.[0-9]+-MariaDB' ;;
esac
if grep -qE "$expect" <<<"$reported"; then
pass "${engine} reports a patch level its image tag does not carry (${reported})"
else
fail "${engine} reported ${reported}, which does not carry a patch level and so could be the tag echoed back"
fi
else
fail "${engine} never reported the version the server gave it"
fi
done
echo "== khaos leaves no role of its own behind =="
for engine in "${ENGINES[@]}"; do
left="$(leftover_roles "$engine")"
if [[ "$left" == "0" ]]; then
pass "${engine} holds no khaos-minted role after the runs"
else
fail "${engine} still holds ${left:-<unreadable>} khaos-minted role(s)"
fi
done
echo "== the classifier is a differential against all three engines =="
for engine in "${ENGINES[@]}"; do
kill_credential_at "$engine" baseline
kill_credential_at "$engine" recover
done
# Including the confirmation oracle's inert control, which is the one assertion
# an oracle gutted to `return true` cannot satisfy.
differential_env=()
for engine in "${ENGINES[@]}"; do
address="$(engine_address "$engine")"
password="$(admin_password "$engine")"
database=""
if [[ "$engine" == "postgres" ]]; then
database="postgres"
fi
if [[ -z "$address" || -z "$password" ]]; then
fail "could not read ${engine}'s address or administrative credential from its own container"
continue
fi
name="KHAOS_DBX_LIVE_$(echo "$engine" | tr '[:lower:]' '[:upper:]')"
differential_env+=(
"${name}=${address}:${ENGINE_PORT[$engine]},${database},${ADMIN_ROLE[$engine]},${password}"
)
done
if [[ "${#differential_env[@]}" == "${#ENGINES[@]}" ]]; then
set +e
differential="$(env "${differential_env[@]}" \
go test ./internal/dbx/ -count=1 2>&1)"
differential_code=$?
set -e
if [[ "$differential_code" == 0 ]]; then
pass "internal/dbx agrees with all three live engines"
else
fail "internal/dbx disagrees with a live engine: $(tail -5 <<<"$differential")"
fi
# A differential that SKIPPED every engine would also exit 0, which is the
# inert-control shape this project has shipped before.
skipped="$(env "${differential_env[@]}" \
go test ./internal/dbx/ -count=1 -v -run TestRotationIsNotRevocation 2>&1 |
grep -c 'SKIP' || true)"
if [[ "$skipped" == "0" ]]; then
pass "the differential ran against every engine rather than skipping them"
else
fail "the differential SKIPPED ${skipped} engine(s), so it proved nothing"
fi
fi
echo "== service to service: a DENY and a SILENCE are different answers =="
start_authz_range() {
local mode="$1"
"$BIN" range down --no-color >/dev/null 2>&1 || true
local started
if ! started="$("$BIN" range up --authz-mode "$mode" --no-color --quiet 2>&1)"; then
fail "range up --authz-mode $mode: $(tail -3 <<<"$started")"
printf '%s\n' "$started" >&2
return 1
fi
return 0
}
declare -A WANT_AUTHZ=(
[collapses]="deviated / PERMITTED"
[distinguishes]="passed / HELD"
[retries]="passed / HELD"
[cached]="passed / HELD"
)
for mode in collapses distinguishes retries cached; do
if ! start_authz_range "$mode"; then
continue
fi
run_experiment credential.apikey_revoked
label="apikey_revoked against a ${mode} caller"
assert_verdict "$label" "${WANT_AUTHZ[$mode]}"
assert_exit "$label" \
"$(exit_for_verdict "${WANT_AUTHZ[$mode]}" "PERMITTED DENIED")"
assert_confirmed "$label"
assert_control_held "$label" "the caller authenticates credentials"
# The confirmation is TWO-SIDED, and both halves are quoted: the credential
# under test is refused AND a control credential khaos never named is still
# allowed. Without the second half a dependency that had started refusing
# everything would confirm every deny khaos armed, and this experiment
# reports a refusal of the valid credential as a defect of the target.
assert_report_has \
"${label}: the confirmation reads both sides of the deny" \
"403 for the credential under test and 200 for a control credential"
# The SECOND WINDOW has to have run. Without this the gate would pass on
# phase one alone: the collapsing caller's acceptance of a denied
# credential is a finding by itself, so deleting the shift entirely would
# leave the verdict and the exit code unchanged.
assert_report_has \
"${label}: khaos measured a second window" \
"plan shift"
assert_report_has \
"${label}: the second window is named in the evidence" \
"answered 503 to everything"
done
echo "== the two phases are read as four callers, not two =="
# The discriminator, stated as the gate's own comparison. The verdicts above
# already differ; what these assert is that khaos SAYS which caller it saw,
# because "deviated" with no mechanism is a finding an operator cannot act on.
if start_authz_range collapses; then
run_experiment credential.apikey_revoked
assert_report_has \
"the collapse is named as serving in both phases" \
"permitted access during both"
assert_report_has \
"both dependency outcomes are named" \
"definitive denial and an unavailable dependency"
assert_report_lacks \
"a collapsing caller is not credited with the third branch" \
"That is the third branch"
fi
if start_authz_range distinguishes; then
run_experiment credential.apikey_revoked
assert_report_has \
"the correct caller is credited with the third branch, observed" \
"That is the third branch, and khaos observed it rather than inferring it"
assert_report_lacks \
"and is not reported as having collapsed anything" \
"permitted access during both"
# The negative control for the retry-storm instrument. A caller that asks
# once per request must NOT be reported as retrying, or the instrument says
# yes to everything and the row below proves nothing.
assert_report_lacks \
"a caller that asked once per request is not called a retry storm" \
"asked again about a credential it had already been told no about"
fi
if start_authz_range retries; then
run_experiment credential.apikey_revoked
# Measured from OUTSIDE the target, by counting what khaos's own dependency
# answered: only the credential under test is ever answered with a
# definitive refusal, so that count is the number of times the caller asked
# about a credential it had already been told no about.
assert_report_has \
"the retry storm is counted against the requests that caused it" \
"asked again about a credential it had already been told no about"
# And it still fails CLOSED, which is why this is a third mode rather than
# a variant of the collapse: the right verdict reached the wrong way.
assert_report_has \
"a retrying caller is still credited with both branches" \
"That is the third branch"
fi
echo "== the regression control: a correct caller that refuses with 503 =="
if start_authz_range cached; then
run_experiment credential.apikey_revoked
assert_verdict "a caller that refuses with 503" "passed / HELD"
assert_exit "a caller that refuses with 503" 0
assert_confirmed "a caller that refuses with 503"
assert_report_lacks \
"a correct caller is not told its fallback is to allow" \
"this target's fallback is to allow"
assert_report_lacks \
"and is not reported as having served in both phases" \
"permitted access during both"
assert_report_lacks \
"and no bypass is claimed against it" \
"was still accepted"
assert_report_has \
"it is credited with the third branch like any correct caller" \
"That is the third branch"
fi
echo "== the authorization fixture is refused by name when it is not running =="
# The refusal has to arrive with a range UP and this one fixture stopped, which
# is the state an operator reaches when a container has died. A missing fixture
# otherwise surfaces as a published port of zero, and the run blames the target
# for not enforcing anything.
if start_authz_range distinguishes; then
docker stop khaos-range-gateway-authz >/dev/null 2>&1 || true
run_unmeasured credential.apikey_revoked
assert_exit "an authorization experiment with no caller fixture" 1
assert_report_has \
"the refusal names the missing fixture" \
"the running range has no khaos-range-gateway-authz"
docker start khaos-range-gateway-authz >/dev/null 2>&1 || true
fi
echo "== census: every catalogued credential. experiment was run here =="
catalogued="$(NO_COLOR=1 "$BIN" run --help |
grep -oE '^ credential\.[a-z_]+' | tr -d ' ' | sort -u || true)"
if [[ -z "$catalogued" ]]; then
fail "the census read no credential experiments at all, so it cannot fail for the right reason"
fi
missing=""
for name in $catalogued; do
found=0
for was in "${ran[@]}"; do
if [[ "$was" == "$name" ]]; then
found=1
break
fi
done
if [[ "$found" == 0 ]]; then
missing="${missing} ${name}"
fi
done
if [[ -z "$missing" ]]; then
pass "every catalogued credential. fault was actually run by this gate"
else
fail "this gate never ran:${missing}"
fi
echo
if [[ "$failures" == 0 ]]; then
echo "credential gate: all assertions passed"
exit 0
fi
echo "credential gate: ${failures} assertion(s) failed"
exit 1