Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
// ©AngelaMos | 2026
// engine.go
package experiment
import (
"context"
"errors"
"fmt"
"math/rand/v2"
"time"
"github.com/CarterPerez-dev/khaos/internal/probe"
)
type Stage string
const (
StageBaseline Stage = "baseline"
StageInject Stage = "inject"
StageConfirm Stage = "confirm"
StageObserve Stage = "observe"
StageAct Stage = "act"
StageShift Stage = "shift"
StageColdStart Stage = "cold-start"
StageIdle Stage = "idle"
StageResume Stage = "resume"
StageRevert Stage = "revert"
StageRecover Stage = "recover"
StageVerdict Stage = "verdict"
)
const (
detailBaseline = "establishing steady state"
detailObserve = "measuring under fault"
detailRevert = "removing the fault"
detailRecover = "confirming the target came back"
detailInjecting = "%s on %s"
detailConfirmed = "fault confirmed by %s"
detailUnconfirm = "the fault was not confirmed, aborting before measuring"
detailVerdict = "%s / %s"
detailActFailed = "the mid-run action failed, so nothing measured after it means anything: %v"
detailShift = "%s, and measuring again"
detailShiftFailed = "the second phase could not be established, so khaos has one window where it needs two: %v"
detailIdle = "a credential leaked; pausing traffic for %s to see whether the cache is being re-armed"
detailResume = "resuming traffic with the fault still applied"
detailColdStart = "restarting the target so it initialises with the dependency already unreachable"
// How long the target gets to answer again after the restart. The
// settling samples are discarded: counting them would report khaos's own
// restart as the target having stopped answering.
settleFor = 30 * time.Second
// Breaks aliasing against the periodic target behaviour these experiments
// measure: cache TTLs, token refresh and crons.
jitterFraction = 0.2
)
type Event struct {
Stage Stage
At time.Time
Detail string
}
type Reporter func(Event)
// Injector keeps the engine away from Docker, so the lifecycle is testable
// without a daemon.
type Injector interface {
Inject(ctx context.Context, kind, target string) error
Confirm(ctx context.Context) (string, error)
Revert(ctx context.Context) error
// Restart restarts the container the fault was applied to, for the one
// experiment whose fault is the target initialising without a dependency.
Restart(ctx context.Context) error
}
var ErrNeverCameBack = errors.New(
"khaos restarted the target and it never answered again, so there is no window to measure; the fault is still applied and was reverted",
)
var ErrNoProbePair = errors.New(
"an experiment needs both a valid and an invalid probe; the invalid one is the negative control that makes the verdict mean anything",
)
type Engine struct {
Injector Injector
Report Reporter
Now func() time.Time
// SettleFor bounds how long a cold-started target gets to answer again.
// Injectable for the same reason Now is: the default is longer than a
// test can wait for.
SettleFor time.Duration
RollbackTimeout time.Duration
}
func (e Engine) rollback(ctx context.Context) error {
timeout := e.RollbackTimeout
if timeout <= 0 {
timeout = 30 * time.Second
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout)
defer cancel()
return e.Injector.Revert(cleanup)
}
func (e Engine) recoverFault(ctx context.Context) {
if failure := recover(); failure != nil {
err := e.rollback(ctx)
if err != nil {
panic(fmt.Errorf("%v; emergency rollback failed: %w", failure, err))
}
panic(failure)
}
}
// abandon reverts the fault and then concludes. The cause is joined with the
// rollback error, not replaced by it: the caller maps a rollback failure to its
// own exit code.
func (e Engine) abandon(
ctx context.Context,
spec Experiment,
result Result,
observation Observation,
cause error,
) (Result, error) {
revertErr := e.rollback(ctx)
observation.RollbackFailed = revertErr != nil
if ctx.Err() != nil {
observation = markInterrupted(observation)
}
return e.conclude(
result,
observation,
spec.Polarity,
errors.Join(cause, revertErr),
)
}
// measureBaseline counts the dependency consultations while the target is still
// healthy: zero means it never used the dependency khaos is about to break.
func (e Engine) measureBaseline(
ctx context.Context,
spec Experiment,
timing Timing,
observation Observation,
) Observation {
e.emit(StageBaseline, detailBaseline)
before, beforeErr := e.consulted(spec)
observation.Baseline = e.sample(
ctx,
spec.Probes,
timing.BaselineFor,
timing.SampleInterval,
)
after, afterErr := e.consulted(spec)
observation.ConsultedWhat = spec.ConsultedWhat
observation.Unsupportable = spec.Unsupportable
observation.CacheRequired = spec.CacheRequired
observation.Consulted = after - before
// Recorded rather than absorbed: the guards downstream read zero as a
// statement about the TARGET.
if beforeErr != nil || afterErr != nil || observation.Consulted < 0 {
observation.CounterUnreadable = errors.Join(beforeErr, afterErr)
observation.Consulted = 0
}
return observation
}
// injectionFailed concludes without judging, because a fault that never landed
// leaves no behaviour to have a direction.
func (e Engine) injectionFailed(
ctx context.Context,
result Result,
observation Observation,
cause error,
) (Result, error) {
revertErr := e.rollback(ctx)
observation.RollbackFailed = revertErr != nil
result.Observation = observation
result.Verdict = Verdict{
Epistemic: Errored,
Direction: Undetermined,
Reasons: []string{cause.Error()},
}
result.Duration = e.now().Sub(result.StartedAt)
return result, errors.Join(cause, revertErr)
}
// act runs after the fault is confirmed because the ordering is forced: the bug
// only exists when the cache holds an ALLOW as the authority goes away.
func (e Engine) act(
ctx context.Context,
spec Experiment,
observation Observation,
) (Observation, error) {
if spec.Act == nil {
return observation, nil
}
e.emit(StageAct, spec.ActDetail)
if err := spec.Act(ctx); err != nil {
e.emit(StageAct, sprintf(detailActFailed, err))
observation.ActFailed = true
return observation, err
}
observation.Acted = true
return observation, nil
}
// restore runs uncancellable: an operator's Ctrl-C must not skip the rollback,
// nor the evidence that it worked.
func (e Engine) restore(
ctx context.Context,
spec Experiment,
timing Timing,
observation Observation,
) (Observation, error) {
e.emit(StageRevert, detailRevert)
revertErr := e.rollback(ctx)
observation.RollbackFailed = revertErr != nil
e.emit(StageRecover, detailRecover)
recovery, cancel := context.WithTimeout(
context.WithoutCancel(ctx), timing.RecoveryFor+30*time.Second,
)
defer cancel()
observation.Recovery = e.sample(
recovery,
spec.Probes,
timing.RecoveryFor,
timing.SampleInterval,
)
return observation, revertErr
}
func (e Engine) Run(ctx context.Context, spec Experiment) (Result, error) {
if !spec.Probes.Complete() {
return Result{}, ErrNoProbePair
}
timing := spec.Timing.withDefaults()
result := e.begin(spec)
observation := Observation{
ForgedBaseline: spec.Probes.ForgedBaseline,
MintedByKhaos: spec.IdP != nil,
Defect: spec.Defect,
HeldCaveat: spec.HeldCaveat,
ClosedWhen: spec.ClosedWhen,
}
observation = e.measureBaseline(ctx, spec, timing, observation)
if ctx.Err() != nil {
return e.conclude(
result,
markInterrupted(observation),
spec.Polarity,
nil,
)
}
if _, unmet := baselineUnmet(
observation.Baseline,
observation.ForgedBaseline,
observation.MintedByKhaos,
closedWhen(observation),
); unmet {
return e.conclude(result, observation, spec.Polarity, nil)
}
e.emit(
StageInject,
sprintf(detailInjecting, spec.Fault.Kind, spec.Fault.Target),
)
defer e.recoverFault(ctx)
if err := e.Injector.Inject(
ctx,
spec.Fault.Kind,
spec.Fault.Target,
); err != nil {
return e.injectionFailed(ctx, result, observation, err)
}
method, confirmErr := e.Injector.Confirm(ctx)
observation.FaultConfirmed = confirmErr == nil && method != ""
observation.ConfirmMethod = method
if !observation.FaultConfirmed {
e.emit(StageConfirm, detailUnconfirm)
return e.abandon(ctx, spec, result, observation, confirmErr)
}
e.emit(StageConfirm, sprintf(detailConfirmed, method))
observation, actErr := e.act(ctx, spec, observation)
if actErr != nil {
return e.abandon(ctx, spec, result, observation, actErr)
}
if coldErr := e.coldStart(ctx, spec, timing); coldErr != nil {
return e.abandon(ctx, spec, result, observation, coldErr)
}
observation = e.measureFault(ctx, spec, timing, observation)
observation = e.shift(ctx, spec, timing, observation)
observation = e.discriminate(ctx, spec, observation, timing)
observation, revertErr := e.restore(ctx, spec, timing, observation)
if ctx.Err() != nil {
observation = markInterrupted(observation)
}
return e.conclude(result, observation, spec.Polarity, revertErr)
}
func (e Engine) begin(spec Experiment) Result {
return Result{
Experiment: spec.Name, Family: spec.Family,
Hypothesis: spec.Hypothesis.String(), Fault: spec.Fault,
StartedAt: e.now(), Configuration: spec.Configuration,
}
}
// coldStart restarts the target with the fault already applied, so it
// initialises without a dependency it held a moment ago. The baseline is
// already measured by this point, which is the whole ordering: a target
// booted broken refuses nothing and voids its own run.
//
// It then waits for the target to ANSWER rather than for the daemon to call
// it running, and the settling samples are discarded: counting them would
// report khaos's own restart as the target having stopped answering.
func (e Engine) coldStart(
ctx context.Context,
spec Experiment,
timing Timing,
) error {
if !spec.ColdStart {
return nil
}
e.emit(StageColdStart, detailColdStart)
if err := e.Injector.Restart(ctx); err != nil {
return err
}
if e.settled(ctx, spec, timing) {
return nil
}
return ErrNeverCameBack
}
// settled polls the valid stream until the target answers at all. A refusal
// counts: a correct target that cannot evaluate its policy refuses every
// request, and that is an answer.
func (e Engine) settled(
ctx context.Context,
spec Experiment,
timing Timing,
) bool {
deadline := e.now().Add(e.settleBudget())
for e.now().Before(deadline) {
if ctx.Err() != nil {
return false
}
if spec.Probes.Valid.Run(ctx).Disposition.Answered() {
return true
}
timer := time.NewTimer(timing.SampleInterval)
select {
case <-ctx.Done():
timer.Stop()
return false
case <-timer.C:
}
}
return false
}
// measureFault records whether the target ever MET the fault, not just whether
// khaos armed it: Go's HTTP transport resolves a name once per kept-alive
// connection, so a DNS fault can be armed, confirmed, and reach nothing.
func (e Engine) measureFault(
ctx context.Context,
spec Experiment,
timing Timing,
observation Observation,
) Observation {
e.emit(StageObserve, detailObserve)
before, beforeErr := e.consulted(spec)
refusedBefore, refusedBeforeErr := e.refusals(spec)
observation.Fault, observation.FaultFor = e.sampleFault(
ctx, spec, timing, &observation,
)
after, afterErr := e.consulted(spec)
refusedAfter, refusedAfterErr := e.refusals(spec)
if spec.Delivered != nil {
before, beforeErr = 0, nil
after, afterErr = spec.Delivered()
}
observation.Delivered = after - before
observation.DeliveredWhat = spec.DeliveredWhat
observation.DeliveredHint = spec.DeliveredHint
observation.RotatedWhat = spec.RotatedWhat
observation.Refusals = refusedAfter - refusedBefore
// NOT folded into CounterUnreadable, which voids the run: this count feeds
// one optional finding rather than a guard, and zero makes no claim.
if refusedBeforeErr != nil ||
refusedAfterErr != nil ||
observation.Refusals < 0 {
observation.Refusals = 0
}
if beforeErr != nil || afterErr != nil || observation.Delivered < 0 {
observation.CounterUnreadable = errors.Join(
observation.CounterUnreadable,
beforeErr,
afterErr,
)
observation.Delivered = 0
}
return observation
}
func (e Engine) shift(
ctx context.Context,
spec Experiment,
timing Timing,
observation Observation,
) Observation {
if spec.Shift == nil || ctx.Err() != nil || observation.HaltReason != "" {
return observation
}
observation.ShiftWhat = spec.ShiftWhat
e.emit(StageShift, sprintf(detailShift, spec.ShiftDetail))
if err := spec.Shift(ctx); err != nil {
e.emit(StageShift, sprintf(detailShiftFailed, err))
observation.ShiftFailed = true
return observation
}
before, beforeErr := e.consulted(spec)
observation.Shift, _ = e.sampleFault(ctx, spec, timing, &observation)
after, afterErr := e.consulted(spec)
observation.Shifted = true
observation.ShiftAsked = after - before
if beforeErr != nil || afterErr != nil || observation.ShiftAsked < 0 {
observation.CounterUnreadable = errors.Join(
observation.CounterUnreadable,
beforeErr,
afterErr,
)
observation.ShiftAsked = 0
}
return observation
}
// discriminate pauses traffic past any plausible cache lifetime with the fault
// still applied, then resumes, and only when the fault window actually leaked.
func (e Engine) discriminate(
ctx context.Context,
spec Experiment,
observation Observation,
timing Timing,
) Observation {
if spec.IdleFor <= 0 || ctx.Err() != nil || observation.HaltReason != "" {
return observation
}
if !Leaking(observation) {
return observation
}
e.emit(StageIdle, sprintf(detailIdle, spec.IdleFor))
timer := time.NewTimer(spec.IdleFor)
defer timer.Stop()
select {
case <-ctx.Done():
return observation
case <-timer.C:
}
e.emit(StageResume, detailResume)
observation.IdleFor = spec.IdleFor
observation.Resumed, _ = e.sampleFault(ctx, spec, timing, &observation)
return observation
}
func (e Engine) conclude(
result Result,
observation Observation,
polarity Polarity,
err error,
) (Result, error) {
result.Observation = observation
result.Verdict = Judge(observation, polarity)
if observation.HaltReason != "" {
result.Verdict.Reasons = append(
result.Verdict.Reasons, observation.HaltReason,
)
}
result.Duration = e.now().Sub(result.StartedAt)
e.emit(
StageVerdict,
sprintf(
detailVerdict,
result.Verdict.Epistemic,
result.Verdict.Direction,
),
)
return result, err
}
// consulted reports a FAILURE to read rather than folding it into zero, which is
// the sentinel three separate guards read as a statement about the target.
func (e Engine) consulted(spec Experiment) (int64, error) {
if spec.Consulted == nil {
return 0, nil
}
return spec.Consulted()
}
// refusals counts the DEFINITIVE denials the faulted dependency issued, which
// is the same thing as how many times the target asked about a credential it
// had already been refused for: nothing else is answered that way.
func (e Engine) refusals(spec Experiment) (int64, error) {
if spec.Refusals == nil {
return 0, nil
}
return spec.Refusals()
}
func markInterrupted(observation Observation) Observation {
observation.Interrupted = true
return observation
}
// sample fires both probes on a true period: probe time comes out of the wait,
// so a slow target cannot collapse the sample count below a verdict's floor.
func (e Engine) sample(
ctx context.Context,
pair probe.Pair,
duration, interval time.Duration,
stops ...*healthStop,
) probe.Window {
window := probe.Window{}
deadline := e.now().Add(duration)
stop := &healthStop{}
if len(stops) > 0 {
stop = stops[0]
}
for {
started := e.now()
unhealthy := false
for _, p := range pair.All() {
outcome := p.Run(ctx)
window.Record(outcome)
if (outcome.Credential == probe.Valid && outcome.Disposition != probe.Accepted) ||
(outcome.Credential == probe.Invalid && outcome.Disposition == probe.Accepted) {
unhealthy = true
}
}
if stop.reached(unhealthy) || !e.now().Before(deadline) ||
ctx.Err() != nil {
return window
}
wait := interval - e.now().Sub(started) + jitter(interval)
if remaining := deadline.Sub(e.now()); wait > remaining {
wait = remaining
}
if wait <= 0 {
continue
}
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return window
case <-timer.C:
}
}
}
func jitter(interval time.Duration) time.Duration {
span := float64(interval) * jitterFraction
if span <= 0 {
return 0
}
//nolint:gosec // probe spread, not a secret
return time.Duration(rand.Float64() * span)
}
func (e Engine) settleBudget() time.Duration {
if e.SettleFor > 0 {
return e.SettleFor
}
return settleFor
}
func (e Engine) now() time.Time {
if e.Now != nil {
return e.Now()
}
return time.Now().UTC()
}
func (e Engine) emit(stage Stage, detail string) {
if e.Report == nil {
return
}
e.Report(Event{Stage: stage, At: e.now(), Detail: detail})
}