The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")"
IMPLS="${IMPLS:-honeycomb traceway-oxc-mem traceway-oxc-disk traceway-goja-mem traceway-goja-disk}"
SCENARIOS="${SCENARIOS:-hot churn oom}"
CHURN_ENTRIES="${CHURN_ENTRIES:-512}"
PAD_KB="${PAD_KB:-256}"
OOM_ENTRIES="${OOM_ENTRIES:-4096}"
OOM_PAD_KB="${OOM_PAD_KB:-1024}"
OOM_MAP_PAD_KB="${OOM_MAP_PAD_KB:-1024}"
OOM_MAPPINGS_PAD_KB="${OOM_MAPPINGS_PAD_KB:-1024}"
OOM_CONNECTIONS="${OOM_CONNECTIONS:-32}"
OOM_DURATION="${OOM_DURATION:-30m}"
CONNECTIONS="${CONNECTIONS:-4,8,16,32,64,128}"
STEP_DURATION="${STEP_DURATION:-30s}"
SPANS_PER_REQUEST="${SPANS_PER_REQUEST:-20}"
SKIP_BUILD="${SKIP_BUILD:-}"
RESULTS="${RESULTS:-./results}"
OCB_154=./bin/ocb-0.154.0
OCB_143=./bin/ocb-0.143.0
build_all() {
mkdir -p bin
[ -x "$OCB_154" ] || { GOBIN="$PWD/bin" go install go.opentelemetry.io/collector/cmd/[email protected] && mv bin/builder "$OCB_154"; }
[ -x "$OCB_143" ] || { GOBIN="$PWD/bin" go install go.opentelemetry.io/collector/cmd/[email protected] && mv bin/builder "$OCB_143"; }
if echo "$IMPLS" | grep -q traceway; then
bash ../../scripts/build-oxc-shim.sh
"$OCB_154" --config manifest-traceway.yaml --skip-compilation
(cd build-traceway && CGO_ENABLED=1 go build -tags oxc -o otelcol-bench-traceway .)
fi
if echo "$IMPLS" | grep -q honeycomb; then
CGO_ENABLED=1 "$OCB_143" --config manifest-honeycomb.yaml
fi
(cd drain && cargo build --release)
(cd loadgen && go mod tidy >/dev/null 2>&1; go build -o loadgen .)
(cd corpusgen && go build -o corpusgen .)
if echo "$IMPLS" | grep -qE 'oxc|goja|(^| )honeycomb( |$)' && [ ! -f ../../testing/symbolication/node-app/dist/app.mjs ]; then
(cd ../../testing/symbolication/node-app && npm install && npm run build)
fi
}
scenario_params() {
local oom_cache="$OOM_ENTRIES"
case "$2" in ios|honeycomb-ios|android) oom_cache="${OOM_CACHE:-128}" ;; esac
case "$1" in
hot) echo "1 $PAD_KB 0:0 128 $CONNECTIONS $STEP_DURATION" ;;
churn) echo "$CHURN_ENTRIES $PAD_KB 0:0 128 $CONNECTIONS $STEP_DURATION" ;;
oom) echo "$OOM_ENTRIES $OOM_PAD_KB $OOM_MAP_PAD_KB:$OOM_MAPPINGS_PAD_KB $oom_cache $OOM_CONNECTIONS $OOM_DURATION" ;;
esac
}
gen_corpus() {
local lang="$1" scenario="$2" entries="$3" pad="$4" mappad="$5"
local dir="./corpus-$lang-$scenario"
if [ ! -f "$dir/corpus.json" ]; then
if [ "$lang" = dart ]; then
./corpusgen/corpusgen --language dart --entries "$entries" --out "$dir" >&2
elif [ "$lang" = ios ]; then
./corpusgen/corpusgen --language ios --entries "$entries" --dsym seeds/ios/app.dsym --trace seeds/ios/trace.txt --out "$dir" >&2
elif [ "$lang" = honeycomb-ios ]; then
./corpusgen/corpusgen --language honeycomb-ios --entries "$entries" --dsym seeds/ios/app.dsym --trace seeds/ios/trace.txt --binary sample --out "$dir" >&2
elif [ "$lang" = android ]; then
./corpusgen/corpusgen --language android --entries "$entries" --mapping seeds/android/mapping.txt --trace seeds/android/obfuscated.txt --map-pad-kb "${mappad%%:*}" --out "$dir" >&2
else
./corpusgen/corpusgen --entries "$entries" --pad-kb "$pad" --map-pad-kb "${mappad%%:*}" --mappings-pad-kb "${mappad##*:}" --out "$dir" >&2
fi
fi
echo "$dir"
}
impl_env() {
case "$1" in
honeycomb) echo "BIN=./build-honeycomb/otelcol-bench-honeycomb CFG=config-honeycomb.yaml PARSER= DISK= LANG=js" ;;
traceway-oxc-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=oxc DISK= LANG=js" ;;
traceway-oxc-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=oxc DISK=1 LANG=js" ;;
traceway-goja-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=goja DISK= LANG=js" ;;
traceway-goja-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER=goja DISK=1 LANG=js" ;;
traceway-dart-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK= LANG=dart" ;;
traceway-dart-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK=1 LANG=dart" ;;
traceway-ios-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK= LANG=ios" ;;
traceway-ios-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK=1 LANG=ios" ;;
honeycomb-ios) echo "BIN=./build-honeycomb/otelcol-bench-honeycomb CFG=config-honeycomb-ios.yaml PARSER= DISK= LANG=honeycomb-ios SIGNAL=logs" ;;
traceway-android-mem) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK= LANG=android SIGNAL=logs" ;;
traceway-android-disk) echo "BIN=./build-traceway/otelcol-bench-traceway CFG=config-traceway.yaml PARSER= DISK=1 LANG=android SIGNAL=logs" ;;
honeycomb-android) echo "BIN=./build-honeycomb/otelcol-bench-honeycomb CFG=config-honeycomb-android.yaml PARSER= DISK= LANG=android SIGNAL=logs" ;;
*) echo "unknown impl $1" >&2; return 1 ;;
esac
}
drain_markers() {
case "$1" in
dart) echo "OK_MARKER=crash.dart FAIL_MARKER=_kDartIsolateSnapshotInstructions" ;;
ios|honeycomb-ios) echo "OK_MARKER=sample.c FAIL_MARKER=sample+0x" ;;
android) echo "OK_MARKER=.kt: FAIL_MARKER=SourceFile:" ;;
*) echo "OK_MARKER=../src/inventory.js FAIL_MARKER=.mjs:1:" ;;
esac
}
run_one() {
local impl="$1" scenario="$2"
local BIN CFG PARSER DISK LANG SIGNAL=traces
eval "$(impl_env "$impl")"
read -r entries pad mappad cachesize conns dur <<< "$(scenario_params "$scenario" "$LANG")"
local store
store=$(gen_corpus "$LANG" "$scenario" "$entries" "$pad" "$mappad")
local OK_MARKER FAIL_MARKER
eval "$(drain_markers "$LANG")"
local tag="$impl-$scenario"
local outdir="$RESULTS/$tag"
mkdir -p "$outdir"
pkill -f 'target/release/drain' 2>/dev/null || true
pkill -f otelcol-bench 2>/dev/null || true
for i in $(seq 1 10); do
lsof -iTCP:9319 -sTCP:LISTEN >/dev/null 2>&1 || lsof -iTCP:4318 -sTCP:LISTEN >/dev/null 2>&1 || break
sleep 1
done
DRAIN_ADDR=127.0.0.1:9319 OK_MARKER="$OK_MARKER" FAIL_MARKER="$FAIL_MARKER" ./drain/target/release/drain &
local drain_pid=$!
sleep 1
curl -sf -X POST http://127.0.0.1:9319/reset > /dev/null
local cache_dir=""
[ -n "$DISK" ] && cache_dir=$(mktemp -d)
STORE_PATH="$store" CACHE_DIR="$cache_dir" CACHE_SIZE="$cachesize" SYMB_PARSER="${PARSER:-goja}" \
DRAIN_ENDPOINT=http://127.0.0.1:9319 \
"$BIN" --config "$CFG" > "$outdir/collector.log" 2>&1 &
local col_pid=$!
for i in $(seq 1 30); do
curl -s -o /dev/null http://127.0.0.1:4318/ 2>/dev/null && break
kill -0 "$col_pid" 2>/dev/null || { echo "collector died at startup, see $outdir/collector.log" >&2; kill "$drain_pid"; return 1; }
sleep 1
done
bash ./rss-sampler.sh "$col_pid" "$outdir/rss.csv" &
local rss_pid=$!
local t0
t0=$(date +%s)
./loadgen/loadgen --target "http://127.0.0.1:4318/v1/$SIGNAL" --corpus "$store/corpus.json" \
--connections "$conns" --step-duration "$dur" \
--spans-per-request "$SPANS_PER_REQUEST" --out "$outdir/loadgen.json" || true
if ! kill -0 "$col_pid" 2>/dev/null; then
echo "$(( $(tail -1 "$outdir/rss.csv" | cut -d, -f1) - t0 ))" > "$outdir/died"
fi
curl -sf http://127.0.0.1:9319/stats > "$outdir/drain.json" || echo '{}' > "$outdir/drain.json"
kill "$col_pid" "$drain_pid" 2>/dev/null || true
wait "$rss_pid" 2>/dev/null || true
[ -n "$cache_dir" ] && rm -rf "$cache_dir"
echo "== $tag =="
cat "$outdir/drain.json"; echo
}
[ -n "$SKIP_BUILD" ] || build_all
for impl in $IMPLS; do
for scenario in $SCENARIOS; do
run_one "$impl" "$scenario"
done
done
bash ./summarize.sh "$RESULTS"