Git designed for AI agents.
#!/bin/sh
# run-benchmarks-incluster: drive a benchmark run as a Kubernetes Job inside
# an existing GitLab/Gitaly cluster.
#
# Required environment variables:
# EXPERIMENT - name of a subdir under experiments/
# WORKLOAD_IDENTITY_GSA - email of the GCP service account bound to
# the in-cluster ServiceAccount via Workload
# Identity. Must have objectAdmin on the bucket.
# Optional on repeat runs: resolved from the
# gitaly-benchmark ServiceAccount if unset.
#
# Optional but commonly set:
# BENCHMARK_RESULTS_BUCKET - GCS bucket name (without gs:// prefix) that
# the uploader sidecar writes to.
# Default: gitaly-benchmark-results
# BENCHMARK_IMAGE - fully-qualified image ref for the benchmark
# client. Defaults to the Artifact Registry repo
# used by `make push-benchmark-image`, tagged
# with the current git short SHA:
# us-east1-docker.pkg.dev/gitaly-benchmark-0150d6cf/
# gitaly-benchmarking/benchmark-client:<git-sha>
# Override to pin to a specific tag or registry.
#
# Optional environment variables:
# KUBE_NAMESPACE - namespace for both the Job and Gitaly.
# Default: gitlab. The Job is always co-located
# with Gitaly so it can read gitlab-gitaly-secret
# and inspect the Gitaly pod without cross-
# namespace RBAC.
# GITALY_SERVICE - in-cluster service name for Gitaly.
# Default: gitlab-gitaly
# GITALY_PORT - Gitaly gRPC port. Default: 8075
# GITALY_POD_SELECTOR - label selector for the Gitaly pod.
# Default: app=gitaly
# GITALY_CONTAINER - container name inside the pod. Default: gitaly
# GITALY_METRICS_PORT - Gitaly Prometheus / pprof port. Default: 9236
# PROFILE - "true"/"false". Default: true. When true,
# the benchmark container scrapes Go pprof
# from each Gitaly pod's /debug/pprof
# endpoint over HTTP (no in-pod exec, no
# SYS_PTRACE required).
# PROFILE_DURATION - seconds. Default: 120
# WORKLOAD_DURATION - duration string, default 60s
# WORKLOAD_WAIT_DURATION - seconds, defaults to WORKLOAD_DURATION + 60.
# Bounds the workload phase (k6 + pprof + upload).
# PUSH_TRANSPORTS - comma list of push-section transports,
# "ssh" and/or "http". Default: ssh,http
# PUSH_ONLY - "1" runs only the push sections, at 1/10
# load. Default: 0
# GITLAB_HTTP_URL - workhorse base URL for repo clones and http
# pushes. Default: the chart's webservice.
# GITLAB_SSH_URL - gitlab-shell base URL for the ssh
# transport. Default: the chart's Service.
# BENCH_GROUP - group of the bench projects; must match
# provision-repos. Default: bench
# GITALY_STORAGE_NAME - storage the target Gitaly serves. Default:
# default. Anything else selects the
# control_* disk paths in config.yml.
# STARTUP_GRACE - seconds budgeted in activeDeadlineSeconds
# for pod startup (node provisioning + image
# pulls). Default: 600.
# CLEAR_GITALY_CACHE - "true"/"false". Default: false. When true
# the wrapper (a) rollout-restarts the
# Gitaly StatefulSet to drop in-process
# caches (catfile, pack, ref) and (b) drops
# the kernel page cache on the Gitaly node
# via a privileged debug pod. Step (b) needs
# `kubectl debug node` permission; failures
# are non-fatal. Default is off so back-to-
# back runs are fast; set to true when you
# need cold-cache numbers comparable to the
# Ansible VM flow's `clear_page_cache`.
# CLEAR_PAGE_CACHE - "true"/"false". Default: false. Set to
# true to drop the node's kernel page cache
# (requires CLEAR_GITALY_CACHE=true too).
# CACHE_CLEAR_DEBUG_IMAGE - image used for the page-cache debug pod.
# Default: busybox.
#
# Grafana run-window annotation (best-effort; see incluster/grafana):
# GRAFANA_ANNOTATE=false - skip marking this run's window on the dashboard.
# BENCHMARK_ANNOTATION_LABEL - annotation-text prefix (the A/B side); set by
# run-benchmarks-compare, empty standalone.
# GRAFANA_URL / GRAFANA_NAMESPACE / GRAFANA_SERVICE - Grafana endpoint; unset
# GRAFANA_URL port-forwards to the Service.
#
# Dedicated node pool:
# The benchmark runs on a dedicated, tainted+labeled node pool (provisioned
# in the cluster's Terraform, gl-gitaly/tf-gke-gitlab-bootstrap) so it is
# isolated from the Gitaly/GitLab workloads it measures and is never
# preempted. The pool autoscales from zero, so it costs nothing until a run
# lands on it. The nodeSelector/tolerations that pin the Job to this pool are
# baked into incluster/job.yaml.j2; the normal invocation needs no scheduling
# env vars:
# EXPERIMENT=incluster-master _support/benchmarking/run-benchmarks-incluster
#
# Exit codes:
# 0 on success, non-zero if the Job fails or any kubectl call errors.
set -eu
die() {
echo "error: $*" >&2
exit 1
}
[ -n "${EXPERIMENT:-}" ] || die "EXPERIMENT must be set"
BENCHMARK_RESULTS_BUCKET="${BENCHMARK_RESULTS_BUCKET:-gitaly-benchmark-results}"
# WORKLOAD_IDENTITY_GSA is validated later, after KUBECTL is set up, so we can
# fall back to the existing ServiceAccount's annotation.
# BENCHMARK_IMAGE precedence: env override > last push (.last-pushed-image) > :latest.
_LAST_IMAGE_FILE="$(cd "$(dirname "$0")" && pwd)/.last-pushed-image"
if [ -z "${BENCHMARK_IMAGE:-}" ] && [ -s "${_LAST_IMAGE_FILE}" ]; then
BENCHMARK_IMAGE="$(head -n1 "${_LAST_IMAGE_FILE}")"
echo "[wrapper] using last-pushed image: ${BENCHMARK_IMAGE}"
fi
if [ -z "${BENCHMARK_IMAGE:-}" ]; then
BENCHMARK_IMAGE="us-east1-docker.pkg.dev/gitaly-benchmark-0150d6cf/gitaly-benchmarking/benchmark-client:latest"
echo "[wrapper] no recorded image; defaulting to :latest (run 'make push-benchmark-image' or set BENCHMARK_IMAGE to pin)"
fi
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
EXP_DIR="${SCRIPT_DIR}/experiments/${EXPERIMENT}"
[ -d "${EXP_DIR}" ] || die "experiment directory not found: ${EXP_DIR}"
[ -f "${EXP_DIR}/k6-benchmark.js" ] || die "${EXP_DIR}/k6-benchmark.js missing"
[ -f "${EXP_DIR}/config.yml" ] || die "${EXP_DIR}/config.yml missing"
# Auto-activate the benchmarking virtualenv (see README "Setup your
# environment"); it carries the PyYAML used to render repositories.json.
if [ -z "${VIRTUAL_ENV:-}" ] && [ -f "${SCRIPT_DIR}/env/bin/activate" ]; then
# shellcheck disable=SC1091
. "${SCRIPT_DIR}/env/bin/activate"
fi
# Probe PyYAML upfront to fail with an actionable message instead of a
# Python traceback mid-pipeline.
if ! python3 -c "import yaml" >/dev/null 2>&1; then
die "python3 cannot import yaml. Either:
(a) create the benchmarking virtualenv: cd ${SCRIPT_DIR} && python3 -m venv env && env/bin/pip install -r requirements.txt
(b) install PyYAML system-wide: pip3 install pyyaml"
fi
# The Job and Gitaly share a single namespace (default "gitlab"): no
# cross-namespace RBAC, and the wrapper reads gitlab-gitaly-secret directly.
KUBE_NAMESPACE="${KUBE_NAMESPACE:-gitlab}"
GITALY_SERVICE="${GITALY_SERVICE:-gitlab-gitaly}"
GITALY_PORT="${GITALY_PORT:-8075}"
GITALY_POD_SELECTOR="${GITALY_POD_SELECTOR:-app=gitaly}"
GITALY_CONTAINER="${GITALY_CONTAINER:-gitaly}"
GITALY_METRICS_PORT="${GITALY_METRICS_PORT:-9236}"
PROFILE="${PROFILE:-true}"
PROFILE_DURATION="${PROFILE_DURATION:-120}"
WORKLOAD_DURATION="${WORKLOAD_DURATION:-60s}"
# WORKLOAD_DURATION + 60s margin, or a static 300s when WORKLOAD_DURATION
# isn't a bare-second value like "120s". Bounds the workload phase (k6 +
# concurrent pprof scrape + result upload), not pod startup.
if [ -z "${WORKLOAD_WAIT_DURATION:-}" ]; then
case "${WORKLOAD_DURATION}" in
*s) WORKLOAD_WAIT_DURATION="$((${WORKLOAD_DURATION%s} + 60))" ;;
*) WORKLOAD_WAIT_DURATION=300 ;;
esac
fi
# Push sections: transports to run, workhorse base URL (clones + http
# pushes), and the bench project group (must match provision-repos).
PUSH_TRANSPORTS="${PUSH_TRANSPORTS:-ssh,http}"
PUSH_ONLY="${PUSH_ONLY:-0}"
GITLAB_HTTP_URL="${GITLAB_HTTP_URL:-http://gitlab-webservice-default.${KUBE_NAMESPACE}.svc:8181}"
GITLAB_SSH_URL="${GITLAB_SSH_URL:-ssh://git@gitlab-gitlab-shell.${KUBE_NAMESPACE}.svc:22}"
BENCH_GROUP="${BENCH_GROUP:-bench}"
GITALY_STORAGE_NAME="${GITALY_STORAGE_NAME:-default}"
# Covers node provisioning, image pulling, and pod startup time.
STARTUP_GRACE="${STARTUP_GRACE:-600}"
# The Job's activeDeadlineSeconds budgets every sequential phase: pod
# startup (STARTUP_GRACE), k6 setup, the mixed workload, the write-path
# sections (phase counts mirror k6-benchmark.js), and branch-cleanup
# teardown. Doubled for headroom: a deadline kill loses the whole run.
_n_transports=0
for _t in $(printf '%s' "${PUSH_TRANSPORTS}" | tr ',' ' '); do
[ "${_t}" = "none" ] || _n_transports=$((_n_transports + 1))
done
_sections=$((4 + 4 * _n_transports))
# 35 = sectionDuration (30s) + GAP_SECONDS (5s) in k6-benchmark.js.
ACTIVE_DEADLINE_SECONDS="$((2 * (STARTUP_GRACE + 300 + WORKLOAD_WAIT_DURATION + _sections * 35 + 600)))"
KUBECTL="kubectl"
[ -n "${KUBECONFIG:-}" ] && KUBECTL="${KUBECTL} --kubeconfig=${KUBECONFIG}"
[ -n "${KUBE_CONTEXT:-}" ] && KUBECTL="${KUBECTL} --context=${KUBE_CONTEXT}"
# Resolve WORKLOAD_IDENTITY_GSA: env var, else read it back from the
# gitaly-benchmark ServiceAccount a prior run created (annotation
# iam.gke.io/gcp-service-account; see incluster/serviceaccount.yaml.j2).
if [ -z "${WORKLOAD_IDENTITY_GSA:-}" ]; then
WORKLOAD_IDENTITY_GSA="$(${KUBECTL} -n "${KUBE_NAMESPACE}" \
get serviceaccount gitaly-benchmark \
-o "jsonpath={.metadata.annotations.iam\.gke\.io/gcp-service-account}" \
2>/dev/null || true)"
[ -n "${WORKLOAD_IDENTITY_GSA}" ] &&
echo "[wrapper] resolved WORKLOAD_IDENTITY_GSA from ServiceAccount: ${WORKLOAD_IDENTITY_GSA}"
fi
if [ -z "${WORKLOAD_IDENTITY_GSA:-}" ]; then
die "WORKLOAD_IDENTITY_GSA unset and not found on the gitaly-benchmark
ServiceAccount in ns ${KUBE_NAMESPACE} (absent on first run). Either:
(a) set it: export WORKLOAD_IDENTITY_GSA=<sa>@<project>.iam.gserviceaccount.com
(b) find it: kubectl -n ${KUBE_NAMESPACE} get serviceaccount gitaly-benchmark \\
-o jsonpath='{.metadata.annotations.iam\.gke\.io/gcp-service-account}'"
fi
# The Gitaly shared auth token lives in the chart's gitlab-gitaly-secret;
# k6 signs every RPC with a v2 HMAC bearer derived from it. The Job reads
# it via secretKeyRef, so the Job and secret must share KUBE_NAMESPACE.
# Verify it now to fail fast instead of CrashLoopBackOff at Job start.
GITALY_AUTH_SECRET_NAME="${GITALY_AUTH_SECRET_NAME:-gitlab-gitaly-secret}"
GITALY_AUTH_SECRET_KEY="${GITALY_AUTH_SECRET_KEY:-token}"
if ! ${KUBECTL} -n "${KUBE_NAMESPACE}" \
get secret "${GITALY_AUTH_SECRET_NAME}" \
-o "jsonpath={.data.${GITALY_AUTH_SECRET_KEY}}" >/dev/null 2>&1; then
die "could not find ${GITALY_AUTH_SECRET_NAME}/${GITALY_AUTH_SECRET_KEY} in ns ${KUBE_NAMESPACE}.
The Gitaly client in this cluster needs a shared auth token. Either:
(a) confirm the secret exists: kubectl -n ${KUBE_NAMESPACE} get secret ${GITALY_AUTH_SECRET_NAME}
(b) override the lookup: GITALY_AUTH_SECRET_NAME=... GITALY_AUTH_SECRET_KEY=..."
fi
RUN_NAME="benchmark-$(date +%s)"
JOB_NAME="gitaly-benchmark-${EXPERIMENT}-$(date +%s)"
CM_NAME="${JOB_NAME}-config"
SA_NAME="gitaly-benchmark"
GITALY_ADDRESS="${GITALY_SERVICE}.${KUBE_NAMESPACE}.svc.cluster.local:${GITALY_PORT}"
echo "[wrapper] experiment=${EXPERIMENT} run=${RUN_NAME}"
echo "[wrapper] target=tcp://${GITALY_SERVICE}.${KUBE_NAMESPACE}.svc:${GITALY_PORT} (k6 GITALY_ADDRESS=${GITALY_ADDRESS})"
echo "[wrapper] image=${BENCHMARK_IMAGE}"
echo "[wrapper] bucket=gs://${BENCHMARK_RESULTS_BUCKET}/${RUN_NAME}"
# Record the deployed Gitaly image in the cluster, as this can be different from
# the changes introduced in the MR.
GITALY_VERSION="$(${KUBECTL} -n "${KUBE_NAMESPACE}" get pods -l "${GITALY_POD_SELECTOR}" \
-o jsonpath="{.items[0].spec.containers[?(@.name==\"${GITALY_CONTAINER}\")].image}" \
2>/dev/null || true)"
echo "[wrapper] gitaly under test=${GITALY_VERSION:-unknown}"
render() {
sed \
-e "s|\${EXPERIMENT}|${EXPERIMENT}|g" \
-e "s|\${RUN_NAME}|${RUN_NAME}|g" \
-e "s|\${JOB_NAME}|${JOB_NAME}|g" \
-e "s|\${CM_NAME}|${CM_NAME}|g" \
-e "s|\${SA_NAME}|${SA_NAME}|g" \
-e "s|\${KUBE_NAMESPACE}|${KUBE_NAMESPACE}|g" \
-e "s|\${BENCHMARK_IMAGE}|${BENCHMARK_IMAGE}|g" \
-e "s|\${BENCHMARK_RESULTS_BUCKET}|${BENCHMARK_RESULTS_BUCKET}|g" \
-e "s|\${WORKLOAD_IDENTITY_GSA}|${WORKLOAD_IDENTITY_GSA}|g" \
-e "s|\${GITALY_ADDRESS}|${GITALY_ADDRESS}|g" \
-e "s|\${GITALY_SERVICE}|${GITALY_SERVICE}|g" \
-e "s|\${GITALY_NAMESPACE}|${KUBE_NAMESPACE}|g" \
-e "s|\${GITALY_POD_SELECTOR}|${GITALY_POD_SELECTOR}|g" \
-e "s|\${GITALY_CONTAINER}|${GITALY_CONTAINER}|g" \
-e "s|\${GITALY_METRICS_PORT}|${GITALY_METRICS_PORT}|g" \
-e "s|\${WORKLOAD_DURATION}|${WORKLOAD_DURATION}|g" \
-e "s|\${ACTIVE_DEADLINE_SECONDS}|${ACTIVE_DEADLINE_SECONDS}|g" \
-e "s|\${PROFILE}|${PROFILE}|g" \
-e "s|\${PROFILE_DURATION}|${PROFILE_DURATION}|g" \
-e "s|\${K6_SCRIPT_B64}|${K6_SCRIPT_B64}|g" \
-e "s|\${REPOS_JSON_B64}|${REPOS_JSON_B64}|g" \
-e "s|\${PLOT_PY_B64}|${PLOT_PY_B64}|g" \
-e "s|\${GITALY_AUTH_SECRET_NAME}|${GITALY_AUTH_SECRET_NAME}|g" \
-e "s|\${GITALY_AUTH_SECRET_KEY}|${GITALY_AUTH_SECRET_KEY}|g" \
-e "s|\${PUSH_TRANSPORTS}|${PUSH_TRANSPORTS}|g" \
-e "s|\${PUSH_ONLY}|${PUSH_ONLY}|g" \
-e "s|\${GITLAB_HTTP_URL}|${GITLAB_HTTP_URL}|g" \
-e "s|\${GITLAB_SSH_URL}|${GITLAB_SSH_URL}|g" \
-e "s|\${BENCH_GROUP}|${BENCH_GROUP}|g" \
-e "s|\${GITALY_STORAGE_NAME}|${GITALY_STORAGE_NAME}|g" \
"$1"
}
# Render repositories.json from the experiment's config.yml. A non-default
# storage swaps in the control_* disk paths/gl_repositories provision-repos
# wrote, so k6 and the push helper address the repos the target serves.
REPOS_JSON_B64="$(python3 -c "
import json, sys, yaml, base64
repos = yaml.safe_load(open('${EXP_DIR}/config.yml'))['repositories']
if '${GITALY_STORAGE_NAME}' != 'default':
for r in repos:
if not r.get('control_disk_path'):
sys.exit(f\"repo {r['name']!r} has no control_disk_path in \"
'config.yml; run incluster/provision-repos --write-config '
'(the control storage must be registered in Rails)')
r['disk_path'] = r['control_disk_path']
r['gl_repository'] = r.get('control_gl_repository', '')
print(base64.b64encode(json.dumps(repos).encode()).decode())
")"
K6_SCRIPT_B64="$(base64 <"${EXP_DIR}/k6-benchmark.js" | tr -d '\n')"
if [ -f "${EXP_DIR}/plot.py" ]; then
PLOT_PY_B64="$(base64 <"${EXP_DIR}/plot.py" | tr -d '\n')"
else
PLOT_PY_B64=""
fi
# The namespace already exists (the chart creates it); deliberately no
# `kubectl create namespace` to avoid fighting the chart's ownership.
render "${SCRIPT_DIR}/incluster/serviceaccount.yaml.j2" | ${KUBECTL} apply -f -
render "${SCRIPT_DIR}/incluster/configmap.yaml.j2" | ${KUBECTL} apply -f -
# Cold-cache prep, modelled on the VM flow's `clear_page_cache` Ansible
# task. Two layers:
#
# 1. Rollout-restart the Gitaly StatefulSet, dropping its in-process
# catfile, pack, and ref caches.
# 2. Drop the kernel page cache on the node hosting Gitaly - the
# analogue of `sync && echo 3 > /proc/sys/vm/drop_caches`. Requires
# `kubectl debug node`; non-fatal, a denial only weakens the
# cold-cache guarantee.
#
# Both default off so back-to-back iteration stays fast.
CLEAR_GITALY_CACHE="${CLEAR_GITALY_CACHE:-false}"
CLEAR_PAGE_CACHE="${CLEAR_PAGE_CACHE:-false}"
CACHE_CLEAR_DEBUG_IMAGE="${CACHE_CLEAR_DEBUG_IMAGE:-busybox}"
if [ "${CLEAR_GITALY_CACHE}" = "true" ]; then
echo "[wrapper] restarting Gitaly to drop in-process caches"
${KUBECTL} -n "${KUBE_NAMESPACE}" rollout restart \
statefulset/"${GITALY_SERVICE}"
${KUBECTL} -n "${KUBE_NAMESPACE}" rollout status \
statefulset/"${GITALY_SERVICE}" --timeout=5m
if [ "${CLEAR_PAGE_CACHE}" = "true" ]; then
# Resolve the node hosting the (now-restarted) Gitaly pod.
gitaly_pod="$(${KUBECTL} -n "${KUBE_NAMESPACE}" \
get pods -l "${GITALY_POD_SELECTOR}" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [ -n "${gitaly_pod}" ]; then
gitaly_node="$(${KUBECTL} -n "${KUBE_NAMESPACE}" \
get pod "${gitaly_pod}" \
-o jsonpath='{.spec.nodeName}' 2>/dev/null || true)"
fi
if [ -n "${gitaly_node:-}" ]; then
echo "[wrapper] dropping kernel page cache on node ${gitaly_node}"
# `kubectl debug node` mounts the node root at /host; chroot so
# the write hits the host's procfs, not the debug container's
# masked /proc.
${KUBECTL} debug "node/${gitaly_node}" \
--image="${CACHE_CLEAR_DEBUG_IMAGE}" \
--profile=sysadmin \
-- chroot /host /bin/sh -c \
'sync && echo 3 > /proc/sys/vm/drop_caches' ||
echo "[wrapper] page-cache drop failed; continuing"
else
echo "[wrapper] could not resolve Gitaly node for page-cache drop; skipping"
fi
fi
fi
render "${SCRIPT_DIR}/incluster/job.yaml.j2" | ${KUBECTL} apply -f -
echo "[wrapper] Job ${JOB_NAME} applied; streaming benchmark container logs"
${KUBECTL} -n "${KUBE_NAMESPACE}" wait --for=condition=Ready pod \
-l "run-name=${RUN_NAME}" --timeout=120s || true
# Grafana run-window start: pod is Ready (k6 about to run), so node/image-pull
# time is excluded from the marked region.
RUN_START_EPOCH="$(date +%s)"
${KUBECTL} -n "${KUBE_NAMESPACE}" logs -f \
job/"${JOB_NAME}" -c benchmark --tail=-1 || true
# Stream the uploader sidecar too. It only acts once the benchmark container
# finishes (it waits for /tmp/run/.done), and on failure the pod is GC'd too
# fast for post-hoc capture, so this is the only reliable way to see an
# upload error (auth, permissions, missing results, ...).
echo "[wrapper] streaming uploader sidecar logs"
${KUBECTL} -n "${KUBE_NAMESPACE}" logs -f \
job/"${JOB_NAME}" -c uploader --tail=-1 || true
echo "[wrapper] waiting for Job to reach a terminal state"
# Poll for Complete OR Failed: `kubectl wait --for=condition=complete` would
# ignore failure and burn the whole timeout while the failed pod (and its
# logs) gets garbage-collected. Breaking out on failure lets the diagnostics
# below capture the pod state and logs in time.
_deadline=$(($(date +%s) + ACTIVE_DEADLINE_SECONDS + 300))
job_status=""
while [ "$(date +%s)" -lt "${_deadline}" ]; do
if [ "$(${KUBECTL} -n "${KUBE_NAMESPACE}" get job "${JOB_NAME}" \
-o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null)" = "True" ]; then
job_status=success
break
fi
if [ "$(${KUBECTL} -n "${KUBE_NAMESPACE}" get job "${JOB_NAME}" \
-o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null)" = "True" ]; then
job_status=failed
break
fi
sleep 5
done
[ -n "${job_status}" ] || {
echo "[wrapper] timed out waiting for terminal state" >&2
job_status=failed
}
echo "[wrapper] Job status: ${job_status}"
# Add a little 30s buffer to capture the tail end of the run
GRAFANA_WINDOW_END_PAD_SECONDS="${GRAFANA_WINDOW_END_PAD_SECONDS:-30}"
RUN_END_EPOCH="$(( $(date +%s) + GRAFANA_WINDOW_END_PAD_SECONDS ))"
# On failure, surface why before the pod is garbage-collected: both
# containers' logs (a Job that fails while the benchmark container exited 0
# points at the uploader) plus pod/Job state and events. Reading the output:
# exit code 137 = SIGKILL (OOM), DeadlineExceeded = activeDeadlineSeconds,
# Evicted = node disk/memory pressure. All read-only, best-effort.
if [ "${job_status}" != "success" ]; then
echo "[wrapper] --- failure diagnostics ---" >&2
${KUBECTL} -n "${KUBE_NAMESPACE}" get pods -l "run-name=${RUN_NAME}" -o wide >&2 || true
${KUBECTL} -n "${KUBE_NAMESPACE}" get pods -l "run-name=${RUN_NAME}" \
-o jsonpath='{range .items[*]}pod {.metadata.name} phase={.status.phase} reason={.status.reason}{"\n"}{range .status.containerStatuses[*]} container {.name}: restarts={.restartCount} state={.state} lastTerminated.reason={.lastState.terminated.reason} exitCode={.lastState.terminated.exitCode}{"\n"}{end}{end}' >&2 || true
echo >&2
echo "[wrapper] Job conditions:" >&2
${KUBECTL} -n "${KUBE_NAMESPACE}" get job "${JOB_NAME}" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} reason={.reason} message={.message}{"\n"}{end}' >&2 || true
echo "[wrapper] uploader container logs:" >&2
${KUBECTL} -n "${KUBE_NAMESPACE}" logs -l "run-name=${RUN_NAME}" -c uploader --prefix --tail=50 >&2 2>&1 || true
echo "[wrapper] benchmark container logs (tail):" >&2
${KUBECTL} -n "${KUBE_NAMESPACE}" logs -l "run-name=${RUN_NAME}" -c benchmark --prefix --tail=20 >&2 2>&1 || true
echo "[wrapper] recent events:" >&2
${KUBECTL} -n "${KUBE_NAMESPACE}" describe job "${JOB_NAME}" 2>/dev/null |
sed -n '/Events:/,$p' >&2 || true
echo "[wrapper] --- end diagnostics ---" >&2
fi
mkdir -p "${EXP_DIR}/results/${RUN_NAME}"
[ -n "${GITALY_VERSION}" ] && printf '%s\n' "${GITALY_VERSION}" \
>"${EXP_DIR}/results/${RUN_NAME}/gitaly-version.txt"
# Record the run window for report.py's Grafana deep link, and mark it on the
# dashboard. Best-effort: a benchmark never fails because Grafana was down.
{
printf 'RUN_NAME=%s\n' "${RUN_NAME}"
printf 'GITALY_SERVICE=%s\n' "${GITALY_SERVICE}"
printf 'RUN_START_EPOCH=%s\n' "${RUN_START_EPOCH}"
printf 'RUN_END_EPOCH=%s\n' "${RUN_END_EPOCH}"
printf 'ANNOTATION_LABEL=%s\n' "${BENCHMARK_ANNOTATION_LABEL:-}"
} >"${EXP_DIR}/results/${RUN_NAME}/grafana-run.env"
if [ -n "${BENCHMARK_ANNOTATION_LABEL:-}" ]; then
_anno_text="${BENCHMARK_ANNOTATION_LABEL} · ${GITALY_SERVICE} · ${RUN_NAME}"
else
_anno_text="${GITALY_SERVICE} · ${RUN_NAME}"
fi
"${SCRIPT_DIR}/incluster/grafana" annotate \
--text "${_anno_text}" \
--tag "${RUN_NAME}" --tag "${GITALY_SERVICE}" --tag "${EXPERIMENT}" \
--start "${RUN_START_EPOCH}" --end "${RUN_END_EPOCH}" || true
# Harvest results from GCS regardless of job status (partial results are useful).
if command -v gcloud >/dev/null 2>&1; then
# --no-user-output-enabled suppresses the per-file "Copying gs://..." lines
# (one per object is noisy in CI logs); errors are still surfaced.
gcloud storage rsync --recursive --no-user-output-enabled \
"gs://${BENCHMARK_RESULTS_BUCKET}/${RUN_NAME}/" \
"${EXP_DIR}/results/${RUN_NAME}/" || true
tar -czf "${EXP_DIR}/results/${RUN_NAME}.tar.gz" \
-C "${EXP_DIR}/results" "${RUN_NAME}" || true
elif command -v gsutil >/dev/null 2>&1; then
# -q quiets gsutil's per-file copy progress.
gsutil -q -m rsync -r "gs://${BENCHMARK_RESULTS_BUCKET}/${RUN_NAME}/" \
"${EXP_DIR}/results/${RUN_NAME}/" || true
tar -czf "${EXP_DIR}/results/${RUN_NAME}.tar.gz" \
-C "${EXP_DIR}/results" "${RUN_NAME}" || true
else
echo "[wrapper] WARNING: neither gcloud nor gsutil on PATH; skipping results download"
echo "[wrapper] results remain at gs://${BENCHMARK_RESULTS_BUCKET}/${RUN_NAME}/"
fi
echo "[wrapper] cleaning up Job and ConfigMap (bucket contents preserved)"
${KUBECTL} -n "${KUBE_NAMESPACE}" delete job "${JOB_NAME}" --ignore-not-found
${KUBECTL} -n "${KUBE_NAMESPACE}" delete configmap "${CM_NAME}" --ignore-not-found
[ "${job_status}" = "success" ] || exit 1
echo "[wrapper] benchmark complete: ${EXP_DIR}/results/${RUN_NAME}.tar.gz"