Skip to content

Git designed for AI agents.

12.6 KB
Raw Blame
#!/bin/sh
# Entry point for the in-cluster benchmark Job.
#
# Reads its configuration from environment variables (set on the Job spec) and
# from files mounted by a ConfigMap at /etc/benchmark/:
#   - /etc/benchmark/k6-benchmark.js : the workload script
#   - /etc/benchmark/repositories.json : test data manifest
#   - /etc/benchmark/plot.py : optional pandas/plotnine post-processor;
#                              if present and non-empty, runs after k6 against
#                              each harvested gitaly-<N>.log to produce PNGs
#                              under plots/gitaly-<N>/.
#
# Env vars read by this script:
#   RUN_NAME              - unique identifier for this run (required)
#   PROFILE               - "true"/"false", default true. When true, scrape
#                           Go pprof profiles from each Gitaly pod's metrics
#                           endpoint (HTTP, no in-pod execution required).
#   PROFILE_DURATION      - seconds to profile (CPU + execution trace), default 120
#   GITALY_NAMESPACE      - namespace containing the Gitaly pod, default "gitlab"
#   GITALY_SERVICE        - Service name the benchmark targets; its Endpoints
#                           determine which pods' logs are harvested
#   GITALY_POD_SELECTOR   - label selector for the Gitaly pod, default "app=gitaly"
#                           (pprof scraping, and log-harvest fallback when
#                           GITALY_SERVICE is unset/unresolvable)
#   GITALY_CONTAINER      - container name within the pod, default "gitaly"
#   GITALY_METRICS_PORT   - Gitaly Prometheus / pprof port, default 9236
#
# Other env vars on the container (GITALY_ADDRESS, GITALY_PROTO_DIR,
# WORKLOAD_DURATION, GITALY_TOKEN) are consumed by k6-benchmark.js, not this
# script; see that file for their meaning.
#
# On exit (success or failure) this script touches the file
# /tmp/run/.done so the GCS uploader sidecar can finalize the run and exit.

set -eu

RESULTS_DIR="/tmp/run/${RUN_NAME}"
mkdir -p "${RESULTS_DIR}"

cleanup() {
    touch /tmp/run/.done
}
trap cleanup EXIT

cp /etc/benchmark/repositories.json /opt/benchmark-gitaly/repositories.json
cp /etc/benchmark/k6-benchmark.js   /opt/benchmark-gitaly/k6-benchmark.js
if [ -s /etc/benchmark/plot.py ]; then
    cp /etc/benchmark/plot.py /opt/benchmark-gitaly/plot.py
fi

PROFILE="${PROFILE:-true}"
PROFILE_DURATION="${PROFILE_DURATION:-120}"
GITALY_NAMESPACE="${GITALY_NAMESPACE:-gitlab}"
GITALY_POD_SELECTOR="${GITALY_POD_SELECTOR:-app=gitaly}"
GITALY_CONTAINER="${GITALY_CONTAINER:-gitaly}"
GITALY_METRICS_PORT="${GITALY_METRICS_PORT:-9236}"

# ssh transport: materialize the key from the Secret at 0600 (ssh refuses
# one others can read). No key leaves the command unset, failing loudly.
GITLAB_SSH_URL="${GITLAB_SSH_URL:-ssh://git@gitlab-gitlab-shell.${GITALY_NAMESPACE}.svc:22}"
export GITLAB_SSH_URL
if [ -n "${BENCH_SSH_KEY:-}" ]; then
    bench_ssh_key=/opt/benchmark-gitaly/ssh/id_ed25519
    mkdir -p "$(dirname "${bench_ssh_key}")"
    printf '%s\n' "${BENCH_SSH_KEY}" > "${bench_ssh_key}"
    chmod 0600 "${bench_ssh_key}"
    BENCH_SSH_COMMAND="ssh -i ${bench_ssh_key} -o BatchMode=yes"
    BENCH_SSH_COMMAND="${BENCH_SSH_COMMAND} -o StrictHostKeyChecking=no"
    BENCH_SSH_COMMAND="${BENCH_SSH_COMMAND} -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
    export BENCH_SSH_COMMAND
    echo "[entrypoint] ssh transport armed against ${GITLAB_SSH_URL}" >&2
else
    echo "[entrypoint] WARNING: BENCH_SSH_KEY unset; ssh transport unavailable" >&2
fi

# Shallow-clone every test repository once: a real `git push` needs local
# history to negotiate against. Anonymous HTTP works because the bench
# projects are public. A failed clone only fails that repo's push checks.
CLONES_DIR="${CLONES_DIR:-/opt/benchmark-gitaly/clones}"
GITLAB_HTTP_URL="${GITLAB_HTTP_URL:-http://gitlab-webservice-default.${GITALY_NAMESPACE}.svc:8181}"
BENCH_GROUP="${BENCH_GROUP:-bench}"
export CLONES_DIR GITLAB_HTTP_URL BENCH_GROUP
mkdir -p "${CLONES_DIR}"
jq -r '.[] | select(.include_in_test) | .name' \
    /opt/benchmark-gitaly/repositories.json |
while read -r repo_name; do
    [ -n "${repo_name}" ] || continue
    echo "[entrypoint] cloning ${BENCH_GROUP}/${repo_name} for push sections" >&2
    git clone --quiet --bare --depth 1 \
        "${GITLAB_HTTP_URL}/${BENCH_GROUP}/${repo_name}.git" \
        "${CLONES_DIR}/${repo_name}.git" ||
        echo "[entrypoint] clone of ${repo_name} failed; its push sections will fail" >&2
done

# Scrape Go pprof from each Gitaly pod over HTTP; replaces the VM flow's
# profile-gitaly, which needed SYS_PTRACE the chart doesn't grant. Pods
# are hit directly (the Service VIP would round-robin). CPU and trace run
# alongside k6; heap/goroutine/allocs/mutex/block snapshots follow after.
profile_pid=""
if [ "${PROFILE}" = "true" ]; then
    mkdir -p "${RESULTS_DIR}/pprof"
    pods="$(kubectl --namespace "${GITALY_NAMESPACE}" \
        get pods -l "${GITALY_POD_SELECTOR}" \
        -o jsonpath='{range .items[*]}{.metadata.name} {.status.podIP}{"\n"}{end}' \
        2>/dev/null)"
    if [ -z "${pods}" ]; then
        echo "[entrypoint] no Gitaly pods found for selector '${GITALY_POD_SELECTOR}' in ns '${GITALY_NAMESPACE}'" >&2
    else
        echo "${pods}" > "${RESULTS_DIR}/pprof/targets.txt"
        echo "[entrypoint] scraping pprof from $(echo "${pods}" | wc -l) Gitaly pod(s) for ${PROFILE_DURATION}s" >&2
        (
            set +e
            echo "${pods}" | while read -r pod ip; do
                [ -n "${pod}" ] || continue
                [ -n "${ip}" ] || continue
                base="http://${ip}:${GITALY_METRICS_PORT}/debug/pprof"
                out="${RESULTS_DIR}/pprof/${pod}"
                mkdir -p "${out}"
                curl -fsS --max-time "$((PROFILE_DURATION + 30))" \
                    -o "${out}/cpu.pprof" \
                    "${base}/profile?seconds=${PROFILE_DURATION}" \
                    2> "${out}/cpu.err" &
                curl -fsS --max-time "$((PROFILE_DURATION + 30))" \
                    -o "${out}/trace.out" \
                    "${base}/trace?seconds=${PROFILE_DURATION}" \
                    2> "${out}/trace.err" &
            done
            wait
        ) &
        profile_pid=$!
    fi
fi

# Stream the Gitaly pods' logs instead of harvesting afterwards: the
# kubelet rotates container logs at 10MB, which a loaded Gitaly exceeds,
# so a post-run harvest only sees the tail of the run. --tail=-1 also
# keeps the pre-run buffer.
gitaly_log_pods=""
if [ -n "${GITALY_SERVICE:-}" ]; then
    gitaly_log_pods="$(kubectl --namespace "${GITALY_NAMESPACE}" \
        get endpoints "${GITALY_SERVICE}" \
        -o jsonpath='{range .subsets[*]}{range .addresses[*]}{.targetRef.name}{"\n"}{end}{range .notReadyAddresses[*]}{.targetRef.name}{"\n"}{end}{end}' \
        2>/dev/null | grep . | sort -u || true)"
    if [ -z "${gitaly_log_pods}" ]; then
        echo "[entrypoint] could not resolve pods from endpoints of service '${GITALY_SERVICE}'; falling back to selector '${GITALY_POD_SELECTOR}' (may include other Gitaly deployments)" >&2
    fi
fi
if [ -z "${gitaly_log_pods}" ]; then
    gitaly_log_pods="$(kubectl --namespace "${GITALY_NAMESPACE}" \
        get pods -l "${GITALY_POD_SELECTOR}" \
        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \
        2>/dev/null | sort)"
fi
log_follow_pids=""
if [ -z "${gitaly_log_pods}" ]; then
    echo "[entrypoint] no Gitaly pods found for service '${GITALY_SERVICE:-}' or selector '${GITALY_POD_SELECTOR}' in ns '${GITALY_NAMESPACE}'; skipping log capture" >&2
else
    i=1
    for pod in ${gitaly_log_pods}; do
        echo "[entrypoint] streaming logs from ${pod} -> gitaly-${i}.log" >&2
        printf 'gitaly-%s.log %s\n' "${i}" "${pod}" \
            >>"${RESULTS_DIR}/gitaly-log-pods.txt"
        kubectl --namespace "${GITALY_NAMESPACE}" logs "${pod}" \
            -c "${GITALY_CONTAINER}" --tail=-1 --follow \
            >"${RESULTS_DIR}/gitaly-${i}.log" 2>&1 &
        log_follow_pids="${log_follow_pids} $!"
        i=$((i + 1))
    done
fi

echo "[entrypoint] running k6 against ${GITALY_ADDRESS}" >&2
set +e
k6 run \
    --verbose \
    --summary-mode=full \
    --summary-export="${RESULTS_DIR}/k6-summary.json" \
    --console-output="${RESULTS_DIR}/k6-console.txt" \
    --log-output="file=${RESULTS_DIR}/k6-log.txt" \
    /opt/benchmark-gitaly/k6-benchmark.js
k6_status=$?
set -e

if [ -n "${profile_pid}" ]; then
    wait "${profile_pid}" || true
fi

if [ "${PROFILE}" = "true" ] && [ -n "${pods:-}" ]; then
    echo "[entrypoint] capturing point-in-time pprof snapshots" >&2
    echo "${pods}" | while read -r pod ip; do
        [ -n "${pod}" ] || continue
        [ -n "${ip}" ] || continue
        base="http://${ip}:${GITALY_METRICS_PORT}/debug/pprof"
        out="${RESULTS_DIR}/pprof/${pod}"
        mkdir -p "${out}"
        for kind in heap goroutine allocs mutex block; do
            curl -fsS --max-time 30 \
                -o "${out}/${kind}.pprof" \
                "${base}/${kind}" \
                2> "${out}/${kind}.err" || true
        done
    done
fi

# Stop the log followers started before k6 (short grace so trailing
# lines land in the files first).
if [ -n "${log_follow_pids}" ]; then
    echo "[entrypoint] stopping Gitaly log streams" >&2
    sleep 5
    for pid in ${log_follow_pids}; do
        kill "${pid}" 2>/dev/null || true
    done
    # shellcheck disable=SC2086 # deliberate word split of the PID list
    wait ${log_follow_pids} 2>/dev/null || true
fi

# Per-RPC latency stats from the harvested logs: Gitaly's server-side
# grpc.time_ms, unlike k6's client-side wall-clock numbers. Gitaly's log
# fields are flat keys containing literal dots, so the queries need agrind's
# ["..."] syntax - bare grpc.time_ms is a nested lookup and matches nothing.
# The "grpc.time_ms" prefilter and null guard drop start-of-call lines,
# which carry grpc.method but no duration.
AGRIND_LATENCY_QUERY='"grpc.time_ms"
    | json
    | where ["grpc.time_ms"] != null
    | avg(["grpc.time_ms"]),
      min(["grpc.time_ms"]),
      p50(["grpc.time_ms"]) as med,
      max(["grpc.time_ms"]),
      p90(["grpc.time_ms"]),
      p95(["grpc.time_ms"])
      by ["grpc.method"]
    | sort by p95 desc'
# Ungrouped variant: the report's all-RPCs percentiles cannot be recombined
# from the per-RPC rows, so they get their own pass over the raw samples.
AGRIND_OVERALL_QUERY='"grpc.time_ms"
    | json
    | where ["grpc.time_ms"] != null
    | avg(["grpc.time_ms"]),
      min(["grpc.time_ms"]),
      p50(["grpc.time_ms"]) as med,
      max(["grpc.time_ms"]),
      p90(["grpc.time_ms"]),
      p95(["grpc.time_ms"])'
AGRIND_CODE_QUERY='"grpc.code" | json
| where ["grpc.code"] != null
| count by ["grpc.method"], ["grpc.code"]
| sort by _count desc'

grpc_stats() {
    stats_log=$1
    stats_out=$2
    agrind --file "${stats_log}" "${AGRIND_LATENCY_QUERY}" \
        > "${stats_out}-latency.txt" 2>/dev/null || true
    agrind --file "${stats_log}" -o json "${AGRIND_LATENCY_QUERY}" \
        > "${stats_out}-latency.json" 2>/dev/null || true
    agrind --file "${stats_log}" -o json "${AGRIND_OVERALL_QUERY}" \
        > "${stats_out}-latency-overall.json" 2>/dev/null || true
    agrind --file "${stats_log}" "${AGRIND_CODE_QUERY}" \
        > "${stats_out}-codes.txt" 2>/dev/null || true
}

if command -v agrind >/dev/null 2>&1; then
    stats_dir="${RESULTS_DIR}/gitaly-stats"
    mkdir -p "${stats_dir}"
    for log in "${RESULTS_DIR}"/gitaly-*.log; do
        [ -s "${log}" ] || continue
        name="$(basename "${log}" .log)"
        echo "[entrypoint] computing gRPC stats from ${name}.log" >&2
        grpc_stats "${log}" "${stats_dir}/${name}"
    done
    combined_log="/tmp/gitaly-combined.log"
    cat "${RESULTS_DIR}"/gitaly-*.log > "${combined_log}" 2>/dev/null || true
    if [ -s "${combined_log}" ]; then
        echo "[entrypoint] computing combined gRPC stats across all Gitaly logs" >&2
        grpc_stats "${combined_log}" "${stats_dir}/all-gitalies"
    fi
    rm -f "${combined_log}"
else
    echo "[entrypoint] agrind not found; skipping gRPC log stats" >&2
fi

if [ -f /opt/benchmark-gitaly/plot.py ]; then
    for log in "${RESULTS_DIR}"/gitaly-*.log; do
        [ -s "${log}" ] || continue
        name="$(basename "${log}" .log)"
        plot_dir="${RESULTS_DIR}/plots/${name}"
        echo "[entrypoint] generating plots from ${name}.log" >&2
        mkdir -p "${plot_dir}"
        python3 /opt/benchmark-gitaly/plot.py \
            "${log}" \
            "${plot_dir}" \
            >"${plot_dir}/plot-stdout.log" \
            2>"${plot_dir}/plot-stderr.log" ||
            echo "[entrypoint] plot.py failed for ${name}.log (non-fatal; see plots/${name}/plot-stderr.log)" >&2
    done
fi

echo "[entrypoint] k6 exited with status ${k6_status}" >&2
exit "${k6_status}"