Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

10.8 KB
Raw Blame
#!/usr/bin/env bash
# Follows the shell-script conventions of precedent #22.

set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$HERE/_measure-lib.sh"

usage() {
  cat <<'EOF'
Usage: packages/app/scripts/perf-prod.sh --scenario=<name> [--runs=N] [--env="K=V ..."]

Prod-fidelity perf measurement wrapper. Orchestrates build → start CLI → run
perf:profile N times → tear down, so a developer measures against the built
artifact (equivalent to what ships to users) instead of `pnpm run dev`, whose
Vite HMR and transform overhead make the numbers uncomparable.

Required:
  --scenario=<name>   Scenario to profile. Names come from
                      packages/app/tests/perf/scenarios/; omitting the flag
                      prints the available list.

Optional:
  --runs=N            Number of profile runs (default: 5). Must be >= 1.
  --env="K=V ..."     Extra environment for each perf:profile run. Word-split, so
                      several assignments can share one flag.
  -h, --help          Show this message.

All three value flags are =-form only: `--scenario cold-pool-warm` is an unknown
argument, not a scenario.

Examples:
  packages/app/scripts/perf-prod.sh --scenario=cold-pool-warm \
    --env="OK_PERF_BIG_DOC=perf-fixtures/medium-doc"
  packages/app/scripts/perf-prod.sh --scenario=mode-toggle --runs=10

What it does:
  1. Runs `pnpm run build` from the repo root (turbo-cached; a no-op when clean).
  2. Starts ONE CLI process on a kernel-assigned port —
     `node packages/cli/dist/cli.mjs start --port 0`, the project server (React
     shell + Hocuspocus + /api/* on one listener) — and polls .ok/local/server.lock
     for up to 30s for the bound port. perf:profile targets that same port.
  3. Runs `pnpm run perf:profile --scenario=<name> --target=http://localhost:<port>
     --headless` N times. Results land at
     packages/app/tests/perf/results/<scenario>.<timestamp>.json. A run that exits
     non-zero is counted and the loop continues.
  4. Sends SIGTERM, waits up to 5s for the process to exit and the lock to
     disappear, then SIGKILLs.
  5. Reads the N most recent result files. For a scenario with a registered
     primary metric, prints each value and the median; otherwise lists the files
     and the jq expression for extracting a custom metric.

Output:
  Progress lines, a SUMMARY block, and `MEDIAN: <n> ms` when a primary metric is
  registered and at least one non-negative value parsed. Partial run failures are
  reported and the median is computed from what remains.

Exit codes:
  0   the run finished; median reported when a primary metric exists
  1   usage error: unknown argument, missing --scenario, or --runs < 1
  2   --runs is not a non-negative integer, `pnpm run build` failed, or the CLI
      binary is missing after the build
  3   jq is not installed, or the server never populated .ok/local/server.lock
      with a bound port
  4   the workspace-root markers are missing, or all N perf:profile runs failed
  5   no result files found for the scenario

Not a replacement for the CI-based baseline-capture protocol — see
packages/app/tests/stress/perf-baseline-update.md §"Local prod-fidelity dry-run",
which lives in Inkeep's internal tree and is absent from the open-source
repository. Runner hardware differs; local medians are directional signal only.
EOF
}

SCENARIO=""
RUNS=5
EXTRA_ENV=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    --scenario=*) SCENARIO="${1#--scenario=}"; shift ;;
    --runs=*)     RUNS="${1#--runs=}"; shift ;;
    --env=*)      EXTRA_ENV="${1#--env=}"; shift ;;
    -h|--help)
      usage
      exit 0
      ;;
    *) echo "error: unknown arg: $1" >&2; exit 1 ;;
  esac
done

if [[ -z "$SCENARIO" ]]; then
  echo "error: --scenario=<name> is required" >&2
  echo "       available scenarios (packages/app/tests/perf/scenarios/):" >&2
  ls "$HERE/../tests/perf/scenarios/" 2>/dev/null | grep '\.ts$' | sed 's/\.ts$//' | sed 's/^/         /' >&2
  exit 1
fi
assert_numeric_flag --runs "$RUNS"
if (( RUNS < 1 )); then
  echo "error: --runs must be >= 1 (got $RUNS)" >&2
  exit 1
fi

require_jq
REPO_ROOT="$(resolve_repo_root)"

CLI_BIN="$REPO_ROOT/packages/cli/dist/cli.mjs"
SERVER_LOCK="$REPO_ROOT/.ok/local/server.lock"
RESULTS_DIR="$REPO_ROOT/packages/app/tests/perf/results"

echo "[perf-prod] Building app + cli (turbo cache-friendly; no-op when clean)…"
cd "$REPO_ROOT"
if ! pnpm run build >/dev/null 2>&1; then
  echo "error: pnpm run build failed — re-run manually to see output:" >&2
  echo "       cd $REPO_ROOT && pnpm run build" >&2
  exit 2
fi

if [[ ! -x "$CLI_BIN" && ! -f "$CLI_BIN" ]]; then
  echo "error: CLI binary not found at $CLI_BIN after build" >&2
  exit 2
fi

for lock in "$SERVER_LOCK"; do
  if [[ -f "$lock" ]]; then
    STALE_PID="$(jq -r '.pid // "0"' "$lock" 2>/dev/null || echo "0")"
    if [[ "$STALE_PID" != "0" ]] && ! kill -0 "$STALE_PID" 2>/dev/null; then
      echo "[perf-prod] Removing stale $(basename "$lock") from dead pid $STALE_PID"
      rm -f "$lock"
    fi
  fi
done

SERVER_LOG="$(mktemp -t perf-prod-server.XXXXXX.log)"

if [[ -d "$REPO_ROOT/tmp" ]]; then
  BROKEN_LINKS=()
  while IFS= read -r -d '' link; do
    if [[ ! -e "$link" ]]; then
      BROKEN_LINKS+=("$link")
    fi
  done < <(find "$REPO_ROOT/tmp" -type l -print0 2>/dev/null)
  if (( ${#BROKEN_LINKS[@]} > 0 )); then
    echo "[perf-prod] Removing ${#BROKEN_LINKS[@]} broken symlink(s) under tmp/ (would crash the static serve):"
    for link in "${BROKEN_LINKS[@]}"; do
      echo "  $link"
      rm -f "$link"
    done
  fi
fi

echo "[perf-prod] Starting open-knowledge (project server) on kernel-assigned port…"
node "$CLI_BIN" start --port 0 >"$SERVER_LOG" 2>&1 &
SERVER_PID=$!

cleanup() {
  local ec=$?
  if [[ -n "${SERVER_PID:-}" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
    echo "[perf-prod] Stopping server (pid $SERVER_PID)…"
    kill -TERM %1 2>/dev/null || true
  fi
  for _ in 1 2 3 4 5 6 7 8 9 10; do
    if ! kill -0 "$SERVER_PID" 2>/dev/null && [[ ! -f "$SERVER_LOCK" ]]; then break; fi
    sleep 0.5
  done
  if [[ -n "${SERVER_PID:-}" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
    echo "[perf-prod] WARN: pid $SERVER_PID did not exit on SIGTERM; sending SIGKILL" >&2
    kill -KILL %1 2>/dev/null || true
  fi
  if [[ $ec -ne 0 ]]; then
    echo "[perf-prod] Script exited non-zero ($ec). Log:" >&2
    echo "  server: $SERVER_LOG" >&2
  else
    rm -f "$SERVER_LOG" 2>/dev/null || true
  fi
  exit $ec
}
trap cleanup EXIT INT TERM

wait_for_lock() {
  local lock="$1"
  local pid="$2"
  local label="$3"
  local log="$4"
  local port=""
  for i in $(seq 1 60); do
    if [[ -f "$lock" ]]; then
      port="$(jq -r '.port // 0' "$lock" 2>/dev/null || echo "0")"
      if [[ "$port" =~ ^[0-9]+$ ]] && (( port > 0 )); then
        echo "[perf-prod] $label listening on port $port (after ${i} * 0.5s = $((i*500))ms)" >&2
        printf '%s\n' "$port"
        return 0
      fi
    fi
    if ! kill -0 "$pid" 2>/dev/null; then
      echo "error: $label process exited before binding a port — log:" >&2
      tail -30 "$log" >&2 || true
      return 1
    fi
    sleep 0.5
  done
  echo "error: $label lock never populated with a real port after 30s — log:" >&2
  tail -30 "$log" >&2 || true
  return 1
}

echo "[perf-prod] Waiting for $SERVER_LOCK with a bound port…"
PORT="$(wait_for_lock "$SERVER_LOCK" "$SERVER_PID" "server" "$SERVER_LOG")" || exit 3

APP_DIR="$REPO_ROOT/packages/app"
cd "$APP_DIR"

run_failures=0
for i in $(seq 1 "$RUNS"); do
  echo "[perf-prod] Run $i/$RUNS — scenario=$SCENARIO target=http://localhost:$PORT"
  if [[ -n "$EXTRA_ENV" ]]; then
    if ! env $EXTRA_ENV pnpm run perf:profile --scenario="$SCENARIO" --target="http://localhost:$PORT" --headless; then
      run_failures=$((run_failures + 1))
      echo "[perf-prod] WARN: run $i exited non-zero — continuing" >&2
    fi
  else
    if ! pnpm run perf:profile --scenario="$SCENARIO" --target="http://localhost:$PORT" --headless; then
      run_failures=$((run_failures + 1))
      echo "[perf-prod] WARN: run $i exited non-zero — continuing" >&2
    fi
  fi
done

if (( run_failures == RUNS )); then
  echo "error: all $RUNS runs failed — see logs above" >&2
  exit 4
fi
if (( run_failures > 0 )); then
  echo "[perf-prod] NOTE: $run_failures/$RUNS runs failed (median computed from remaining)" >&2
fi

case "$SCENARIO" in
  warm-switch-cached)      METRIC=".metrics.warmSwitchCachedMs" ;;
  cold-pool-warm)          METRIC=".metrics.coldPoolWarmMs" ;;
  cold-load-with-fallback) METRIC=".metrics.interactiveReadyMs" ;;
  cold-load-big-doc)       METRIC=".metrics.coldLoadMs" ;;
  mode-toggle)             METRIC=".metrics.modeToggleMs" ;;
  warm-switch)             METRIC=".metrics.warmSwitchMs" ;;
  outline-polling)         METRIC=".metrics.hoverToOutlineMs" ;;
  *)                       METRIC="" ;;
esac

cd "$RESULTS_DIR"
latest_files="$(ls -t "$SCENARIO".*.json 2>/dev/null | head -n "$RUNS" || true)"
if [[ -z "$latest_files" ]]; then
  echo "error: no result files found for scenario=$SCENARIO in $RESULTS_DIR" >&2
  exit 5
fi

echo ""
echo "[perf-prod] ===== SUMMARY — $SCENARIO ====="
echo "[perf-prod] Runs: $RUNS (failures: $run_failures)"
echo "[perf-prod] Results: $RESULTS_DIR"
echo ""
if [[ -z "$METRIC" ]]; then
  echo "[perf-prod] No primary metric registered for this scenario — listing result files:"
  echo "$latest_files" | sed 's/^/  /'
  echo ""
  echo "[perf-prod] To extract a custom metric: jq '.metrics.<field>' <result-file>"
else
  echo "[perf-prod] Primary metric: $METRIC"
  echo ""
  values=""
  while IFS= read -r f; do
    [[ -z "$f" ]] && continue
    v="$(jq -r "$METRIC // -1" "$f" 2>/dev/null || echo "-1")"
    printf '[perf-prod]   %s: %s ms\n' "$f" "$v"
    values+="$v"$'\n'
  done <<< "$latest_files"

  # The median uses a POSIX awk insertion sort rather than GNU asort() (precedent #22(d)).
  median="$(printf '%s' "$values" | LC_ALL=C awk '
    /^-?[0-9]+(\.[0-9]+)?$/ && $1 >= 0 { a[c++] = $1 + 0 }
    END {
      if (c == 0) { print "NO_VALID_VALUES"; exit }
      # Insertion sort — portable across BSD awk + gawk + mawk.
      for (i = 1; i < c; i++) {
        key = a[i]
        j = i - 1
        while (j >= 0 && a[j] > key) {
          a[j+1] = a[j]
          j--
        }
        a[j+1] = key
      }
      if (c % 2 == 1) { print a[(c-1)/2] }
      else { print (a[c/2-1] + a[c/2]) / 2 }
    }
  ')"
  echo ""
  if [[ "$median" == "NO_VALID_VALUES" ]]; then
    echo "[perf-prod] WARN: no valid (non-negative) values to compute median from" >&2
  else
    printf '[perf-prod] MEDIAN: %s ms\n' "$median"
  fi
fi
echo ""
echo "[perf-prod] NOTE: For the canonical baseline capture, follow the CI-based"
echo "[perf-prod]       protocol in packages/app/tests/stress/perf-baseline-update.md"
echo "[perf-prod]       (Inkeep-internal; absent from the open-source repository)."
echo "[perf-prod]       Local medians are directional signal, not the authority."