Skip to content

Harness engineering beginner tutorial, from 0 to 1

5.0 KB
Raw Blame
#!/usr/bin/env bash
# benchmark.sh -- Performance benchmark suite for the Knowledge Base application.
#
# Measures import throughput, indexing speed, query latency, and data integrity.
# This script simulates operations against the services layer.
#
# Usage: bash scripts/benchmark.sh
set -euo pipefail

BENCH_DIR=$(mktemp -d)
SAMPLE_DIR="$(cd "$(dirname "$0")/.." && pwd)/data/sample-documents"

echo "=== Knowledge Base Benchmark Suite ==="
echo ""
echo "Working directory: $BENCH_DIR"
echo "Sample data: $SAMPLE_DIR"
echo ""

PASS_COUNT=0
FAIL_COUNT=0
TOTAL_TASKS=4

# ---- Task 1: Import ----

echo "[1/4] Import Benchmark"
IMPORT_START=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)

import_count=0
for file in "$SAMPLE_DIR"/*; do
  if [ -f "$file" ]; then
    # Simulate import: copy to benchmark directory
    filename=$(basename "$file")
    size=$(wc -c < "$file" | tr -d ' ')
    cp "$file" "$BENCH_DIR/$filename"
    echo "  Imported: $filename ($size bytes)"
    import_count=$((import_count + 1))
  fi
done

IMPORT_END=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
IMPORT_MS=$(( (IMPORT_END - IMPORT_START) * 1000 ))

if [ "$import_count" -ge 3 ]; then
  echo "  PASS: $import_count files imported in ${IMPORT_MS}ms"
  PASS_COUNT=$((PASS_COUNT + 1))
else
  echo "  FAIL: Only $import_count files imported (expected 3)"
  FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""

# ---- Task 2: Indexing ----

echo "[2/4] Indexing Benchmark"
INDEX_START=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)

total_chunks=0
for file in "$BENCH_DIR"/*; do
  if [ -f "$file" ]; then
    content=$(cat "$file")
    # Simulate chunking: split on double newlines, count chunks
    # Approximate ~500 char chunks
    para_count=$(echo "$content" | awk 'BEGIN{RS="\n\n"} NF{c++} END{print c+0}')
    # Estimate chunks (merge short paragraphs)
    estimated_chunks=$(( (para_count + 1) / 2 ))
    if [ "$estimated_chunks" -lt 1 ]; then
      estimated_chunks=1
    fi
    total_chunks=$((total_chunks + estimated_chunks))
    echo "  $(basename "$file"): ~${estimated_chunks} estimated chunks"
  fi
done

INDEX_END=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
INDEX_MS=$(( (INDEX_END - INDEX_START) * 1000 ))

if [ "$total_chunks" -ge 5 ]; then
  echo "  PASS: ~$total_chunks total chunks estimated in ${INDEX_MS}ms"
  PASS_COUNT=$((PASS_COUNT + 1))
else
  echo "  FAIL: Only ~$total_chunks chunks estimated (expected 5+)"
  FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""

# ---- Task 3: Query ----

echo "[3/4] Query Benchmark"

queries=(
  "What is the system architecture?"
  "How does document import work?"
  "Explain the indexing pipeline"
  "What was discussed in the meeting?"
  "How does logging work?"
)

QUERY_START=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)

query_count=0
for query in "${queries[@]}"; do
  # Simulate query processing: keyword matching against content
  keyword_count=0
  for file in "$BENCH_DIR"/*; do
    if [ -f "$file" ]; then
      # Count keyword matches in file content
      for word in $query; do
        if [ ${#word} -gt 2 ]; then
          matches=$(grep -oi "$word" "$file" 2>/dev/null | wc -l | tr -d ' ')
          keyword_count=$((keyword_count + matches))
        fi
      done
    fi
  done
  echo "  Query: \"$query\" -> $keyword_count keyword matches"
  query_count=$((query_count + 1))
done

QUERY_END=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
QUERY_MS=$(( (QUERY_END - QUERY_START) * 1000 ))

if [ "$query_count" -eq 5 ]; then
  avg_ms=$((QUERY_MS / query_count))
  echo "  PASS: $query_count queries in ${QUERY_MS}ms (${avg_ms}ms avg)"
  PASS_COUNT=$((PASS_COUNT + 1))
else
  echo "  FAIL: Only $query_count queries completed (expected 5)"
  FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""

# ---- Task 4: Verify ----

echo "[4/4] Data Integrity Verification"

verify_ok=true
sample_files=("design-notes.md" "meeting-summary.txt" "retrieval-plan.md")

for sf in "${sample_files[@]}"; do
  if [ ! -f "$BENCH_DIR/$sf" ]; then
    echo "  FAIL: Missing imported file: $sf"
    verify_ok=false
  else
    original_size=$(wc -c < "$SAMPLE_DIR/$sf" | tr -d ' ')
    imported_size=$(wc -c < "$BENCH_DIR/$sf" | tr -d ' ')
    if [ "$original_size" -ne "$imported_size" ]; then
      echo "  FAIL: Size mismatch for $sf ($original_size vs $imported_size)"
      verify_ok=false
    fi
  fi
done

if [ "$verify_ok" = true ]; then
  echo "  PASS: All files verified (size match)"
  PASS_COUNT=$((PASS_COUNT + 1))
else
  FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""

# ---- Summary ----

echo "=== Benchmark Results ==="
echo "Import: $import_count files in ${IMPORT_MS}ms"
echo "Index:  ~$total_chunks chunks in ${INDEX_MS}ms"
echo "Query:  $query_count queries in ${QUERY_MS}ms"
echo "Verify: Data integrity check"
echo ""
echo "=== Summary: $PASS_COUNT/$TOTAL_TASKS tasks passed ==="

# Cleanup
rm -rf "$BENCH_DIR"

if [ "$PASS_COUNT" -eq "$TOTAL_TASKS" ]; then
  echo "ALL BENCHMARKS PASSED"
  exit 0
else
  echo "SOME BENCHMARKS FAILED ($FAIL_COUNT failures)"
  exit 1
fi