Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

21.5 KB
Raw Blame
#!/usr/bin/env bash
# restore.sh — AiSOC full-stack restore from S3/R2
#
# Restores:
#   1. PostgreSQL (download + verify + decrypt + gunzip + psql)
#   2. ClickHouse (download + verify + decrypt + gunzip + HTTP INSERT)
#
# Artifacts written by backup.sh are AES-256-GCM encrypted and recorded in a
# SHA-256 manifest. Restore fetches the manifest first and checks the digest of
# every artifact it downloads against it, then decrypts. A digest mismatch
# aborts: restoring a corrupted dump is worse than not restoring, because it
# looks like it worked.
#   3. Plugin store (download + extract artifacts)
#
# Required environment variables (same as backup.sh):
#   BACKUP_S3_BUCKET      — s3://your-bucket or r2://your-bucket
#   BACKUP_S3_PREFIX      — key prefix inside bucket
#   POSTGRES_URL          — postgresql://user:pass@host:5432/dbname
#   CLICKHOUSE_HOST       — ClickHouse HTTP endpoint host
#   CLICKHOUSE_PORT       — ClickHouse HTTP port (default: 8123)
#   CLICKHOUSE_USER       — ClickHouse user
#   CLICKHOUSE_PASSWORD   — ClickHouse password
#   CLICKHOUSE_DATABASE   — ClickHouse database to restore
#   BACKUP_ENCRYPTION_KEY / BACKUP_ENCRYPTION_KEY_FILE — the key the backup
#                           was written with (see scripts/backup_crypt.py)
#   AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_ENDPOINT_URL
#
# Usage:
#   ./scripts/restore.sh --timestamp 20260503T120000Z \
#       [--component postgres|clickhouse|plugins|neo4j|qdrant|redis|all]
#   ./scripts/restore.sh --latest [--component all]
#   ./scripts/restore.sh --list   # show available backups

set -euo pipefail

# ── defaults ──────────────────────────────────────────────────────────────────
BACKUP_S3_BUCKET="${BACKUP_S3_BUCKET:-}"
BACKUP_S3_PREFIX="${BACKUP_S3_PREFIX:-aisoc-backups}"
POSTGRES_URL="${POSTGRES_URL:-}"
CLICKHOUSE_HOST="${CLICKHOUSE_HOST:-localhost}"
CLICKHOUSE_PORT="${CLICKHOUSE_PORT:-8123}"
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aisoc}"
TIMESTAMP=""
COMPONENT="all"
DO_LIST=false
USE_LATEST=false
RESTORE_DIR="/tmp/aisoc-restore-$$"

# ── arg parsing ───────────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    --timestamp)  TIMESTAMP="$2"; shift ;;
    --component)  COMPONENT="$2"; shift ;;
    --latest)     USE_LATEST=true ;;
    --list)       DO_LIST=true ;;
    *)            echo "Unknown arg: $1"; exit 1 ;;
  esac
  shift
done

# ── helpers ───────────────────────────────────────────────────────────────────
log()  { echo "[$(date -u +%T)] $*"; }

require() {
  command -v "$1" &>/dev/null || { echo "Missing required command: $1" >&2; exit 1; }
}

s3_args() {
  local args=()
  [[ -n "${AWS_ENDPOINT_URL:-}" ]] && args+=(--endpoint-url "$AWS_ENDPOINT_URL")
  echo "${args[@]}"
}

s3_ls() {
  local prefix="$1"
  # shellcheck disable=SC2046
  aws s3 ls $(s3_args) "${prefix}/" 2>/dev/null | awk '{print $4}' | sort
}

s3_download() {
  local src="$1" dest="$2"
  # shellcheck disable=SC2046
  aws s3 cp $(s3_args) "$src" "$dest"
}


# ── manifest verification + decryption ────────────────────────────────────────
# A backup you cannot verify is a backup you are guessing about. fetch_artifact
# is the only download path used below, so no component can skip the check.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MANIFEST_FILE=""

load_manifest() {
  local remote="${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/manifests/manifest-${TIMESTAMP}.json"
  MANIFEST_FILE="${RESTORE_DIR}/manifest.json"
  if s3_download "$remote" "$MANIFEST_FILE" 2>/dev/null; then
    log "Manifest loaded: $remote"
  else
    MANIFEST_FILE=""
    log "No manifest at ${remote}."
    log "  This backup predates manifest support, so its integrity cannot be"
    log "  verified. Continuing, but treat the restore as unverified."
  fi
}

manifest_lookup() {
  # artifact name -> "<sha256_ciphertext_or_empty>\t<encrypted:true|false>"
  [[ -z "$MANIFEST_FILE" ]] && { echo -e "\t"; return; }
  python3 - "$MANIFEST_FILE" "$1" <<'PYEOF'
import json, sys
doc = json.load(open(sys.argv[1]))
for a in doc.get("artifacts", []):
    if a["name"] == sys.argv[2]:
        print(f"{a.get('sha256_ciphertext') or ''}\t{'true' if a.get('encrypted') else 'false'}")
        break
else:
    print("\t")
PYEOF
}

fetch_artifact() {
  # Downloads <s3 dir>/<base>[.enc], verifies, decrypts. Echoes the local
  # plaintext path. Prefers the encrypted object when both exist.
  local s3dir="$1" base="$2" dest="$3"
  local remote_enc="${s3dir}/${base}.enc"
  local local_enc="${dest}.enc"

  if aws s3 ls $(s3_args) "$remote_enc" &>/dev/null; then
    s3_download "$remote_enc" "$local_enc"
    verify_artifact "$(basename "$remote_enc")" "$local_enc"
    require python3
    python3 "${SCRIPT_DIR}/backup_crypt.py" decrypt "$local_enc" --output "$dest" >/dev/null || {
      echo "[ERROR] Could not decrypt ${remote_enc}." >&2
      echo "        Check BACKUP_ENCRYPTION_KEY / BACKUP_ENCRYPTION_KEY_FILE matches" >&2
      echo "        the key this backup was written with." >&2
      exit 1
    }
    rm -f "$local_enc"
  else
    s3_download "${s3dir}/${base}" "$dest"
    verify_artifact "$base" "$dest"
  fi
  echo "$dest"
}

verify_artifact() {
  local name="$1" path="$2"
  [[ -z "$MANIFEST_FILE" ]] && return 0
  local expected actual
  expected=$(manifest_lookup "$name" | cut -f1)
  if [[ -z "$expected" ]]; then
    log "  ${name}: not listed in the manifest; integrity unverified"
    return 0
  fi
  actual=$(python3 "${SCRIPT_DIR}/backup_crypt.py" sha256 "$path")
  if [[ "$actual" != "$expected" ]]; then
    echo "[ERROR] Integrity check failed for ${name}." >&2
    echo "        manifest: ${expected}" >&2
    echo "        actual:   ${actual}" >&2
    echo "        Refusing to restore: a corrupted dump restores as if it worked." >&2
    exit 1
  fi
  log "  ${name}: sha256 verified"
}

# ── list backups ──────────────────────────────────────────────────────────────
list_backups() {
  [[ -z "$BACKUP_S3_BUCKET" ]] && { echo "BACKUP_S3_BUCKET is required" >&2; exit 1; }
  echo "=== Available PostgreSQL backups ==="
  s3_ls "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres" | sed 's/^/  /'
  echo ""
  echo "=== Available ClickHouse backup timestamps (first table) ==="
  # shellcheck disable=SC2046
  aws s3 ls $(s3_args) "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/clickhouse/" 2>/dev/null \
    | awk '{print $2}' | sed 's|/$||' | head -1 \
    | xargs -I{} aws s3 ls $(s3_args) "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/clickhouse/{}/" 2>/dev/null \
    | awk '{print $4}' | sed 's/^/  /' || echo "  (none)"
  echo ""
  echo "=== Available plugin store backups ==="
  s3_ls "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/plugins" | sed 's/^/  /'
}

if [[ "$DO_LIST" == "true" ]]; then
  list_backups
  exit 0
fi

# ── resolve timestamp ─────────────────────────────────────────────────────────
resolve_timestamp() {
  if [[ "$USE_LATEST" == "true" ]]; then
    log "Resolving latest backup timestamp…"
    # grep -oE, not -oP: BSD grep (macOS) has no -P, and the `|| true` below
    # turned that into the misleading "no postgres backups found" rather than
    # "your grep cannot do this" — on the machine someone is most likely to be
    # running a recovery from.
    local listing
    listing=$(s3_ls "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres" || true)
    TIMESTAMP=$(printf '%s\n' "$listing" \
      | grep -oE '[0-9]{8}T[0-9]{6}Z' | sort | tail -1 || true)
    if [[ -z "$TIMESTAMP" ]]; then
      echo "No postgres backups found to determine latest timestamp." >&2
      echo "  Looked in: ${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres" >&2
      if [[ -n "$listing" ]]; then
        echo "  Objects present but none matched <YYYYMMDD>T<HHMMSS>Z:" >&2
        printf '%s\n' "$listing" | sed 's/^/    /' >&2
      else
        echo "  The prefix is empty or unreadable — check credentials and endpoint." >&2
      fi
      exit 1
    fi
    log "Latest timestamp: $TIMESTAMP"
  fi
  if [[ -z "$TIMESTAMP" ]]; then
    echo "--timestamp or --latest is required" >&2
    exit 1
  fi
  # Return explicit success: with the previous `[[ -z … ]] && { … }` form, the
  # test evaluates false (exit 1) once a timestamp is resolved, and as the
  # function's last command that non-zero status propagated out and tripped
  # `set -e` in the caller — aborting every restore before it began.
  return 0
}

# ── pre-flight ─────────────────────────────────────────────────────────────────
require aws
require psql
require gzip
require curl

[[ -z "$BACKUP_S3_BUCKET" ]] && { echo "BACKUP_S3_BUCKET is required" >&2; exit 1; }

resolve_timestamp

mkdir -p "$RESTORE_DIR"
trap 'rm -rf "$RESTORE_DIR"' EXIT

load_manifest

log "=== AiSOC Restore started: timestamp=${TIMESTAMP} ==="
log "Component: ${COMPONENT}"

# ── confirmation prompt ────────────────────────────────────────────────────────
if [[ -t 0 ]]; then
  echo ""
  echo "WARNING: This will OVERWRITE the target database / files."
  read -r -p "Are you sure you want to restore from ${TIMESTAMP}? [yes/N] " confirm
  [[ "$confirm" != "yes" ]] && { log "Restore cancelled."; exit 0; }
fi

# ── 1. PostgreSQL restore ──────────────────────────────────────────────────────
restore_postgres() {
  [[ -z "$POSTGRES_URL" ]] && { echo "POSTGRES_URL is not set; skipping postgres restore" >&2; return; }
  log "--- PostgreSQL restore ---"
  local s3dir="${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres"
  local base="postgres-${TIMESTAMP}.sql.gz"
  local local_file="${RESTORE_DIR}/${base}"

  log "Downloading ${s3dir}/${base}…"
  fetch_artifact "$s3dir" "$base" "$local_file" >/dev/null
  log "Download complete ($(du -sh "$local_file" | cut -f1))"

  log "Restoring to ${POSTGRES_URL}…"
  gunzip -c "$local_file" | psql "$POSTGRES_URL" --single-transaction
  log "PostgreSQL restore complete"
}

# ── 2. ClickHouse restore ──────────────────────────────────────────────────────
restore_clickhouse() {
  log "--- ClickHouse restore ---"
  local ch_url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}"
  local auth_args=()
  [[ -n "$CLICKHOUSE_USER" ]]     && auth_args+=(--user "${CLICKHOUSE_USER}")
  [[ -n "$CLICKHOUSE_PASSWORD" ]] && auth_args+=(--password "${CLICKHOUSE_PASSWORD}")

  # List tables available in this backup
  local ch_s3_prefix="${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/clickhouse"
  # shellcheck disable=SC2046
  local tables
  tables=$(aws s3 ls $(s3_args) "${ch_s3_prefix}/" 2>/dev/null \
    | awk '{print $2}' | sed 's|/$||' || echo "")

  if [[ -z "$tables" ]]; then
    log "No ClickHouse backup directories found; skipping"
    return
  fi

  while IFS= read -r table; do
    [[ -z "$table" ]] && continue
    local s3dir="${ch_s3_prefix}/${table}"
    local base="${table}-${TIMESTAMP}.tsv.gz"
    local local_file="${RESTORE_DIR}/ch-${base}"

    log "  Restoring table: ${CLICKHOUSE_DATABASE}.${table}"
    # shellcheck disable=SC2046
    if ! aws s3 ls $(s3_args) "${s3dir}/${base}" &>/dev/null \
       && ! aws s3 ls $(s3_args) "${s3dir}/${base}.enc" &>/dev/null; then
      log "  Skipping ${table}: no object at ${s3dir}/${base}[.enc]"
      continue
    fi

    fetch_artifact "$s3dir" "$base" "$local_file" >/dev/null

    # The query goes in the URL and the data goes in the body. The previous
    # form combined --data-binary with --get, which turns the request into a
    # GET and URL-encodes the entire gzipped dump into the query string: every
    # ClickHouse restore failed, and the failure was masked because curl -sf
    # was the last command in a loop body.
    local encoded_query
    encoded_query=$(python3 -c \
      "import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1]))" \
      "INSERT INTO ${CLICKHOUSE_DATABASE}.${table} FORMAT TabSeparatedWithNames")

    if ! gunzip -c "$local_file" | curl -sf -X POST "${ch_url}/?query=${encoded_query}" \
      ${auth_args[@]+"${auth_args[@]}"} \
      --data-binary @-; then
      echo "[ERROR] ClickHouse INSERT failed for table ${table}" >&2
      exit 1
    fi

    log "  Table ${table} restored"
  done <<< "$tables"

  log "ClickHouse restore complete"
}

# ── 3. Plugin store restore ─────────────────────────────────────────────────────
restore_plugins() {
  log "--- Plugin store restore ---"
  local repo_root
  repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
  local s3prefix="${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/plugins"
  local outdir="${RESTORE_DIR}/plugins"
  mkdir -p "$outdir"

  # Download marketplace index
  local mkt_src="${s3prefix}/marketplace-index-${TIMESTAMP}.json"
  if aws s3 ls $(s3_args) "$mkt_src" &>/dev/null; then
    s3_download "$mkt_src" "${outdir}/marketplace-index.json"
    cp "${outdir}/marketplace-index.json" "${repo_root}/marketplace/index.json"
    cp "${outdir}/marketplace-index.json" "${repo_root}/apps/web/public/marketplace/index.json"
    log "  Marketplace index restored"
  fi

  # Restore detections archive
  local det_src="${s3prefix}/detections-${TIMESTAMP}.tar.gz"
  if aws s3 ls $(s3_args) "$det_src" &>/dev/null; then
    s3_download "$det_src" "${outdir}/detections.tar.gz"
    log "  Extracting detections archive…"
    tar -xzf "${outdir}/detections.tar.gz" -C "$repo_root"
    log "  Detections restored"
  fi

  # Restore plugin SDK packages
  for pkg in plugin-sdk-go plugin-sdk-py; do
    local pkg_src="${s3prefix}/${pkg}-${TIMESTAMP}.tar.gz"
    if aws s3 ls $(s3_args) "$pkg_src" &>/dev/null; then
      s3_download "$pkg_src" "${outdir}/${pkg}.tar.gz"
      log "  Extracting ${pkg}…"
      tar -xzf "${outdir}/${pkg}.tar.gz" -C "${repo_root}/packages"
      log "  ${pkg} restored"
    fi
  done

  log "Plugin store restore complete"
}

# ── run selected components ───────────────────────────────────────────────────
# ── Neo4j ─────────────────────────────────────────────────────────────────────
#
# Gap-closure wave 15. Neo4j, Qdrant and Redis each had a backup path
# and **no restore**, which is a backup nobody has ever proven they can
# use — and the first time anyone finds out is during the recovery.
#
# Neo4j is restored by replaying the Cypher the backup exported.
# `neo4j-admin load` is not an option here for the same reason the
# backup does not use `neo4j-admin dump`: it needs the database
# stopped, and a restore that requires downtime on a store the rest of
# the platform degrades gracefully without is a worse trade than a
# slower replay.
restore_neo4j() {
  log "--- Neo4j restore ---"
  local base="neo4j-${TIMESTAMP}.cypher.gz"
  local local_file="${RESTORE_DIR}/${base}"

  if ! fetch_artifact "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/neo4j" "$base" "$local_file"; then
    log "No Neo4j backup for ${TIMESTAMP}; skipping"
    return
  fi
  verify_artifact "$local_file" "neo4j/${base}"

  if [[ "$DRY_RUN" == "true" ]]; then
    log "[dry-run] Would replay $(gzip -dc "$local_file" | grep -c . || echo 0) Cypher statements"
    return
  fi

  # Statement at a time rather than one transaction: the export is
  # large, and a single failing MERGE must not discard the whole graph
  # restore. Failures are counted and reported rather than swallowed —
  # a restore that reports success having skipped half the graph is
  # the failure this whole wave is about.
  local applied=0 failed=0 stmt
  while IFS= read -r stmt; do
    [[ -z "$stmt" || "$stmt" == //* ]] && continue
    if curl -sf -u "${NEO4J_USER}:${NEO4J_PASSWORD}" \
         -H 'Content-Type: application/json' \
         -d "$(python3 -c 'import json,sys; print(json.dumps({"statements":[{"statement":sys.stdin.read()}]}))' <<<"$stmt")" \
         "${NEO4J_HTTP_URL}/db/neo4j/tx/commit" >/dev/null 2>&1; then
      applied=$((applied + 1))
    else
      failed=$((failed + 1))
    fi
  done < <(gzip -dc "$local_file")

  log "Neo4j: ${applied} statements applied, ${failed} failed"
  if [[ "$failed" -gt 0 ]]; then
    echo "ERROR: ${failed} Cypher statements failed; the graph is incomplete" >&2
    return 1
  fi
}

# ── Qdrant ────────────────────────────────────────────────────────────────────
#
# Restored per collection from the snapshot API. Collections are
# recreated by the upload rather than pre-created here: the vector
# dimension is immutable, and creating one with the wrong dimension
# would make the restore fail in a way that looks like corrupt data.
restore_qdrant() {
  log "--- Qdrant restore ---"
  local prefix="${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/qdrant"
  # shellcheck disable=SC2046
  local collections
  collections=$(aws s3 ls $(s3_args) "${prefix}/" 2>/dev/null | awk '{print $2}' | sed 's|/$||' || echo "")

  if [[ -z "$collections" ]]; then
    log "No Qdrant backup directories found; skipping"
    return
  fi

  local restored=0
  while IFS= read -r collection; do
    [[ -z "$collection" ]] && continue
    local base="${collection}-${TIMESTAMP}.snapshot"
    local local_file="${RESTORE_DIR}/qdrant-${base}"

    if ! fetch_artifact "${prefix}/${collection}" "$base" "$local_file"; then
      log "No snapshot for collection ${collection} at ${TIMESTAMP}; skipping"
      continue
    fi
    verify_artifact "$local_file" "qdrant/${collection}/${base}"

    if [[ "$DRY_RUN" == "true" ]]; then
      log "[dry-run] Would upload snapshot to collection ${collection}"
      continue
    fi

    if curl -sf -X POST \
         -H 'api-key: '"${QDRANT_API_KEY:-}" \
         -F "snapshot=@${local_file}" \
         "${QDRANT_URL}/collections/${collection}/snapshots/upload?priority=snapshot" >/dev/null; then
      restored=$((restored + 1))
    else
      echo "ERROR: Qdrant collection ${collection} failed to restore" >&2
      return 1
    fi
  done <<< "$collections"

  log "Qdrant: ${restored} collection(s) restored"
}

# ── Redis ─────────────────────────────────────────────────────────────────────
#
# Redis holds cache, the OIDC state store and rate-limit counters —
# all of which the platform rebuilds. So this restores the RDB only
# when explicitly asked, and the default guidance is to start empty:
# replacing a running cache with an hours-old one serves stale answers
# with full confidence, which is worse than a cold start.
restore_redis() {
  log "--- Redis restore ---"
  local base="redis-${TIMESTAMP}.rdb"
  local local_file="${RESTORE_DIR}/${base}"

  if ! fetch_artifact "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/redis" "$base" "$local_file"; then
    log "No Redis backup for ${TIMESTAMP}; skipping"
    return
  fi
  verify_artifact "$local_file" "redis/${base}"

  if [[ "${REDIS_RESTORE_CONFIRM:-}" != "yes" ]]; then
    log "Redis holds only rebuildable state (cache, OIDC states, rate limits)."
    log "A restored cache serves stale answers with full confidence, which is worse"
    log "than a cold start. Set REDIS_RESTORE_CONFIRM=yes to restore it anyway."
    return
  fi

  if [[ "$DRY_RUN" == "true" ]]; then
    log "[dry-run] Would load ${base} into ${REDIS_URL}"
    return
  fi

  # The RDB has to be placed where the server reads it, which means
  # the restore runs against the container rather than the protocol.
  if ! command -v docker &>/dev/null; then
    echo "ERROR: Redis restore needs docker to place the RDB; none found" >&2
    return 1
  fi
  docker cp "$local_file" "${REDIS_CONTAINER:-redis}:/data/dump.rdb"
  docker restart "${REDIS_CONTAINER:-redis}" >/dev/null
  log "Redis: RDB restored and the server restarted"
}

case "$COMPONENT" in
  postgres)   restore_postgres ;;
  clickhouse) restore_clickhouse ;;
  plugins)    restore_plugins ;;
  neo4j)      restore_neo4j ;;
  qdrant)     restore_qdrant ;;
  redis)      restore_redis ;;
  all)
    restore_postgres
    restore_clickhouse
    restore_plugins
    # Added in wave 15. Each had a backup path and no restore, which
    # is a backup nobody has proven they can use.
    restore_neo4j
    restore_qdrant
    restore_redis
    ;;
  *)
    echo "Unknown component: $COMPONENT (choose: postgres|clickhouse|plugins|neo4j|qdrant|redis|all)" >&2
    exit 1
    ;;
esac

log "=== Restore completed successfully ==="