Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

12.5 KB
Raw Blame
# shellcheck shell=bash
# Profile-persistence round-trip helpers (P3a). Verifies localStorage + cookie
# survive a container stop/restart when /data is backed by a Docker volume.
# Mutates globals: CONTAINER_STARTED, FIXTURES_STARTED, HOST_PORT.

setup_profile_volume() {
  local name="$1"
  [ -n "$name" ] || fail "setup_profile_volume: name required"
  # Remove any stale volume so we start from an empty profile.
  if docker volume inspect "$name" >/dev/null 2>&1; then
    docker volume rm "$name" >/dev/null 2>&1 || true
  fi
  docker volume create "$name" >/dev/null || fail "failed to create docker volume $name"
  echo "$name"
}

cleanup_profile_volume() {
  local name="$1"
  [ -n "$name" ] || return 0
  if ! docker volume inspect "$name" >/dev/null 2>&1; then
    return 0
  fi
  # Tolerate "volume in use" so a failed leg doesn't mask the original error.
  if ! docker volume rm "$name" >/dev/null 2>&1; then
    echo "  warn: docker volume rm $name failed (likely still in use); leaving for manual cleanup" >&2
  fi
}

_read_marker_from_managed_instance() {
  local host_port="$1"
  local token="$2"
  local fixtures_url="$3"

  local nav_body
  nav_body="$(curl -fsS -X POST \
    -H "Authorization: Bearer ${token}" \
    -H "Content-Type: application/json" \
    -d "{\"url\":\"${fixtures_url}/persistence-marker.html\",\"waitFor\":\"selector\",\"waitSelector\":\"#marker\"}" \
    "http://127.0.0.1:${host_port}/navigate" 2>&1)" \
    || fail "navigate to persistence-marker.html failed: $nav_body"

  local tab_id
  tab_id="$(echo "$nav_body" | jq -r '.tabId // empty')"
  [ -n "$tab_id" ] || fail "navigate response missing tabId: $nav_body"

  local html_body
  html_body="$(curl -fsS \
    -H "Authorization: Bearer ${token}" \
    "http://127.0.0.1:${host_port}/tabs/${tab_id}/html?selector=%23marker" 2>&1)" \
    || fail "GET /tabs/${tab_id}/html for #marker failed: $html_body"

  local raw_html
  raw_html="$(echo "$html_body" | jq -r '.html // empty')"
  [ -n "$raw_html" ] || fail "marker html was empty: $html_body"

  local marker
  marker="$(printf '%s' "$raw_html" | sed -E 's/<[^>]+>//g' | tr -d '[:space:]')"
  [ -n "$marker" ] || fail "marker text was empty after strip: $raw_html"
  [ "$marker" != "pending" ] || fail "marker text was still 'pending' (fixture JS did not run): $raw_html"

  echo "$marker"
}

assert_persistence_round_trip() {
  local provider="$1"
  local image="$2"
  local name="$3"
  local config_path="$4"
  local fixtures_host="$5"
  local fixtures_port="$6"
  local fixtures_url="$7"
  local host_fixtures_url="$8"
  local token="$9"
  local root="${10}"
  local profile_volume="${11}"

  [ -n "$profile_volume" ] || fail "assert_persistence_round_trip requires a profile volume"

  echo "→ Phase 1: capture marker (fresh volume, first boot)"
  local first_marker
  first_marker="$(_read_marker_from_managed_instance "$HOST_PORT" "$token" "$fixtures_url")"
  echo "  first-boot marker: $first_marker"

  echo "→ Phase 2: stop container (graceful) to flush profile to volume $profile_volume"
  # Must stop the managed instance via the API first: only the full graceful
  # path (Browser.close + wait-for-exit) makes Chrome flush localStorage to
  # leveldb. `docker stop` alone is too fast.
  local inst_id
  inst_id="$(curl -fsS -H "Authorization: Bearer ${token}" \
    "http://127.0.0.1:${HOST_PORT}/instances" 2>/dev/null \
    | jq -r '.[]? | select(.status=="running") | .id' | head -1)"
  if [ -n "$inst_id" ]; then
    echo "  stopping managed instance $inst_id"
    curl -fsS -X POST -H "Authorization: Bearer ${token}" \
      "http://127.0.0.1:${HOST_PORT}/instances/${inst_id}/stop" >/dev/null 2>&1 || true
    local i
    for i in $(seq 1 30); do
      local still
      still="$(curl -fsS -H "Authorization: Bearer ${token}" \
        "http://127.0.0.1:${HOST_PORT}/instances/${inst_id}" 2>/dev/null \
        | jq -r '.status // "gone"')"
      if [ "$still" != "running" ]; then
        break
      fi
      sleep 1
    done
  fi
  # Extend grace period beyond the 10s default in case I/O is still in flight.
  docker stop --time 30 "$name" >/dev/null || fail "docker stop $name failed"
  teardown_container "$name"
  CONTAINER_STARTED=0
  FIXTURES_STARTED=0

  echo "→ Phase 3: restart container with identical name + config + token + volume"
  PROFILE_VOLUME="$profile_volume" start_provider_container \
    "$name" "$image" "$config_path" "$fixtures_host" "$fixtures_port" "$root"
  HOST_PORT="$(resolve_host_port "$name")"
  echo "  Container $name restarted — PinchTab on host port $HOST_PORT"

  start_fixture_server "$name" "$fixtures_port"
  wait_fixtures_ready "$name" "$fixtures_url" "$host_fixtures_url"

  echo "  Waiting for PinchTab health on :${HOST_PORT}..."
  wait_for_health "$HOST_PORT" "$token"
  echo "  Waiting for managed ${provider} instance (post-restart)..."
  wait_for_instance_running "$HOST_PORT" "$token"

  echo "→ Phase 4: re-read marker and assert equality"
  local second_marker
  second_marker="$(_read_marker_from_managed_instance "$HOST_PORT" "$token" "$fixtures_url")"
  echo "  post-restart marker: $second_marker"

  if [ "$first_marker" != "$second_marker" ]; then
    fail "profile persistence round-trip failed: first=${first_marker} second=${second_marker}"
  fi
  echo "  marker matches across restart — localStorage + cookie persisted via $profile_volume"
}

# Stale profile-lock recovery (P3b) helpers. After the P3a round-trip,
# plants SingletonLock/Socket/Cookie referencing an unused PID and asserts
# PinchTab recovers. Plant container chowns to 1000:1000 so the non-root
# pinchtab user can later read+delete the files.

_pick_unused_pid() {
  local pid=99999
  local guard=0
  while kill -0 "$pid" 2>/dev/null; do
    pid=$((RANDOM + 10000))
    guard=$((guard + 1))
    if [ "$guard" -gt 100 ]; then
      fail "could not find an unused PID after 100 attempts"
    fi
  done
  echo "$pid"
}

# Plants the three Singleton* files inside the volume. profile_subpath is
# volume-relative (no leading slash), e.g. "profiles/cloak-docker-smoke".
plant_stale_lock() {
  local volume_name="$1"
  local profile_subpath="$2"
  local fake_pid="$3"

  [ -n "$volume_name" ]    || fail "plant_stale_lock: volume_name required"
  [ -n "$profile_subpath" ] || fail "plant_stale_lock: profile_subpath required"
  [ -n "$fake_pid" ]       || fail "plant_stale_lock: fake_pid required"

  # Short-lived root container so we can chown to 1000:1000; the PinchTab
  # container itself never runs as root.
  docker run --rm \
    --user 0:0 \
    -v "${volume_name}:/data:rw" \
    -e "PROFILE_SUBPATH=${profile_subpath}" \
    -e "FAKE_PID=${fake_pid}" \
    alpine:3.20 sh -lc '
      set -eu
      profile_dir="/data/${PROFILE_SUBPATH}"
      if [ ! -d "$profile_dir" ]; then
        echo "plant_stale_lock: profile dir $profile_dir does not exist; current /data layout:" >&2
        find /data -maxdepth 3 -type d -printf "%p\n" 2>/dev/null | sed -n "1,40p" >&2
        exit 1
      fi
      # Sentinel "STALE_PLANT_" lets assert_lock_files_gone distinguish
      # planted files from fresh ones Chrome re-creates after recovery.
      content="STALE_PLANT_${FAKE_PID}_$(date +%s)"
      for f in SingletonLock SingletonSocket SingletonCookie; do
        printf "%s" "$content" > "$profile_dir/$f"
      done
      chown -R 1000:1000 "$profile_dir"
      echo "planted stale lock files in $profile_dir (pid=$FAKE_PID)"
      ls -la "$profile_dir" | sed -n "1,20p"
    ' >&2 || fail "plant_stale_lock: helper container failed"
}

# Asserts no planted sentinel files remain. Chrome may re-create fresh
# Singleton* files post-recovery; only those still carrying the
# "STALE_PLANT_" sentinel indicate recovery missed them.
assert_lock_files_gone() {
  local volume_name="$1"
  local profile_subpath="$2"

  [ -n "$volume_name" ]    || fail "assert_lock_files_gone: volume_name required"
  [ -n "$profile_subpath" ] || fail "assert_lock_files_gone: profile_subpath required"

  local out
  out="$(docker run --rm \
    -v "${volume_name}:/data:ro" \
    -e "PROFILE_SUBPATH=${profile_subpath}" \
    alpine:3.20 sh -lc '
      set -eu
      profile_dir="/data/${PROFILE_SUBPATH}"
      if [ ! -d "$profile_dir" ]; then
        echo "MISSING_DIR"
        exit 0
      fi
      survivors=""
      for f in SingletonLock SingletonSocket SingletonCookie; do
        if [ -e "$profile_dir/$f" ] || [ -L "$profile_dir/$f" ]; then
          # Skip symlinks: Chrome re-creates SingletonLock as a symlink to
          # its own live PID — healthy state, not a stale plant.
          if [ -L "$profile_dir/$f" ]; then
            continue
          fi
          if grep -q "STALE_PLANT_" "$profile_dir/$f" 2>/dev/null; then
            survivors="${survivors}${f} "
          fi
        fi
      done
      if [ -n "$survivors" ]; then
        echo "SURVIVORS: $survivors"
        ls -la "$profile_dir" | sed -n "1,40p"
        exit 1
      fi
      echo "OK"
    ' 2>&1)" || fail "assert_lock_files_gone: stale singleton files remain in ${profile_subpath}: $out"

  echo "  $out"
}

# Full lock-recovery round-trip on top of P3a: stops the container, plants
# stale Singleton* files, restarts, and asserts health, preserved marker,
# and that the planted sentinel files were cleared.
assert_lock_recovery_round_trip() {
  local provider="$1"
  local image="$2"
  local name="$3"
  local config_path="$4"
  local fixtures_host="$5"
  local fixtures_port="$6"
  local fixtures_url="$7"
  local host_fixtures_url="$8"
  local token="$9"
  local root="${10}"
  local profile_volume="${11}"
  local profile_subpath="${12}"

  [ -n "$profile_volume" ]  || fail "assert_lock_recovery_round_trip requires a profile volume"
  [ -n "$profile_subpath" ] || fail "assert_lock_recovery_round_trip requires a profile subpath"

  assert_persistence_round_trip \
    "$provider" "$image" "$name" "$config_path" \
    "$fixtures_host" "$fixtures_port" \
    "$fixtures_url" "$host_fixtures_url" \
    "$token" "$root" "$profile_volume"

  local seeded_marker
  seeded_marker="$(_read_marker_from_managed_instance "$HOST_PORT" "$token" "$fixtures_url")"
  echo "→ Phase L1: captured post-P3a marker for recovery leg: $seeded_marker"

  echo "→ Phase L2: stop container to plant stale lock on volume ${profile_volume}"
  local inst_id
  inst_id="$(curl -fsS -H "Authorization: Bearer ${token}" \
    "http://127.0.0.1:${HOST_PORT}/instances" 2>/dev/null \
    | jq -r '.[]? | select(.status=="running") | .id' | head -1)"
  if [ -n "$inst_id" ]; then
    curl -fsS -X POST -H "Authorization: Bearer ${token}" \
      "http://127.0.0.1:${HOST_PORT}/instances/${inst_id}/stop" >/dev/null 2>&1 || true
    local i
    for i in $(seq 1 30); do
      local still
      still="$(curl -fsS -H "Authorization: Bearer ${token}" \
        "http://127.0.0.1:${HOST_PORT}/instances/${inst_id}" 2>/dev/null \
        | jq -r '.status // "gone"')"
      [ "$still" != "running" ] && break
      sleep 1
    done
  fi
  docker stop --time 30 "$name" >/dev/null || fail "docker stop $name failed"
  teardown_container "$name"
  CONTAINER_STARTED=0
  FIXTURES_STARTED=0

  echo "→ Phase L3: plant stale singleton files in ${profile_subpath}"
  local fake_pid
  fake_pid="$(_pick_unused_pid)"
  echo "  using fake pid=${fake_pid} (verified not alive on host)"
  plant_stale_lock "$profile_volume" "$profile_subpath" "$fake_pid"

  echo "→ Phase L4: restart container — PinchTab must detect + clear stale lock"
  PROFILE_VOLUME="$profile_volume" start_provider_container \
    "$name" "$image" "$config_path" "$fixtures_host" "$fixtures_port" "$root"
  HOST_PORT="$(resolve_host_port "$name")"
  echo "  Container $name restarted — PinchTab on host port $HOST_PORT"

  start_fixture_server "$name" "$fixtures_port"
  wait_fixtures_ready "$name" "$fixtures_url" "$host_fixtures_url"

  echo "  Waiting for PinchTab health on :${HOST_PORT} (post stale-lock plant)..."
  wait_for_health "$HOST_PORT" "$token"
  echo "  Waiting for managed ${provider} instance (post-recovery)..."
  wait_for_instance_running "$HOST_PORT" "$token"

  echo "→ Phase L5: re-read marker and assert it still matches seeded UUID"
  local post_recovery_marker
  post_recovery_marker="$(_read_marker_from_managed_instance "$HOST_PORT" "$token" "$fixtures_url")"
  echo "  post-recovery marker: $post_recovery_marker"
  if [ "$seeded_marker" != "$post_recovery_marker" ]; then
    fail "lock-recovery round-trip failed: seeded=${seeded_marker} post=${post_recovery_marker}"
  fi
  echo "  marker survives stale-lock recovery — user state intact"

  echo "→ Phase L6: assert all three singleton files cleared by recovery"
  assert_lock_files_gone "$profile_volume" "$profile_subpath"
  echo "  recovery confirmed: SingletonLock/SingletonSocket/SingletonCookie removed"
}