veza/scripts/staging-stability-check.sh

#!/usr/bin/env bash
# v0.14.0 TASK-STAG-004: Stability validation script
# Monitors staging for N minutes, checking 5xx rate and availability.
#
# Usage:
#   STAGING_API_URL=https://staging.veza.app/api/v1 DURATION_MINUTES=10 bash scripts/staging-stability-check.sh
#
# Environment:
#   STAGING_API_URL   — Base API URL (default: http://localhost:8080/api/v1)
#   DURATION_MINUTES  — How long to monitor (default: 10)
#   INTERVAL_SECONDS  — Seconds between checks (default: 10)
#   MAX_5XX_RATE      — Maximum 5xx rate as decimal (default: 0.001 = 0.1%)

set -euo pipefail

API_URL="${STAGING_API_URL:-http://localhost:8080/api/v1}"
DURATION="${DURATION_MINUTES:-10}"
INTERVAL="${INTERVAL_SECONDS:-10}"
MAX_RATE="${MAX_5XX_RATE:-0.001}"

TOTAL_REQUESTS=0
TOTAL_5XX=0
TOTAL_ERRORS=0
START_TIME=$(date +%s)
END_TIME=$((START_TIME + DURATION * 60))

ENDPOINTS=(
  "/health"
  "/healthz"
  "/readyz"
  "/health/deep"
  "/tracks?page=1&limit=5"
  "/search?q=test&limit=5"
)

echo "═══════════════════════════════════════════"
echo "  Staging Stability Check"
echo "  URL: ${API_URL}"
echo "  Duration: ${DURATION} minutes"
echo "  Interval: ${INTERVAL}s"
echo "  Max 5xx rate: $(echo "${MAX_RATE} * 100" | bc)%"
echo "═══════════════════════════════════════════"
echo ""

while [ "$(date +%s)" -lt "$END_TIME" ]; do
  for endpoint in "${ENDPOINTS[@]}"; do
    HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "${API_URL}${endpoint}" 2>/dev/null || echo "000")
    TOTAL_REQUESTS=$((TOTAL_REQUESTS + 1))

    if [ "$HTTP_CODE" = "000" ]; then
      TOTAL_ERRORS=$((TOTAL_ERRORS + 1))
      echo "[$(date '+%H:%M:%S')] ERROR: ${endpoint} — connection failed"
    elif [ "$HTTP_CODE" -ge 500 ]; then
      TOTAL_5XX=$((TOTAL_5XX + 1))
      echo "[$(date '+%H:%M:%S')] 5XX: ${endpoint} — HTTP ${HTTP_CODE}"
    fi
  done

  ELAPSED=$(( $(date +%s) - START_TIME ))
  REMAINING=$(( END_TIME - $(date +%s) ))
  if [ "$TOTAL_REQUESTS" -gt 0 ]; then
    CURRENT_RATE=$(echo "scale=6; $TOTAL_5XX / $TOTAL_REQUESTS" | bc)
    printf "\r[%ds/%ds] Requests: %d | 5xx: %d | Errors: %d | 5xx rate: %s" \
      "$ELAPSED" "$((DURATION * 60))" "$TOTAL_REQUESTS" "$TOTAL_5XX" "$TOTAL_ERRORS" "$CURRENT_RATE"
  fi

  sleep "$INTERVAL"
done

echo ""
echo ""

# Calculate final rate
if [ "$TOTAL_REQUESTS" -gt 0 ]; then
  FINAL_RATE=$(echo "scale=6; $TOTAL_5XX / $TOTAL_REQUESTS" | bc)
  FINAL_RATE_PCT=$(echo "scale=3; $FINAL_RATE * 100" | bc)
  ERROR_RATE=$(echo "scale=6; $TOTAL_ERRORS / $TOTAL_REQUESTS" | bc)
  ERROR_RATE_PCT=$(echo "scale=3; $ERROR_RATE * 100" | bc)
else
  FINAL_RATE="0"
  FINAL_RATE_PCT="0"
  ERROR_RATE="0"
  ERROR_RATE_PCT="0"
fi

# Generate report
cat > stability-report.json <<EOF
{
  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "duration_minutes": ${DURATION},
  "total_requests": ${TOTAL_REQUESTS},
  "total_5xx": ${TOTAL_5XX},
  "total_connection_errors": ${TOTAL_ERRORS},
  "rate_5xx": ${FINAL_RATE},
  "rate_5xx_percent": "${FINAL_RATE_PCT}%",
  "rate_errors": ${ERROR_RATE},
  "max_5xx_rate": ${MAX_RATE},
  "passed": $(echo "${FINAL_RATE} <= ${MAX_RATE}" | bc -l)
}
EOF

echo "═══════════════════════════════════════════"
echo "  Stability Check Results"
echo "═══════════════════════════════════════════"
echo "  Duration:         ${DURATION} minutes"
echo "  Total requests:   ${TOTAL_REQUESTS}"
echo "  5xx responses:    ${TOTAL_5XX} (${FINAL_RATE_PCT}%)"
echo "  Connection errors: ${TOTAL_ERRORS} (${ERROR_RATE_PCT}%)"
echo "  Max allowed 5xx:  $(echo "${MAX_RATE} * 100" | bc)%"

PASS=$(echo "${FINAL_RATE} <= ${MAX_RATE}" | bc -l)
if [ "$PASS" -eq 1 ] && [ "$TOTAL_ERRORS" -eq 0 ]; then
  echo "  Result:           ✅ PASS"
  echo "═══════════════════════════════════════════"
  exit 0
else
  echo "  Result:           ❌ FAIL"
  echo "═══════════════════════════════════════════"
  exit 1
fi