veza/infra/ansible/tests/test_redis_failover.sh

#!/usr/bin/env bash
# test_redis_failover.sh — validate Sentinel promotes a replica to master
# in < 30s when the current master dies.
#
# Run on the Incus host that owns the redis-1/2/3 containers (typically
# the lab R720). Assumes the redis_sentinel playbook has been applied
# so the formation is healthy at script start — bails early otherwise.
#
# v1.0.9 W3 Day 11 — acceptance for the verification gate :
#   "kill Redis master, verify promotion automatique d'un replica en < 30s".
#
# Usage:
#   REDIS_PASS=... SENTINEL_PASS=... bash infra/ansible/tests/test_redis_failover.sh
#
# Exit codes:
#   0  — promotion completed in < 30s (acceptance met)
#   1  — formation not healthy at start
#   2  — promotion did not complete within 30s
#   3  — required tool missing on the host
set -euo pipefail

REDIS_CONTAINERS=(redis-1 redis-2 redis-3)
MASTER_NAME=${MASTER_NAME:-veza-master}
RTO_TARGET_SECONDS=${RTO_TARGET_SECONDS:-30}
SENTINEL_PORT=${SENTINEL_PORT:-26379}
REDIS_PORT=${REDIS_PORT:-6379}
REDIS_PASS=${REDIS_PASS:-?}
SENTINEL_PASS=${SENTINEL_PASS:-?}

log()  { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" >&2; }
fail() { log "FAIL: $*"; exit "${2:-2}"; }

require() {
  command -v "$1" >/dev/null 2>&1 || fail "required tool missing on host: $1" 3
}

require incus
require date

if [ "$REDIS_PASS" = "?" ] || [ "$SENTINEL_PASS" = "?" ]; then
  fail "REDIS_PASS and SENTINEL_PASS env vars are required (read them from the vault before invoking)" 3
fi

# Helper : ask any sentinel which host:port is currently master.
get_master_addr() {
  local ct=$1
  incus exec "$ct" -- redis-cli -p "$SENTINEL_PORT" -a "$SENTINEL_PASS" --no-auth-warning \
    SENTINEL get-master-addr-by-name "$MASTER_NAME" 2>/dev/null | tr '\n' ' '
}

# -----------------------------------------------------------------------------
# 0. Sanity — formation must be healthy at start.
# -----------------------------------------------------------------------------
log "step 0: pre-flight — Sentinel reports current master"
master_addr_before=$(get_master_addr "${REDIS_CONTAINERS[0]}")
if [ -z "$master_addr_before" ]; then
  fail "no master visible in Sentinel — refusing to test from a degraded baseline" 1
fi
log "current master (host port) : $master_addr_before"

# Resolve which container hosts the current master so we know whom to kill.
master_host=$(echo "$master_addr_before" | awk '{print $1}')
master_container=""
for ct in "${REDIS_CONTAINERS[@]}"; do
  ip=$(incus list "$ct" -c 4 -f csv 2>/dev/null | head -1 | awk '{print $1}' | tr -d ',')
  # accept either the .lxd hostname or the IP. The .lxd suffix is what
  # sentinel.conf hands out ; the IP is what `incus list` shows.
  if [ "$ct.lxd" = "$master_host" ] || [ "$ip" = "$master_host" ]; then
    master_container=$ct
    break
  fi
done

if [ -z "$master_container" ]; then
  fail "could not map master host '$master_host' to a known container" 1
fi
log "master container resolved to: $master_container"

# -----------------------------------------------------------------------------
# 1. Kill master container — simulates hardware/process death.
# -----------------------------------------------------------------------------
log "step 1: stopping $master_container — start timer"
t0=$(date +%s)
incus stop --force "$master_container"

# -----------------------------------------------------------------------------
# 2. Poll surviving sentinels until they announce a new master.
# -----------------------------------------------------------------------------
log "step 2: polling sentinels for new master (target RTO ${RTO_TARGET_SECONDS}s)"
deadline=$((t0 + RTO_TARGET_SECONDS))
promoted=0
new_master=""
while [ "$(date +%s)" -lt "$deadline" ]; do
  for ct in "${REDIS_CONTAINERS[@]}"; do
    if [ "$ct" = "$master_container" ]; then continue; fi
    addr=$(get_master_addr "$ct")
    if [ -n "$addr" ] && [ "$addr" != "$master_addr_before" ]; then
      new_master=$addr
      promoted=1
      break 2
    fi
  done
  sleep 1
done

t1=$(date +%s)
elapsed=$((t1 - t0))

# -----------------------------------------------------------------------------
# 3. Restart the killed container so it can rejoin as replica for the
#    next run.
# -----------------------------------------------------------------------------
log "step 3: restarting $master_container (will rejoin as replica once it catches up)"
incus start "$master_container" || true

# -----------------------------------------------------------------------------
# 4. Verdict.
# -----------------------------------------------------------------------------
if [ "$promoted" -eq 1 ] && [ "$elapsed" -le "$RTO_TARGET_SECONDS" ]; then
  log "PASS: master flipped from '$master_addr_before' to '$new_master' in ${elapsed}s (target ${RTO_TARGET_SECONDS}s)"
  exit 0
fi

log "final Sentinel view:"
for ct in "${REDIS_CONTAINERS[@]}"; do
  if [ "$ct" = "$master_container" ]; then continue; fi
  echo "    $ct: $(get_master_addr "$ct")" >&2
done
fail "no replica promoted within ${RTO_TARGET_SECONDS}s (elapsed ${elapsed}s, promoted=${promoted})"