veza/scripts/bootstrap/verify-r720.sh

#!/usr/bin/env bash
# verify-r720.sh — read-only checks on the R720 itself.
#
# Run as root :
#   sudo bash scripts/bootstrap/verify-r720.sh
#
# Symmetric to verify-local.sh — exit code = number of failures.

set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
. "$SCRIPT_DIR/lib.sh"

[[ $EUID -ne 0 ]] && warn "running without root — some checks may fail (incus list, ZFS)"

declare -i PASS=0 FAIL=0
check() {
    local name=$1 cmd=$2
    if eval "$cmd" >/dev/null 2>&1; then ok "$name"; PASS+=1; else err "$name"; FAIL+=1; fi
}
check_with_hint() {
    local name=$1 cmd=$2 hint=$3
    if eval "$cmd" >/dev/null 2>&1; then ok "$name"; PASS+=1
    else err "$name"; printf >&2 '         %shint:%s %s\n' "$_YELLOW" "$_RESET" "$hint"; FAIL+=1
    fi
}

section "Host prerequisites"
check "incus binary"             "command -v incus"
check "ansible binary"           "command -v ansible"
check "zfs binary"               "command -v zfs"
check "incus daemon reachable"   "incus list"

section "Incus profiles"
check_with_hint "profile veza-app" "incus profile show veza-app" \
    "rerun bootstrap-r720.sh phase 4"
check_with_hint "profile veza-data" "incus profile show veza-data" \
    "rerun bootstrap-r720.sh phase 4"

section "Incus networks"
check_with_hint "net-veza network exists" "incus network show net-veza" \
    "incus network create net-veza ipv4.address=10.0.20.1/24 ipv4.nat=true"

section "Forgejo"
check "forgejo container exists"      "incus info forgejo"
check "forgejo container RUNNING"     "incus list forgejo -f csv -c s 2>/dev/null | grep -q RUNNING"
check "forgejo HTTP responds"          "curl -ksSf -o /dev/null --max-time 5 https://10.0.20.105:3000/api/v1/version || curl -sSf -o /dev/null --max-time 5 http://10.0.20.105:3000/api/v1/version"

section "forgejo-runner"
check "runner container exists"        "incus info forgejo-runner"
check "runner container RUNNING"       "incus list forgejo-runner -f csv -c s 2>/dev/null | grep -q RUNNING"
check_with_hint "incus-socket device attached" \
    "incus config device show forgejo-runner | grep -q '^incus-socket:'" \
    "rerun bootstrap-r720.sh phase 4"
check_with_hint "security.nesting=true" \
    "[[ \$(incus config get forgejo-runner security.nesting) == true ]]" \
    "incus config set forgejo-runner security.nesting=true && incus restart forgejo-runner"
check_with_hint "incus binary in runner" \
    "incus exec forgejo-runner -- test -x /usr/local/bin/incus" \
    "rerun bootstrap-r720.sh phase 4"
check_with_hint "runner has 'incus' label" \
    "incus exec forgejo-runner -- bash -c 'for f in /etc/forgejo-runner/.runner /var/lib/forgejo-runner/.runner /opt/forgejo-runner/.runner; do [[ -f \$f ]] && grep -q incus \$f && exit 0; done; exit 1'" \
    "rerun bootstrap-r720.sh phase 4 (will re-register)"
check_with_hint "runner systemd unit active" \
    "incus exec forgejo-runner -- bash -c 'systemctl is-active forgejo-runner.service 2>/dev/null || systemctl is-active act_runner.service'" \
    "incus exec forgejo-runner -- journalctl -u forgejo-runner -n 50"

section "Edge HAProxy (post-haproxy.yml run)"
if incus info veza-haproxy >/dev/null 2>&1; then
    check "veza-haproxy RUNNING" "incus list veza-haproxy -f csv -c s | grep -q RUNNING"
    check_with_hint "haproxy systemd unit active" \
        "incus exec veza-haproxy -- systemctl is-active haproxy" \
        "incus exec veza-haproxy -- journalctl -u haproxy -n 50"
    check_with_hint "haproxy.cfg validates" \
        "incus exec veza-haproxy -- haproxy -f /etc/haproxy/haproxy.cfg -c -q" \
        "rerun playbooks/haproxy.yml — config syntax error"
    check_with_hint "Let's Encrypt cert dir has at least 1 .pem" \
        "incus exec veza-haproxy -- bash -c 'ls /usr/local/etc/tls/haproxy/*.pem 2>/dev/null | grep -q .'" \
        "verify port 80 reachable from Internet ; rerun playbooks/haproxy.yml"
else
    warn "veza-haproxy doesn't exist yet — run bootstrap-r720.sh phase 4"
fi

section "ZFS"
check "rpool exists" "zpool list rpool"

section "State file"
if [[ -f "$TALAS_STATE_FILE" ]]; then
    info "phases recorded :"
    sed 's/^/  /' "$TALAS_STATE_FILE"
else
    warn "no state file at $TALAS_STATE_FILE — bootstrap-r720.sh hasn't run yet"
fi

section "Result"
if (( FAIL == 0 )); then
    ok "$PASS / $((PASS + FAIL)) checks passed"
    exit 0
else
    err "$FAIL FAIL out of $((PASS + FAIL))  ($PASS passed)"
    exit 1
fi