veza/scripts/bootstrap/verify-remote.sh

#!/usr/bin/env bash
# verify-remote.sh — read-only checks of R720 state (Incus profiles,
# runner labels, container reachability, certs). Run on the R720 itself
# (locally or via `ssh r720 verify-remote.sh`).
#
# Exit 0 if everything passes ; non-zero with a count of failures.

set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib.sh
. "$SCRIPT_DIR/lib.sh"

: "${FORGEJO_API_URL:=https://forgejo.talas.group}"

declare -i PASS=0 FAIL=0

check() {
    local name=$1 cmd=$2
    if eval "$cmd" >/dev/null 2>&1; then
        ok "$name"
        PASS+=1
    else
        err "$name"
        FAIL+=1
    fi
}

check_with_hint() {
    local name=$1 cmd=$2 hint=$3
    if eval "$cmd" >/dev/null 2>&1; then
        ok "$name"
        PASS+=1
    else
        err "$name"
        printf >&2 '         %shint:%s %s\n' "$_YELLOW" "$_RESET" "$hint"
        FAIL+=1
    fi
}

section "R720 prerequisites"
check "incus available"      "command -v incus"
check "zfs available"        "command -v zfs"
check "incus list works"     "incus list"

section "Incus profiles"
for p in veza-app veza-data veza-net; do
    check_with_hint "profile $p exists" \
        "incus profile show $p" \
        "run scripts/bootstrap/bootstrap-remote.sh as root"
done

section "Forgejo container"
check "container 'forgejo' exists" "incus info forgejo"
check "container 'forgejo' RUNNING" \
    "incus list forgejo -f csv -c s 2>/dev/null | grep -q RUNNING"
check_with_hint "Forgejo HTTP responds on :3000" \
    "curl -ksSf -o /dev/null --max-time 5 http://10.0.20.105:3000/ || curl -ksSf -o /dev/null --max-time 5 https://10.0.20.105:3000/" \
    "incus exec forgejo -- systemctl status forgejo"

section "Forgejo runner"
check "container 'forgejo-runner' exists" "incus info forgejo-runner"
check "container 'forgejo-runner' RUNNING" \
    "incus list forgejo-runner -f csv -c s 2>/dev/null | grep -q RUNNING"
check_with_hint "incus-socket device attached" \
    "incus config device show forgejo-runner | grep -q '^incus-socket:'" \
    "PHASE=2 sudo bash scripts/bootstrap/bootstrap-remote.sh"
check_with_hint "security.nesting=true" \
    "[[ \$(incus config get forgejo-runner security.nesting) == true ]]" \
    "incus config set forgejo-runner security.nesting=true && incus restart forgejo-runner"
check_with_hint "incus-client installed in runner" \
    "incus exec forgejo-runner -- command -v incus" \
    "incus exec forgejo-runner -- apt install -y incus-client"
check_with_hint "runner can incus list (socket reachable)" \
    "incus exec forgejo-runner -- incus list" \
    "verify the unix-socket disk device + nesting"
check_with_hint "runner config has 'incus' label" \
    "incus exec forgejo-runner -- bash -c 'for f in /etc/forgejo-runner/.runner /var/lib/forgejo-runner/.runner /opt/forgejo-runner/.runner ; do [[ -f \$f ]] && grep -q incus \$f && exit 0 ; done ; exit 1'" \
    "PHASE=3 sudo bash scripts/bootstrap/bootstrap-remote.sh"
check_with_hint "runner systemd unit active" \
    "incus exec forgejo-runner -- bash -c 'systemctl is-active forgejo-runner.service 2>/dev/null || systemctl is-active act_runner.service'" \
    "incus exec forgejo-runner -- journalctl -u forgejo-runner -n 50"

section "Edge HAProxy (only after running playbooks/haproxy.yml)"
if incus info veza-haproxy >/dev/null 2>&1; then
    check "container 'veza-haproxy' RUNNING" \
        "incus list veza-haproxy -f csv -c s | grep -q RUNNING"
    check_with_hint "haproxy systemd unit active" \
        "incus exec veza-haproxy -- systemctl is-active haproxy" \
        "incus exec veza-haproxy -- journalctl -u haproxy -n 50"
    check_with_hint "haproxy.cfg present" \
        "incus exec veza-haproxy -- test -f /etc/haproxy/haproxy.cfg" \
        "ansible-playbook -i inventory/staging.yml playbooks/haproxy.yml"
    check_with_hint "haproxy.cfg passes self-validation" \
        "incus exec veza-haproxy -- haproxy -f /etc/haproxy/haproxy.cfg -c -q" \
        "config syntax error — re-run ansible-playbook to re-render"
    check_with_hint "Let's Encrypt cert dir has at least 1 .pem" \
        "incus exec veza-haproxy -- bash -c 'ls /usr/local/etc/tls/haproxy/*.pem 2>/dev/null | wc -l | grep -q -E \"^[1-9]\"'" \
        "rerun ansible-playbook ; verify port 80 reachable from Internet for HTTP-01"
else
    warn "container 'veza-haproxy' does not exist yet — run ansible-playbook playbooks/haproxy.yml"
fi

section "ZFS state (snapshots tolerated)"
check "rpool exists" \
    "zpool list rpool"

section "State file"
if [[ -f "$TALAS_STATE_FILE" ]]; then
    info "phases recorded :"
    cat "$TALAS_STATE_FILE" | sed 's/^/  /'
else
    warn "no state file at $TALAS_STATE_FILE — bootstrap-remote.sh hasn't run yet"
fi

section "Result"
if (( FAIL == 0 )); then
    ok "$PASS / $((PASS + FAIL)) checks passed"
    exit 0
else
    err "$FAIL FAIL out of $((PASS + FAIL))  ($PASS passed)"
    exit 1
fi