veza/config/incus/deploy-service-native.sh

#!/bin/bash
# Deploy a Veza service to Incus without Docker (native deployment)
# Usage: ./deploy-service-native.sh <service-name>

set -euo pipefail

SERVICE=$1
PROJECT_NAME="veza"
NETWORK="veza-network"
PROFILE="veza-profile"
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
BUILD_DIR="${PROJECT_ROOT}/.build/incus"

if [ -z "$SERVICE" ]; then
    echo "Usage: $0 <service-name>"
    echo "Services: backend-api, chat-server, stream-server, web, haproxy, infra"
    exit 1
fi

CONTAINER_NAME="${PROJECT_NAME}-${SERVICE}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

echo "📦 Deploying ${SERVICE} to Incus (native, no Docker)..."

# Ensure basic Incus infrastructure is set up
if [ -f "${SCRIPT_DIR}/setup-basic-incus.sh" ]; then
    echo "Ensuring basic Incus infrastructure is configured..."
    "${SCRIPT_DIR}/setup-basic-incus.sh" >/dev/null 2>&1 || {
        echo "❌ ERROR: Failed to setup basic Incus infrastructure"
        echo "   Run manually: ${SCRIPT_DIR}/setup-basic-incus.sh"
        exit 1
    }
fi

# Verify network exists and is properly configured
if ! incus network show ${NETWORK} >/dev/null 2>&1; then
    echo "❌ ERROR: Network ${NETWORK} does not exist"
    echo "   Run: ${SCRIPT_DIR}/setup-basic-incus.sh"
    exit 1
fi

# Verify network has required configuration
NETWORK_CONFIG=$(incus network show ${NETWORK})
if ! echo "${NETWORK_CONFIG}" | grep -q "ipv4.nat: \"true\""; then
    echo "❌ ERROR: Network ${NETWORK} does not have NAT enabled"
    echo "   This is required for Internet connectivity"
    echo "   Run: ${SCRIPT_DIR}/setup-basic-incus.sh"
    exit 1
fi

# Verify host has firewall/NAT tooling available (required for Incus NAT to actually work)
if ! command -v nft >/dev/null 2>&1 && ! command -v iptables >/dev/null 2>&1; then
    echo "❌ ERROR: Host is missing NAT tooling ('nft' or 'iptables')."
    echo "   Containers may reach the gateway (10.10.10.1) but will NOT reach the Internet."
    echo ""
    echo "   Fedora:"
    echo "     sudo dnf install -y nftables iptables-nft"
    echo ""
    echo "   Then run:"
    echo "     ${SCRIPT_DIR}/fix-network-now.sh"
    exit 1
fi

# Verify profile exists and has required devices
if ! incus profile show ${PROFILE} >/dev/null 2>&1; then
    echo "❌ ERROR: Profile ${PROFILE} does not exist"
    echo "   Run: ${SCRIPT_DIR}/setup-basic-incus.sh"
    exit 1
fi

# Verify profile has root device
PROFILE_CONFIG=$(incus profile show ${PROFILE})
if ! echo "${PROFILE_CONFIG}" | grep -q "root:"; then
    echo "❌ ERROR: Profile ${PROFILE} does not have root device configured"
    echo "   Run: ${SCRIPT_DIR}/setup-basic-incus.sh"
    exit 1
fi

# Verify profile has network device
if ! echo "${PROFILE_CONFIG}" | grep -q "eth0:"; then
    echo "❌ ERROR: Profile ${PROFILE} does not have network device configured"
    echo "   Run: ${SCRIPT_DIR}/setup-basic-incus.sh"
    exit 1
fi

# Delete existing container if it exists
if incus list -c n --format csv | grep -q "^${CONTAINER_NAME}$"; then
    echo "Removing existing container ${CONTAINER_NAME}..."
    incus delete ${CONTAINER_NAME} --force
fi

# Create and start container
echo "Creating container ${CONTAINER_NAME}..."
incus init images:debian/13 ${CONTAINER_NAME} --profile ${PROFILE}
incus start ${CONTAINER_NAME}

# Wait for container to be ready
echo "Waiting for container to be ready..."
for i in {1..30}; do
    if incus exec ${CONTAINER_NAME} -- systemctl is-system-running >/dev/null 2>&1; then
        break
    fi
    sleep 1
done
sleep 2

# Configure static IP based on service
case ${SERVICE} in
    backend-api)
        STATIC_IP="10.10.10.2"
        ;;
    chat-server)
        STATIC_IP="10.10.10.3"
        ;;
    stream-server)
        STATIC_IP="10.10.10.4"
        ;;
    web)
        STATIC_IP="10.10.10.5"
        ;;
    haproxy)
        STATIC_IP="10.10.10.6"
        ;;
    infra)
        STATIC_IP="10.10.10.10"
        ;;
    *)
        STATIC_IP=""
        ;;
esac

# Set static IP if configured
if [ -n "${STATIC_IP}" ]; then
    echo "Configuring static IP ${STATIC_IP}..."
    # Override the device from profile to set static IP
    incus config device override ${CONTAINER_NAME} eth0 ipv4.address=${STATIC_IP} 2>/dev/null || \
        incus config device set ${CONTAINER_NAME} eth0 ipv4.address=${STATIC_IP} 2>/dev/null || \
        incus config device add ${CONTAINER_NAME} eth0 nic network=${NETWORK} ipv4.address=${STATIC_IP} 2>/dev/null || true
    incus restart ${CONTAINER_NAME}
    sleep 5
    # Configure network inside container (IP, default route, and DNS)
    echo "Configuring network inside container..."
    
    # Wait for container to be fully ready
    for i in {1..10}; do
        if incus exec ${CONTAINER_NAME} -- systemctl is-system-running >/dev/null 2>&1; then
            break
        fi
        sleep 1
    done
    sleep 2
    
    # Ensure IP forwarding is enabled on host (required for NAT)
    if [ "$(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo 0)" != "1" ]; then
        echo "⚠️  IP forwarding is disabled on host, enabling..."
        echo "1" | sudo tee /proc/sys/net/ipv4/ip_forward >/dev/null 2>&1 || {
            echo "❌ ERROR: Cannot enable IP forwarding (NAT requires this)"
            echo "   Run as root or with sudo: echo 1 > /proc/sys/net/ipv4/ip_forward"
            exit 1
        }
    fi
    
    if ! incus exec ${CONTAINER_NAME} -- bash -c "
        set -e
        # Remove any existing IP on eth0
        ip addr flush dev eth0 2>/dev/null || true
        # Add static IP
        ip addr add ${STATIC_IP}/24 dev eth0 || ip addr replace ${STATIC_IP}/24 dev eth0
        # Remove any existing routes
        ip route del default 2>/dev/null || true
        ip route del 10.10.10.0/24 2>/dev/null || true
        # Add local network route (required for local communication)
        ip route add 10.10.10.0/24 dev eth0 proto kernel scope link src ${STATIC_IP}
        # Add default route (CRITICAL for NAT to work)
        ip route add default via 10.10.10.1 dev eth0
        # Verify routes were added
        if ! ip route | grep -q 'default via 10.10.10.1'; then
            echo 'ERROR: Default route not configured'
            exit 1
        fi
        if ! ip route | grep -q '10.10.10.0/24'; then
            echo 'ERROR: Local network route not configured'
            exit 1
        fi
        # Configure DNS persistently
        mkdir -p /etc/systemd/resolved.conf.d
        cat > /etc/systemd/resolved.conf.d/dns_servers.conf <<'EOF'
[Resolve]
DNS=8.8.8.8 1.1.1.1
FallbackDNS=8.8.4.4 1.0.0.1
EOF
        # Also update resolv.conf directly for immediate effect
        rm -f /etc/resolv.conf
        echo 'nameserver 8.8.8.8' > /etc/resolv.conf
        echo 'nameserver 1.1.1.1' >> /etc/resolv.conf
        # Restart systemd-resolved if available
        systemctl restart systemd-resolved 2>/dev/null || true
        # Verify IP is configured
        if ! ip addr show eth0 | grep -q '${STATIC_IP}'; then
            echo 'ERROR: IP address not configured correctly'
            exit 1
        fi
    "; then
        echo "❌ ERROR: Failed to configure network inside container"
        echo "   Run diagnostic: ${SCRIPT_DIR}/diagnose-network.sh ${CONTAINER_NAME}"
        echo "   Run fix: ${SCRIPT_DIR}/fix-network.sh ${CONTAINER_NAME}"
        exit 1
    fi
    
    # Verify network connectivity - CRITICAL: Do not continue if network fails
    echo "Verifying network connectivity (CRITICAL - deployment will fail if this doesn't work)..."
    
    # Test 1: Ping gateway
    echo "  Testing gateway connectivity (10.10.10.1)..."
    if ! incus exec ${CONTAINER_NAME} -- ping -c 3 -W 3 10.10.10.1 >/dev/null 2>&1; then
        echo "❌ ERROR: Cannot reach gateway (10.10.10.1)"
        echo "   Network configuration failed. Deployment aborted."
        exit 1
    fi
    
    # Test 2: Ping external DNS
    echo "  Testing Internet connectivity (8.8.8.8)..."
    if ! incus exec ${CONTAINER_NAME} -- ping -c 3 -W 3 8.8.8.8 >/dev/null 2>&1; then
        echo "❌ ERROR: Cannot reach Internet (8.8.8.8)"
        echo "   Internet connectivity is required for package installation."
        echo "   Check NAT configuration: incus network show ${NETWORK}"
        exit 1
    fi
    
    # Test 3: DNS resolution
    echo "  Testing DNS resolution (google.com)..."
    if ! incus exec ${CONTAINER_NAME} -- getent hosts google.com >/dev/null 2>&1; then
        echo "❌ ERROR: DNS resolution failed"
        echo "   Cannot resolve hostnames. DNS configuration failed."
        echo "   Check DNS configuration in container: incus exec ${CONTAINER_NAME} -- cat /etc/resolv.conf"
        exit 1
    fi
    
    echo "✅ Network connectivity fully verified (gateway, Internet, DNS)"
    echo "   Network is fully functional. Continuing with deployment..."
else
    # Even without static IP, verify basic connectivity
    echo "Verifying basic network connectivity..."
    if ! incus exec ${CONTAINER_NAME} -- ping -c 2 -W 2 8.8.8.8 >/dev/null 2>&1; then
        echo "❌ ERROR: Basic Internet connectivity failed"
        echo "   Internet connectivity is required for package installation."
        exit 1
    fi
    echo "✅ Basic network connectivity verified"
fi

# Install base dependencies
echo "Installing base dependencies..."
incus exec ${CONTAINER_NAME} -- bash -c "
    set -e
    export DEBIAN_FRONTEND=noninteractive
    # Update package lists with retries
    for i in {1..3}; do
        if apt-get update -qq; then
            break
        fi
        echo \"Retry $i/3...\"
        sleep 2
    done
    # Install essential packages
    apt-get install -y -qq \
        curl \
        wget \
        ca-certificates \
        systemd \
        systemd-sysv \
        iproute2 \
        net-tools \
        iputils-ping \
        dnsutils \
        || echo \"Warning: Some packages failed to install\"
"

# Service-specific deployment
case ${SERVICE} in
    backend-api)
        echo "Installing Go runtime dependencies..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            apt-get install -y -qq ca-certificates libc6
        "
        
        # Copy binary
        echo "Copying backend-api binary..."
        if [ ! -f "${BUILD_DIR}/veza-backend-api" ]; then
            echo "ERROR: Binary not found at ${BUILD_DIR}/veza-backend-api"
            echo "Please run: make build-all-native or ./config/incus/build-native.sh backend-api"
            exit 1
        fi
        incus file push "${BUILD_DIR}/veza-backend-api" ${CONTAINER_NAME}/usr/local/bin/veza-backend-api
        incus exec ${CONTAINER_NAME} -- chmod +x /usr/local/bin/veza-backend-api
        
        # Create directories
        incus exec ${CONTAINER_NAME} -- bash -c "
            mkdir -p /opt/veza/backend-api
            mkdir -p /var/log/veza
            mkdir -p /etc/veza
        "
        
        # Copy wrapper script
        incus exec ${CONTAINER_NAME} -- bash -c "mkdir -p /opt/veza/backend-api"
        incus file push "${PROJECT_ROOT}/config/incus/scripts/start-backend-api.sh" \
            ${CONTAINER_NAME}/opt/veza/backend-api/start-backend-api.sh
        incus exec ${CONTAINER_NAME} -- chmod +x /opt/veza/backend-api/start-backend-api.sh
        
        # Copy systemd service
        incus file push "${PROJECT_ROOT}/config/incus/systemd/veza-backend-api.service" \
            ${CONTAINER_NAME}/etc/systemd/system/veza-backend-api.service
        
        # Copy environment file - CRITICAL: Must exist and be copied
        ENV_FILE="${PROJECT_ROOT}/config/incus/env/backend-api.env"
        if [ ! -f "${ENV_FILE}" ]; then
            echo "❌ ERROR: Environment file not found: ${ENV_FILE}"
            echo "   Please ensure the file exists before deploying."
            exit 1
        fi
        echo "📄 Copying environment file: ${ENV_FILE}"
        if ! incus file push "${ENV_FILE}" ${CONTAINER_NAME}/etc/veza/backend-api.env; then
            echo "❌ ERROR: Failed to push environment file to container"
            exit 1
        fi
        
        # Verify the file was copied correctly
        if ! incus exec ${CONTAINER_NAME} -- test -f /etc/veza/backend-api.env; then
            echo "❌ ERROR: Failed to copy environment file to container"
            exit 1
        fi
        
        # Verify file is readable and has content
        if ! incus exec ${CONTAINER_NAME} -- test -r /etc/veza/backend-api.env; then
            echo "❌ ERROR: Environment file is not readable"
            exit 1
        fi
        
        # Check file has at least some content (not empty)
        FILE_SIZE=$(incus exec ${CONTAINER_NAME} -- stat -c%s /etc/veza/backend-api.env 2>/dev/null || echo "0")
        if [ "${FILE_SIZE}" -lt 100 ]; then
            echo "⚠️  WARNING: Environment file seems too small (${FILE_SIZE} bytes)"
        fi
        
        echo "✅ Environment file copied successfully (${FILE_SIZE} bytes)"
        
        # Copy .env file to working directory for godotenv.Load() compatibility
        echo "📄 Copying .env file to working directory: /opt/veza/backend-api/.env"
        if ! incus file push "${ENV_FILE}" ${CONTAINER_NAME}/opt/veza/backend-api/.env; then
            echo "❌ ERROR: Failed to push .env file to working directory"
            exit 1
        fi
        
        # Verify the .env file was copied correctly to working directory
        if ! incus exec ${CONTAINER_NAME} -- test -f /opt/veza/backend-api/.env; then
            echo "❌ ERROR: Failed to copy .env file to working directory"
            exit 1
        fi
        
        # Verify .env file is readable and has content
        if ! incus exec ${CONTAINER_NAME} -- test -r /opt/veza/backend-api/.env; then
            echo "❌ ERROR: .env file in working directory is not readable"
            exit 1
        fi
        
        # Set correct permissions on .env file
        incus exec ${CONTAINER_NAME} -- chmod 644 /opt/veza/backend-api/.env
        
        # Check .env file has at least some content (not empty)
        ENV_FILE_SIZE=$(incus exec ${CONTAINER_NAME} -- stat -c%s /opt/veza/backend-api/.env 2>/dev/null || echo "0")
        if [ "${ENV_FILE_SIZE}" -lt 100 ]; then
            echo "⚠️  WARNING: .env file in working directory seems too small (${ENV_FILE_SIZE} bytes)"
        fi
        
        echo "✅ .env file copied to working directory successfully (${ENV_FILE_SIZE} bytes)"
        
        # Verify critical environment variables are present in .env file
        echo "🔍 Verifying critical environment variables in .env file..."
        CRITICAL_VARS=("DATABASE_URL" "JWT_SECRET" "APP_ENV")
        for var in "${CRITICAL_VARS[@]}"; do
            if ! incus exec ${CONTAINER_NAME} -- grep -q "^${var}=" /opt/veza/backend-api/.env; then
                echo "⚠️  WARNING: Critical variable ${var} not found in .env file"
            else
                echo "  ✅ ${var} found"
            fi
        done
        
        # Verify both .env files are identical
        if ! incus exec ${CONTAINER_NAME} -- diff -q /etc/veza/backend-api.env /opt/veza/backend-api/.env >/dev/null 2>&1; then
            echo "⚠️  WARNING: .env files in /etc/veza/ and working directory differ"
        else
            echo "✅ .env files are identical"
        fi
        
        # Test that wrapper script can load environment variables
        echo "🧪 Testing environment variable loading..."
        if incus exec ${CONTAINER_NAME} -- bash -c 'set -a; source /etc/veza/backend-api.env; set +a; [ -n "$DATABASE_URL" ] && [ -n "$JWT_SECRET" ]'; then
            echo "✅ Environment variables can be loaded successfully"
        else
            echo "❌ ERROR: Failed to load environment variables"
            exit 1
        fi
        
        # Reload systemd to pick up the new environment file
        incus exec ${CONTAINER_NAME} -- systemctl daemon-reload
        
        # Verify systemd can see the environment file
        if incus exec ${CONTAINER_NAME} -- systemctl show veza-backend-api --property=EnvironmentFiles --no-pager 2>/dev/null | grep -q "backend-api.env"; then
            echo "✅ Systemd environment file configured correctly"
        else
            echo "⚠️  WARNING: Systemd may not be reading the environment file"
            echo "   Checking service file..."
            incus exec ${CONTAINER_NAME} -- cat /etc/systemd/system/veza-backend-api.service | grep -E "EnvironmentFile|Environment" || true
        fi
        
        # Enable and start service
        incus exec ${CONTAINER_NAME} -- systemctl enable veza-backend-api
        incus exec ${CONTAINER_NAME} -- systemctl start veza-backend-api || echo "Warning: Service start failed, check logs"
        ;;
        
    chat-server)
        echo "Installing Rust runtime dependencies..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            apt-get install -y -qq ca-certificates libc6 libssl3
        "
        
        # Copy binary
        echo "Copying chat-server binary..."
        if [ ! -f "${BUILD_DIR}/veza-chat-server" ]; then
            echo "ERROR: Binary not found at ${BUILD_DIR}/veza-chat-server"
            echo "Please run: make build-all-native or ./config/incus/build-native.sh chat-server"
            exit 1
        fi
        incus file push "${BUILD_DIR}/veza-chat-server" ${CONTAINER_NAME}/usr/local/bin/veza-chat-server
        incus exec ${CONTAINER_NAME} -- chmod +x /usr/local/bin/veza-chat-server
        
        # Create directories
        incus exec ${CONTAINER_NAME} -- bash -c "
            mkdir -p /opt/veza/chat-server
            mkdir -p /var/log/veza
            mkdir -p /etc/veza
        "
        
        # Copy systemd service
        incus file push "${PROJECT_ROOT}/config/incus/systemd/veza-chat-server.service" \
            ${CONTAINER_NAME}/etc/systemd/system/veza-chat-server.service
        
        # Copy environment file template
        incus file push "${PROJECT_ROOT}/config/incus/env/chat-server.env" \
            ${CONTAINER_NAME}/etc/veza/chat-server.env 2>/dev/null || true
        
        # Enable and start service
        incus exec ${CONTAINER_NAME} -- systemctl daemon-reload
        incus exec ${CONTAINER_NAME} -- systemctl enable veza-chat-server
        incus exec ${CONTAINER_NAME} -- systemctl start veza-chat-server || echo "Warning: Service start failed, check logs"
        ;;
        
    stream-server)
        echo "Installing Rust runtime dependencies..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            apt-get install -y -qq ca-certificates libc6 libssl3
        "
        
        # Copy binary
        echo "Copying stream-server binary..."
        if [ ! -f "${BUILD_DIR}/veza-stream-server" ]; then
            echo "ERROR: Binary not found at ${BUILD_DIR}/veza-stream-server"
            echo "Please run: make build-all-native or ./config/incus/build-native.sh stream-server"
            exit 1
        fi
        incus file push "${BUILD_DIR}/veza-stream-server" ${CONTAINER_NAME}/usr/local/bin/veza-stream-server
        incus exec ${CONTAINER_NAME} -- chmod +x /usr/local/bin/veza-stream-server
        
        # Create directories
        incus exec ${CONTAINER_NAME} -- bash -c "
            mkdir -p /opt/veza/stream-server
            mkdir -p /opt/veza/stream-server/audio
            mkdir -p /var/log/veza
            mkdir -p /etc/veza
        "
        
        # Copy systemd service
        incus file push "${PROJECT_ROOT}/config/incus/systemd/veza-stream-server.service" \
            ${CONTAINER_NAME}/etc/systemd/system/veza-stream-server.service
        
        # Copy environment file template
        incus file push "${PROJECT_ROOT}/config/incus/env/stream-server.env" \
            ${CONTAINER_NAME}/etc/veza/stream-server.env 2>/dev/null || true
        
        # Enable and start service
        incus exec ${CONTAINER_NAME} -- systemctl daemon-reload
        incus exec ${CONTAINER_NAME} -- systemctl enable veza-stream-server
        incus exec ${CONTAINER_NAME} -- systemctl start veza-stream-server || echo "Warning: Service start failed, check logs"
        ;;
        
    web)
        echo "Installing web server (Apache/httpd)..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            apt-get install -y -qq apache2
        "
        
        # Copy web files
        echo "Copying web frontend files..."
        if [ ! -d "${BUILD_DIR}/web" ] || [ ! -f "${BUILD_DIR}/web/index.html" ]; then
            echo "ERROR: Web build not found at ${BUILD_DIR}/web"
            echo "Please run: make build-all-native or ./config/incus/build-native.sh web"
            exit 1
        fi
        incus exec ${CONTAINER_NAME} -- mkdir -p /var/www/html
        incus exec ${CONTAINER_NAME} -- rm -rf /var/www/html/* || true
        incus file push -r "${BUILD_DIR}/web"/* ${CONTAINER_NAME}/var/www/html/
        
        # Copy Apache config
        incus file push "${PROJECT_ROOT}/config/incus/apache/veza-web.conf" \
            ${CONTAINER_NAME}/etc/apache2/sites-available/veza-web.conf
        
        # Enable Apache site and modules
        incus exec ${CONTAINER_NAME} -- bash -c "
            a2enmod rewrite
            a2enmod headers
            a2enmod expires
            a2enmod deflate
            a2dissite 000-default
            a2ensite veza-web
            apache2ctl configtest
        "
        
        # Configure network (IP may be lost on container restart)
        incus exec ${CONTAINER_NAME} -- bash -c "
            if ! ip addr show eth0 | grep -q 'inet 10.10.10.5/24'; then
                ip addr flush dev eth0 2>/dev/null || true
                ip addr add 10.10.10.5/24 dev eth0 2>/dev/null || true
                ip link set eth0 up 2>/dev/null || true
                ip route add default via 10.10.10.1 dev eth0 2>/dev/null || true
            fi
        "

        
        # Create network setup script for container restart
        # CRITICAL FIX: Enhanced network setup with retries and better error handling
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /usr/local/bin/setup-network.sh << "EOF"
#!/bin/bash
# Configure network on container start
# CRITICAL: This script ensures persistent IP configuration for veza-web container

set -e

MAX_RETRIES=5
RETRY_DELAY=2

configure_network() {
    local retry=0
    while [ $retry -lt $MAX_RETRIES ]; do
        # Check if IP is already configured
        if ip addr show eth0 2>/dev/null | grep -q "inet 10.10.10.5/24"; then
            echo "[Network Setup] IP 10.10.10.5/24 already configured"
            return 0
        fi
        
        # Flush existing addresses
        ip addr flush dev eth0 2>/dev/null || true
        
        # Add static IP
        if ip addr add 10.10.10.5/24 dev eth0 2>/dev/null; then
            echo "[Network Setup] IP 10.10.10.5/24 added successfully"
        else
            echo "[Network Setup] Failed to add IP (attempt $((retry+1))/$MAX_RETRIES)"
            sleep $RETRY_DELAY
            retry=$((retry+1))
            continue
        fi
        
        # Bring interface up
        ip link set eth0 up 2>/dev/null || true
        
        # Configure routes
        ip route del default 2>/dev/null || true
        ip route add default via 10.10.10.1 dev eth0 2>/dev/null || true
        ip route add 10.10.10.0/24 dev eth0 proto kernel scope link src 10.10.10.5 2>/dev/null || true
        
        # Verify connectivity
        if ping -c 1 -W 2 10.10.10.1 >/dev/null 2>&1; then
            echo "[Network Setup] Network configuration verified successfully"
            return 0
        else
            echo "[Network Setup] Network verification failed (attempt $((retry+1))/$MAX_RETRIES)"
            sleep $RETRY_DELAY
            retry=$((retry+1))
        fi
    done
    
    echo "[Network Setup] ERROR: Failed to configure network after $MAX_RETRIES attempts"
    return 1
}

configure_network
EOF
chmod +x /usr/local/bin/setup-network.sh'
        
        # Create systemd service to run network setup on boot
        # CRITICAL FIX: Enhanced systemd service with retries and dependencies
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /etc/systemd/system/veza-network-setup.service << "EOF"
[Unit]
Description=Veza Network Setup - Persistent IP Configuration
After=network-online.target network-pre.target
Wants=network-online.target
Before=apache2.service

[Service]
Type=oneshot
ExecStart=/usr/local/bin/setup-network.sh
RemainAfterExit=yes
Restart=on-failure
RestartSec=5
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable veza-network-setup.service
systemctl start veza-network-setup.service'

        # Create timer to re-apply IP periodically (prevents IP loss)
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /etc/systemd/system/veza-network-setup.timer << "EOF"
[Unit]
Description=Veza Network Setup - Periodic Check

[Timer]
OnBootSec=15s
OnUnitActiveSec=30s
Unit=veza-network-setup.service

[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable veza-network-setup.timer
systemctl start veza-network-setup.timer'
        
        # Enable and restart Apache to apply vhost changes
        incus exec ${CONTAINER_NAME} -- systemctl enable apache2
        incus exec ${CONTAINER_NAME} -- systemctl restart apache2 || echo "Warning: Apache restart failed, check logs"
        ;;
        
    haproxy)
        echo "Installing HAProxy..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            apt-get install -y -qq haproxy
        "
        
        # Copy HAProxy config (Incus-specific)
        incus file push "${PROJECT_ROOT}/config/incus/haproxy.cfg" \
            ${CONTAINER_NAME}/etc/haproxy/haproxy.cfg

        # Install TLS certificate for HTTPS
        incus exec ${CONTAINER_NAME} -- mkdir -p /etc/haproxy/certs
        incus file push "${PROJECT_ROOT}/docker/haproxy/certs/veza.crt" \
            ${CONTAINER_NAME}/etc/haproxy/certs/veza.crt
        incus file push "${PROJECT_ROOT}/docker/haproxy/certs/veza.key" \
            ${CONTAINER_NAME}/etc/haproxy/certs/veza.key
        incus exec ${CONTAINER_NAME} -- bash -c "
            cat /etc/haproxy/certs/veza.crt /etc/haproxy/certs/veza.key > /etc/haproxy/certs/veza.pem
            chmod 600 /etc/haproxy/certs/veza.pem
        "
        
        # Configure HAProxy to start
        incus exec ${CONTAINER_NAME} -- bash -c "
            echo 'ENABLED=1' > /etc/default/haproxy
        "
        
        # Configure network (IP may be lost on container restart)
        incus exec ${CONTAINER_NAME} -- bash -c "
            if ! ip addr show eth0 | grep -q 'inet 10.10.10.6/24'; then
                ip addr flush dev eth0 2>/dev/null || true
                ip addr add 10.10.10.6/24 dev eth0 2>/dev/null || true
                ip link set eth0 up 2>/dev/null || true
                ip route add default via 10.10.10.1 dev eth0 2>/dev/null || true
            fi
        "

        
        # Create network setup script for container restart
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /usr/local/bin/setup-network.sh << "EOF"
#!/bin/bash
# Configure network on container start
if ! ip addr show eth0 | grep -q "inet 10.10.10.6/24"; then
    ip addr flush dev eth0 2>/dev/null || true
    ip addr add 10.10.10.6/24 dev eth0 2>/dev/null || true
    ip link set eth0 up 2>/dev/null || true
    ip route del default 2>/dev/null || true
    ip route add default via 10.10.10.1 dev eth0 2>/dev/null || true
    ip route add 10.10.10.0/24 dev eth0 proto kernel scope link src 10.10.10.6 2>/dev/null || true
fi
EOF
chmod +x /usr/local/bin/setup-network.sh'
        
        # Create systemd service to run network setup on boot
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /etc/systemd/system/veza-network-setup.service << "EOF"
[Unit]
Description=Veza Network Setup
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/setup-network.sh
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable veza-network-setup.service
systemctl start veza-network-setup.service'

        # Create timer to re-apply IP periodically (prevents IP loss)
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /etc/systemd/system/veza-network-setup.timer << "EOF"
[Unit]
Description=Veza Network Setup - Periodic Check

[Timer]
OnBootSec=15s
OnUnitActiveSec=30s
Unit=veza-network-setup.service

[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable veza-network-setup.timer
systemctl start veza-network-setup.timer'
        
        # Install and configure rsyslog for HAProxy logging
        echo "Configuring rsyslog for HAProxy logging..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            if ! command -v rsyslogd >/dev/null 2>&1; then
                apt-get update -qq && apt-get install -y -qq rsyslog
            fi
        "
        
        # Configure rsyslog for HAProxy
        incus exec ${CONTAINER_NAME} -- bash -c 'cat > /etc/rsyslog.d/49-haproxy.conf << "EOF"
# HAProxy logging configuration
# Enable UDP server for syslog
$ModLoad imudp
$UDPServerRun 514
$UDPServerAddress 127.0.0.1

# Also listen on Unix socket
$ModLoad imuxsock
$WorkDirectory /var/spool/rsyslog

# HAProxy local0 (debug/info) -> /var/log/haproxy.log
local0.*    /var/log/haproxy.log
& stop

# HAProxy local1 (notices) -> /var/log/haproxy-notices.log  
local1.*    /var/log/haproxy-notices.log
& stop
EOF
'
        
        # Create log files
        incus exec ${CONTAINER_NAME} -- bash -c "
            mkdir -p /var/log
            touch /var/log/haproxy.log /var/log/haproxy-notices.log
            chmod 644 /var/log/haproxy*.log
        "
        
        # Enable and start rsyslog
        incus exec ${CONTAINER_NAME} -- systemctl enable rsyslog 2>/dev/null || true
        incus exec ${CONTAINER_NAME} -- systemctl restart rsyslog 2>/dev/null || true
        
        # Enable and restart HAProxy
        incus exec ${CONTAINER_NAME} -- systemctl enable haproxy
        incus exec ${CONTAINER_NAME} -- systemctl restart haproxy || echo "Warning: HAProxy restart failed, check logs"
        ;;
        
    infra)
        echo "Installing infrastructure services (PostgreSQL, Redis)..."
        incus exec ${CONTAINER_NAME} -- bash -c "
            export DEBIAN_FRONTEND=noninteractive
            apt-get install -y -qq postgresql redis-server
        "
        
        # Configure PostgreSQL
        incus exec ${CONTAINER_NAME} -- bash -c "
            sudo -u postgres psql -c \"CREATE USER veza WITH PASSWORD 'password';\" || true
            sudo -u postgres psql -c \"CREATE DATABASE veza OWNER veza;\" || true
            sudo -u postgres psql -c \"GRANT ALL PRIVILEGES ON DATABASE veza TO veza;\" || true
        "
        
        # Configure PostgreSQL to listen on all interfaces
        incus exec ${CONTAINER_NAME} -- bash -c "
            # Find postgresql.conf
            PG_CONF=\$(find /etc/postgresql -name postgresql.conf | head -1)
            if [ -n \"\${PG_CONF}\" ]; then
                # Uncomment and set listen_addresses
                sed -i \"s/^#listen_addresses = 'localhost'/listen_addresses = '*'/\" \${PG_CONF} || \
                sed -i \"s/^listen_addresses = 'localhost'/listen_addresses = '*'/\" \${PG_CONF} || \
                echo \"listen_addresses = '*'\" >> \${PG_CONF}
                # Allow app network access in pg_hba.conf (non-SSL + SSL)
                PG_HBA=\$(dirname \${PG_CONF})/pg_hba.conf
                if [ -f \"\${PG_HBA}\" ]; then
                    echo \"host all all 10.10.10.0/24 scram-sha-256\" >> \${PG_HBA}
                    echo \"hostnossl all all 10.10.10.0/24 scram-sha-256\" >> \${PG_HBA}
                fi
                # Restart PostgreSQL to apply changes
                systemctl restart postgresql || true
            fi
        "
        
        # Configure Redis
        incus exec ${CONTAINER_NAME} -- bash -c "
            sed -i 's/^bind 127.0.0.1/bind 0.0.0.0/' /etc/redis/redis.conf || true
            sed -i 's/^protected-mode yes/protected-mode no/' /etc/redis/redis.conf || true
            systemctl restart redis-server || true
        "
        
        # Enable and start services
        incus exec ${CONTAINER_NAME} -- systemctl enable postgresql
        incus exec ${CONTAINER_NAME} -- systemctl start postgresql || echo "Warning: PostgreSQL start failed"
        incus exec ${CONTAINER_NAME} -- systemctl enable redis-server
        incus exec ${CONTAINER_NAME} -- systemctl start redis-server || echo "Warning: Redis start failed"
        
        # Wait for services to be ready
        echo "Waiting for infrastructure services to be ready..."
        sleep 5
        ;;
        
    *)
        echo "Unknown service: ${SERVICE}"
        exit 1
        ;;
esac

# Verify deployment
echo "Verifying deployment..."
if incus list ${CONTAINER_NAME} --format csv | grep -q "${CONTAINER_NAME}"; then
    CONTAINER_STATE=$(incus list ${CONTAINER_NAME} --format csv | cut -d',' -f2)
    echo "Container state: ${CONTAINER_STATE}"
    
    # Check service status if applicable
    case ${SERVICE} in
        backend-api|chat-server|stream-server)
            SERVICE_NAME="veza-${SERVICE}"
            if incus exec ${CONTAINER_NAME} -- systemctl is-active ${SERVICE_NAME} >/dev/null 2>&1; then
                echo "✅ Service ${SERVICE_NAME} is running"
            else
                echo "⚠️  Service ${SERVICE_NAME} is not running (check with: incus exec ${CONTAINER_NAME} -- systemctl status ${SERVICE_NAME})"
            fi
            ;;
        web)
            if incus exec ${CONTAINER_NAME} -- systemctl is-active apache2 >/dev/null 2>&1; then
                echo "✅ Apache is running"
            else
                echo "⚠️  Apache is not running"
            fi
            ;;
        haproxy)
            if incus exec ${CONTAINER_NAME} -- systemctl is-active haproxy >/dev/null 2>&1; then
                echo "✅ HAProxy is running"
            else
                echo "⚠️  HAProxy is not running"
            fi
            ;;
        infra)
            if incus exec ${CONTAINER_NAME} -- systemctl is-active postgresql >/dev/null 2>&1; then
                echo "✅ PostgreSQL is running"
            else
                echo "⚠️  PostgreSQL is not running"
            fi
            if incus exec ${CONTAINER_NAME} -- systemctl is-active redis-server >/dev/null 2>&1; then
                echo "✅ Redis is running"
            else
                echo "⚠️  Redis is not running"
            fi
            ;;
    esac
fi

echo ""
echo "✅ ${SERVICE} deployed successfully!"
echo "Container: ${CONTAINER_NAME}"
echo "Access: incus exec ${CONTAINER_NAME} -- bash"
echo ""
echo "Useful commands:"
echo "  Status: incus exec ${CONTAINER_NAME} -- systemctl status veza-${SERVICE}"
echo "  Logs:   incus exec ${CONTAINER_NAME} -- journalctl -u veza-${SERVICE} -f"