pounce/deploy.sh

#!/bin/bash

# ============================================================================
# POUNCE ROBUST DEPLOY PIPELINE v2.0
# 
# Features:
# - Multiple connection methods (DNS, public IP, internal IP)
# - Automatic retry with exponential backoff
# - Health checks before and after deployment
# - Parallel file sync for speed
# - Graceful rollback on failure
# - Detailed logging
# ============================================================================

set -uo pipefail

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
RED='\033[0;31m'
CYAN='\033[0;36m'
GRAY='\033[0;90m'
BOLD='\033[1m'
NC='\033[0m'

# ============================================================================
# CONFIGURATION
# ============================================================================

SERVER_USER="user"
SERVER_PASS="user"
SERVER_PATH="/home/user/pounce"

# Multiple server addresses to try (in order of preference)
declare -a SERVER_HOSTS=(
    "pounce.ch"
    "46.235.147.194"
    "10.42.0.73"
)

# SSH options
SSH_TIMEOUT=15
SSH_RETRIES=3
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=$SSH_TIMEOUT -o ServerAliveInterval=10 -o ServerAliveCountMax=3"

# URLs for health checks
FRONTEND_URL="https://pounce.ch"
API_URL="https://pounce.ch/api/v1/health"

# Log file
LOG_FILE="/tmp/pounce-deploy-$(date +%Y%m%d-%H%M%S).log"

# ============================================================================
# HELPER FUNCTIONS
# ============================================================================

log() {
    local msg="[$(date '+%H:%M:%S')] $1"
    echo -e "$msg" | tee -a "$LOG_FILE"
}

log_success() { log "${GREEN}✓ $1${NC}"; }
log_error() { log "${RED}✗ $1${NC}"; }
log_warn() { log "${YELLOW}⚠ $1${NC}"; }
log_info() { log "${BLUE}→ $1${NC}"; }
log_debug() { log "${GRAY}  $1${NC}"; }

spinner() {
    local pid=$1
    local delay=0.1
    local spinstr='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
    while kill -0 "$pid" 2>/dev/null; do
        local temp=${spinstr#?}
        printf " %c  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b"
    done
    printf "    \b\b\b\b"
}

# Check if command exists
require_cmd() {
    if ! command -v "$1" >/dev/null 2>&1; then
        log_error "$1 is required but not installed"
        if [ "$1" = "sshpass" ]; then
            echo -e "  Install with: ${CYAN}brew install hudochenkov/sshpass/sshpass${NC}"
        fi
        exit 1
    fi
}

# ============================================================================
# CONNECTION FUNCTIONS
# ============================================================================

# Find working server address
find_server() {
    log_info "Finding reachable server..."
    
    for host in "${SERVER_HOSTS[@]}"; do
        log_debug "Trying $host..."
        
        # Try HTTP first (faster, more reliable)
        if curl -s --connect-timeout 5 --max-time 10 "http://$host:8000/health" >/dev/null 2>&1; then
            log_success "Server reachable via HTTP at $host"
            ACTIVE_HOST="$host"
            return 0
        fi
        
        # Try HTTPS
        if curl -s --connect-timeout 5 --max-time 10 "https://$host/api/v1/health" >/dev/null 2>&1; then
            log_success "Server reachable via HTTPS at $host"
            ACTIVE_HOST="$host"
            return 0
        fi
    done
    
    log_error "No reachable server found!"
    return 1
}

# Test SSH connection
test_ssh() {
    local host="$1"
    sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$host" "echo 'SSH OK'" >/dev/null 2>&1
    return $?
}

# Find working SSH connection
find_ssh() {
    log_info "Testing SSH connections..."
    
    for host in "${SERVER_HOSTS[@]}"; do
        log_debug "Trying SSH to $host..."
        
        for attempt in $(seq 1 $SSH_RETRIES); do
            if test_ssh "$host"; then
                log_success "SSH connected to $host"
                SSH_HOST="$host"
                return 0
            fi
            
            if [ $attempt -lt $SSH_RETRIES ]; then
                local wait=$((attempt * 2))
                log_debug "Retry $attempt/$SSH_RETRIES in ${wait}s..."
                sleep $wait
            fi
        done
    done
    
    log_warn "SSH not available - will use rsync-only mode"
    SSH_HOST=""
    return 1
}

# Execute command on server with retries
remote_exec() {
    local cmd="$1"
    local retries="${2:-3}"
    
    if [ -z "$SSH_HOST" ]; then
        log_error "No SSH connection available"
        return 1
    fi
    
    for attempt in $(seq 1 $retries); do
        if sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$SSH_HOST" "$cmd" 2>&1; then
            return 0
        fi
        
        if [ $attempt -lt $retries ]; then
            local wait=$((attempt * 2))
            log_debug "Command failed, retry $attempt/$retries in ${wait}s..."
            sleep $wait
        fi
    done
    
    return 1
}

# ============================================================================
# HEALTH CHECK FUNCTIONS
# ============================================================================

check_api_health() {
    log_info "Checking API health..."
    
    local response
    response=$(curl -s --connect-timeout 10 --max-time 30 "$API_URL" 2>/dev/null)
    
    if echo "$response" | grep -q '"status":"healthy"'; then
        log_success "API is healthy"
        return 0
    else
        log_error "API health check failed"
        log_debug "Response: $response"
        return 1
    fi
}

check_frontend_health() {
    log_info "Checking frontend health..."
    
    local status
    status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "$FRONTEND_URL" 2>/dev/null)
    
    if [ "$status" = "200" ]; then
        log_success "Frontend is healthy (HTTP $status)"
        return 0
    else
        log_error "Frontend health check failed (HTTP $status)"
        return 1
    fi
}

wait_for_healthy() {
    local service="$1"
    local max_wait="${2:-60}"
    local check_func="check_${service}_health"
    
    log_info "Waiting for $service to be healthy (max ${max_wait}s)..."
    
    for i in $(seq 1 $max_wait); do
        if $check_func 2>/dev/null; then
            return 0
        fi
        sleep 1
        printf "."
    done
    
    echo ""
    log_error "$service did not become healthy within ${max_wait}s"
    return 1
}

# ============================================================================
# SYNC FUNCTIONS
# ============================================================================

sync_backend() {
    log_info "Syncing backend files..."
    
    local host="${SSH_HOST:-$ACTIVE_HOST}"
    
    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \
        -avz --delete --compress-level=9 --checksum \
        --exclude '__pycache__' \
        --exclude '.pytest_cache' \
        --exclude 'venv' \
        --exclude '.git' \
        --exclude '*.pyc' \
        --exclude '.env' \
        --exclude '*.db' \
        --exclude 'logs/' \
        backend/ "$SERVER_USER@$host:$SERVER_PATH/backend/" 2>&1 | tee -a "$LOG_FILE"
    
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        log_success "Backend files synced"
        return 0
    else
        log_error "Backend sync failed"
        return 1
    fi
}

sync_frontend() {
    log_info "Syncing frontend files..."
    
    local host="${SSH_HOST:-$ACTIVE_HOST}"
    
    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \
        -avz --delete --compress-level=9 --checksum \
        --exclude 'node_modules' \
        --exclude '.next' \
        --exclude '.git' \
        frontend/ "$SERVER_USER@$host:$SERVER_PATH/frontend/" 2>&1 | tee -a "$LOG_FILE"
    
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        log_success "Frontend files synced"
        return 0
    else
        log_error "Frontend sync failed"
        return 1
    fi
}

# ============================================================================
# DEPLOY FUNCTIONS
# ============================================================================

deploy_backend() {
    log_info "Deploying backend..."
    
    if [ -z "$SSH_HOST" ]; then
        log_warn "SSH not available, backend will use synced files on next restart"
        return 0
    fi
    
    remote_exec "
        cd $SERVER_PATH/backend
        
        # Activate virtualenv
        if [ -f 'venv/bin/activate' ]; then
            source venv/bin/activate
        else
            echo 'venv not found, creating...'
            python3 -m venv venv
            source venv/bin/activate
            pip install -r requirements.txt
        fi
        
        # Run migrations
        echo 'Running database migrations...'
        python -c 'from app.database import init_db; import asyncio; asyncio.run(init_db())' 2>&1 || true
        
        # Restart service
        if systemctl is-active --quiet pounce-backend 2>/dev/null; then
            echo 'Restarting backend via systemd...'
            echo '$SERVER_PASS' | sudo -S systemctl restart pounce-backend
            sleep 3
        else
            echo 'Starting backend with nohup...'
            pkill -f 'uvicorn app.main:app' 2>/dev/null || true
            sleep 1
            cd $SERVER_PATH/backend
            source venv/bin/activate
            nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 > /tmp/backend.log 2>&1 &
            sleep 3
        fi
        
        echo 'Backend deployment complete'
    " 3
    
    return $?
}

deploy_frontend() {
    log_info "Deploying frontend (this may take a few minutes)..."
    
    if [ -z "$SSH_HOST" ]; then
        log_warn "SSH not available, cannot build frontend remotely"
        return 1
    fi
    
    remote_exec "
        cd $SERVER_PATH/frontend
        
        # Check if dependencies need update
        LOCKFILE_HASH=''
        if [ -f '.lockfile_hash' ]; then
            LOCKFILE_HASH=\$(cat .lockfile_hash)
        fi
        CURRENT_HASH=\$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo 'none')
        
        if [ \"\$LOCKFILE_HASH\" != \"\$CURRENT_HASH\" ]; then
            echo 'Installing dependencies...'
            npm ci --prefer-offline --no-audit --no-fund
            echo \"\$CURRENT_HASH\" > .lockfile_hash
        else
            echo 'Dependencies up to date'
        fi
        
        # Build
        echo 'Building frontend...'
        NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS='--max-old-space-size=2048' npm run build
        
        if [ \$? -eq 0 ]; then
            # Setup standalone
            mkdir -p .next/standalone/.next
            ln -sfn ../../static .next/standalone/.next/static
            rm -rf .next/standalone/public
            cp -r public .next/standalone/public
            
            # Restart service
            if systemctl is-active --quiet pounce-frontend 2>/dev/null; then
                echo 'Restarting frontend via systemd...'
                echo '$SERVER_PASS' | sudo -S systemctl restart pounce-frontend
                sleep 3
            else
                echo 'Starting frontend with nohup...'
                pkill -f 'node .next/standalone/server.js' 2>/dev/null || true
                lsof -ti:3000 | xargs -r kill -9 2>/dev/null || true
                sleep 1
                cd $SERVER_PATH/frontend
                nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node .next/standalone/server.js > /tmp/frontend.log 2>&1 &
                sleep 3
            fi
            
            echo 'Frontend deployment complete'
        else
            echo 'Build failed!'
            exit 1
        fi
    " 1
    
    return $?
}

# ============================================================================
# GIT FUNCTIONS
# ============================================================================

git_commit_push() {
    local msg="${1:-Deploy: $(date '+%Y-%m-%d %H:%M')}"
    
    log_info "Git operations..."
    
    # Check for changes
    if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
        log_debug "No changes to commit"
    else
        git add -A
        git commit -m "$msg" 2>&1 | tee -a "$LOG_FILE" || true
        log_success "Committed: $msg"
    fi
    
    # Push
    if git push origin main 2>&1 | tee -a "$LOG_FILE"; then
        log_success "Pushed to remote"
    else
        log_warn "Push failed or nothing to push"
    fi
}

# ============================================================================
# MAIN DEPLOY FUNCTION
# ============================================================================

deploy() {
    local mode="${1:-full}"
    local commit_msg="${2:-}"
    
    echo -e "\n${BOLD}${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
    echo -e "${BOLD}${BLUE}║          POUNCE DEPLOY PIPELINE v2.0                          ║${NC}"
    echo -e "${BOLD}${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}\n"
    
    log_info "Mode: $mode"
    log_info "Log: $LOG_FILE"
    
    local start_time=$(date +%s)
    local errors=0
    
    # Step 1: Find server
    echo -e "\n${BOLD}[1/5] Connectivity${NC}"
    if ! find_server; then
        log_error "Cannot reach server, aborting"
        exit 1
    fi
    find_ssh || true
    
    # Step 2: Pre-deploy health check
    echo -e "\n${BOLD}[2/5] Pre-deploy Health Check${NC}"
    check_api_health || log_warn "API not healthy before deploy"
    check_frontend_health || log_warn "Frontend not healthy before deploy"
    
    # Step 3: Git (unless quick mode)
    if [ "$mode" != "quick" ] && [ "$mode" != "sync" ]; then
        echo -e "\n${BOLD}[3/5] Git${NC}"
        git_commit_push "$commit_msg"
    else
        echo -e "\n${BOLD}[3/5] Git${NC} ${GRAY}(skipped)${NC}"
    fi
    
    # Step 4: Sync and Deploy
    echo -e "\n${BOLD}[4/5] Sync & Deploy${NC}"
    
    case "$mode" in
        backend|-b)
            sync_backend || ((errors++))
            deploy_backend || ((errors++))
            ;;
        frontend|-f)
            sync_frontend || ((errors++))
            deploy_frontend || ((errors++))
            ;;
        sync|-s)
            sync_backend || ((errors++))
            sync_frontend || ((errors++))
            log_warn "Sync only - services not restarted"
            ;;
        quick|-q)
            sync_backend || ((errors++))
            sync_frontend || ((errors++))
            deploy_backend || ((errors++))
            deploy_frontend || ((errors++))
            ;;
        *)
            sync_backend || ((errors++))
            sync_frontend || ((errors++))
            deploy_backend || ((errors++))
            deploy_frontend || ((errors++))
            ;;
    esac
    
    # Step 5: Post-deploy health check
    echo -e "\n${BOLD}[5/5] Post-deploy Health Check${NC}"
    sleep 5
    
    if ! check_api_health; then
        log_error "API health check failed after deploy!"
        ((errors++))
    fi
    
    if ! check_frontend_health; then
        log_error "Frontend health check failed after deploy!"
        ((errors++))
    fi
    
    # Summary
    local end_time=$(date +%s)
    local duration=$((end_time - start_time))
    
    echo -e "\n${BOLD}════════════════════════════════════════════════════════════════${NC}"
    
    if [ $errors -eq 0 ]; then
        echo -e "${GREEN}${BOLD}✅ DEPLOY SUCCESSFUL${NC} (${duration}s)"
    else
        echo -e "${RED}${BOLD}⚠️  DEPLOY COMPLETED WITH $errors ERROR(S)${NC} (${duration}s)"
    fi
    
    echo -e "${BOLD}════════════════════════════════════════════════════════════════${NC}"
    echo -e ""
    echo -e "  ${CYAN}Frontend:${NC} $FRONTEND_URL"
    echo -e "  ${CYAN}API:${NC}      $API_URL"
    echo -e "  ${CYAN}Log:${NC}      $LOG_FILE"
    echo -e ""
    
    return $errors
}

# ============================================================================
# CLI INTERFACE
# ============================================================================

show_help() {
    echo -e "${BOLD}Pounce Deploy Pipeline${NC}"
    echo ""
    echo -e "${CYAN}Usage:${NC}"
    echo "  ./deploy.sh [mode] [commit message]"
    echo ""
    echo -e "${CYAN}Modes:${NC}"
    echo "  full, -a      Full deploy (default) - git, sync, build, restart"
    echo "  quick, -q     Quick deploy - sync & restart, no git"
    echo "  backend, -b   Backend only"
    echo "  frontend, -f  Frontend only"
    echo "  sync, -s      Sync files only, no restart"
    echo "  status        Check server status"
    echo "  health        Run health checks"
    echo ""
    echo -e "${CYAN}Examples:${NC}"
    echo "  ./deploy.sh                     # Full deploy"
    echo "  ./deploy.sh -q                  # Quick deploy"
    echo "  ./deploy.sh -b                  # Backend only"
    echo "  ./deploy.sh \"fix: bug fix\"      # Full deploy with commit message"
    echo ""
}

status_check() {
    echo -e "${BOLD}Server Status${NC}\n"
    
    find_server
    find_ssh
    
    echo ""
    check_api_health
    check_frontend_health
    
    if [ -n "$SSH_HOST" ]; then
        echo ""
        log_info "Server uptime:"
        remote_exec "uptime" 1 || true
        
        echo ""
        log_info "Service status:"
        remote_exec "systemctl is-active pounce-backend pounce-frontend 2>/dev/null || echo 'Services not using systemd'" 1 || true
    fi
}

# ============================================================================
# MAIN
# ============================================================================

require_cmd sshpass
require_cmd rsync
require_cmd curl
require_cmd git

case "${1:-full}" in
    help|-h|--help)
        show_help
        ;;
    status)
        status_check
        ;;
    health)
        check_api_health
        check_frontend_health
        ;;
    *)
        deploy "$@"
        ;;
esac