pounce/deploy.sh

#!/bin/bash

# ============================================================================
# POUNCE ZERO-DOWNTIME DEPLOY PIPELINE v3.0
# 
# Features:
# - ZERO-DOWNTIME: Build happens while old server still runs
# - Atomic switchover only after successful build
# - Multiple connection methods (DNS, public IP, internal IP)
# - Automatic retry with exponential backoff
# - Health checks before and after deployment
# - Parallel file sync for speed
# - Detailed logging
# ============================================================================

set -uo pipefail

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
RED='\033[0;31m'
CYAN='\033[0;36m'
GRAY='\033[0;90m'
BOLD='\033[1m'
NC='\033[0m'

# ============================================================================
# CONFIGURATION
# ============================================================================

SERVER_USER="user"
SERVER_PASS="user"
SERVER_PATH="/home/user/pounce"

# Multiple server addresses to try (in order of preference)
declare -a SERVER_HOSTS=(
    "pounce.ch"
    "46.235.147.194"
    "10.42.0.73"
)

# SSH options
SSH_TIMEOUT=15
SSH_RETRIES=3
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=$SSH_TIMEOUT -o ServerAliveInterval=10 -o ServerAliveCountMax=3"

# URLs for health checks
FRONTEND_URL="https://pounce.ch"
API_URL="https://pounce.ch/api/v1/health"

# Log file
LOG_FILE="/tmp/pounce-deploy-$(date +%Y%m%d-%H%M%S).log"

# ============================================================================
# HELPER FUNCTIONS
# ============================================================================

log() {
    local msg="[$(date '+%H:%M:%S')] $1"
    echo -e "$msg" | tee -a "$LOG_FILE"
}

log_success() { log "${GREEN}✓ $1${NC}"; }
log_error() { log "${RED}✗ $1${NC}"; }
log_warn() { log "${YELLOW}⚠ $1${NC}"; }
log_info() { log "${BLUE}→ $1${NC}"; }
log_debug() { log "${GRAY}  $1${NC}"; }

# Check if command exists
require_cmd() {
    if ! command -v "$1" >/dev/null 2>&1; then
        log_error "$1 is required but not installed"
        if [ "$1" = "sshpass" ]; then
            echo -e "  Install with: ${CYAN}brew install hudochenkov/sshpass/sshpass${NC}"
        fi
        exit 1
    fi
}

# ============================================================================
# CONNECTION FUNCTIONS
# ============================================================================

# Find working server address
find_server() {
    log_info "Finding reachable server..."
    
    for host in "${SERVER_HOSTS[@]}"; do
        log_debug "Trying $host..."
        if curl -s --connect-timeout 5 "https://$host" >/dev/null 2>&1 || \
           curl -s --connect-timeout 5 "http://$host" >/dev/null 2>&1; then
            ACTIVE_HOST="$host"
            log_success "Server reachable via HTTPS at $host"
            return 0
        fi
    done
    
    log_error "No server reachable"
    return 1
}

# Test SSH connection with retries
test_ssh() {
    local host="$1"
    local retries="${2:-$SSH_RETRIES}"
    
    for i in $(seq 1 $retries); do
        if sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$host" "echo 'SSH OK'" >/dev/null 2>&1; then
            return 0
        fi
        if [ $i -lt $retries ]; then
            log_debug "Retry $i/$retries in ${i}s..."
            sleep $((i * 2))
        fi
    done
    return 1
}

# Find working SSH connection
find_ssh() {
    log_info "Testing SSH connections..."
    
    for host in "${SERVER_HOSTS[@]}"; do
        log_debug "Trying SSH to $host..."
        if test_ssh "$host" 2; then
            SSH_HOST="$host"
            log_success "SSH connected to $host"
            return 0
        fi
    done
    
    SSH_HOST=""
    log_warn "No SSH connection available"
    return 1
}

# Execute remote command with timeout
remote_exec() {
    local cmd="$1"
    local timeout="${2:-1}"  # 1=no timeout limit for builds
    
    if [ -z "$SSH_HOST" ]; then
        log_error "No SSH connection"
        return 1
    fi
    
    sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$SSH_HOST" "$cmd" 2>&1 | tee -a "$LOG_FILE"
    return ${PIPESTATUS[0]}
}

# ============================================================================
# HEALTH CHECK FUNCTIONS
# ============================================================================

check_api_health() {
    log_info "Checking API health..."
    
    local status
    status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "$API_URL" 2>/dev/null)
    
    if [ "$status" = "200" ]; then
        log_success "API is healthy"
        return 0
    else
        log_error "API health check failed (HTTP $status)"
        return 1
    fi
}

check_frontend_health() {
    log_info "Checking frontend health..."
    
    local status
    status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "$FRONTEND_URL" 2>/dev/null)
    
    if [ "$status" = "200" ]; then
        log_success "Frontend is healthy (HTTP $status)"
        return 0
    else
        log_error "Frontend health check failed (HTTP $status)"
        return 1
    fi
}

# ============================================================================
# SYNC FUNCTIONS
# ============================================================================

sync_backend() {
    log_info "Syncing backend files..."
    
    local host="${SSH_HOST:-$ACTIVE_HOST}"
    
    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \
        -avz --delete --compress-level=9 --checksum \
        --exclude '__pycache__' \
        --exclude '.pytest_cache' \
        --exclude 'venv' \
        --exclude '.git' \
        --exclude '*.pyc' \
        --exclude '.env' \
        --exclude '*.db' \
        --exclude 'logs/' \
        backend/ "$SERVER_USER@$host:$SERVER_PATH/backend/" 2>&1 | tee -a "$LOG_FILE"
    
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        log_success "Backend files synced"
        return 0
    else
        log_error "Backend sync failed"
        return 1
    fi
}

sync_frontend() {
    log_info "Syncing frontend files..."
    
    local host="${SSH_HOST:-$ACTIVE_HOST}"
    
    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \
        -avz --delete --compress-level=9 --checksum \
        --exclude 'node_modules' \
        --exclude '.next' \
        --exclude '.git' \
        frontend/ "$SERVER_USER@$host:$SERVER_PATH/frontend/" 2>&1 | tee -a "$LOG_FILE"
    
    if [ ${PIPESTATUS[0]} -eq 0 ]; then
        log_success "Frontend files synced"
        return 0
    else
        log_error "Frontend sync failed"
        return 1
    fi
}

# ============================================================================
# DEPLOY FUNCTIONS
# ============================================================================

deploy_backend() {
    log_info "Deploying backend..."
    
    if [ -z "$SSH_HOST" ]; then
        log_warn "SSH not available, backend will use synced files on next restart"
        return 0
    fi
    
    remote_exec "
        cd $SERVER_PATH/backend
        
        # Activate virtualenv
        if [ -f 'venv/bin/activate' ]; then
            source venv/bin/activate
        else
            echo 'venv not found, creating...'
            python3 -m venv venv
            source venv/bin/activate
            pip install -r requirements.txt
        fi
        
        # Run migrations
        echo 'Running database migrations...'
        python -c 'from app.database import init_db; import asyncio; asyncio.run(init_db())' 2>&1 || true
        
        # Graceful restart (SIGHUP for uvicorn)
        if systemctl is-active --quiet pounce-backend 2>/dev/null; then
            echo 'Graceful backend restart via systemd...'
            echo '$SERVER_PASS' | sudo -S systemctl reload-or-restart pounce-backend
            sleep 2
        else
            echo 'Starting backend with nohup...'
            pkill -f 'uvicorn app.main:app' 2>/dev/null || true
            sleep 1
            cd $SERVER_PATH/backend
            source venv/bin/activate
            nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 > /tmp/backend.log 2>&1 &
            sleep 3
        fi
        
        echo 'Backend deployment complete'
    " 3
    
    return $?
}

# ZERO-DOWNTIME FRONTEND DEPLOYMENT
deploy_frontend_zero_downtime() {
    log_info "Zero-downtime frontend deployment..."
    
    if [ -z "$SSH_HOST" ]; then
        log_warn "SSH not available, cannot build frontend remotely"
        return 1
    fi
    
    remote_exec "
        cd $SERVER_PATH/frontend
        
        # Create build timestamp for tracking
        BUILD_ID=\$(date +%Y%m%d-%H%M%S)
        echo \"Starting build \$BUILD_ID while server continues running...\"
        
        # Check if dependencies need update
        LOCKFILE_HASH=''
        if [ -f '.lockfile_hash' ]; then
            LOCKFILE_HASH=\$(cat .lockfile_hash)
        fi
        CURRENT_HASH=\$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo 'none')
        
        if [ \"\$LOCKFILE_HASH\" != \"\$CURRENT_HASH\" ]; then
            echo 'Installing dependencies...'
            npm ci --prefer-offline --no-audit --no-fund
            echo \"\$CURRENT_HASH\" > .lockfile_hash
        else
            echo 'Dependencies up to date (skipping npm ci)'
        fi
        
        # ===== CRITICAL: Build WHILE old server still runs =====
        echo ''
        echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
        echo '🚀 Building new version (server still running)...'
        echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
        echo ''
        
        # Build to .next directory
        NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS='--max-old-space-size=2048' npm run build
        
        if [ \$? -ne 0 ]; then
            echo '❌ Build failed! Server continues with old version.'
            exit 1
        fi
        
        echo ''
        echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
        echo '✅ Build successful! Preparing atomic switchover...'
        echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
        echo ''
        
        # Setup standalone directory with new build
        mkdir -p .next/standalone/.next
        
        # Copy static assets (must be real files, not symlinks for reliability)
        rm -rf .next/standalone/.next/static
        cp -r .next/static .next/standalone/.next/
        
        rm -rf .next/standalone/public
        cp -r public .next/standalone/public
        
        echo 'New build prepared. Starting atomic switchover...'
        
        # ===== ATOMIC SWITCHOVER: Stop old, start new immediately =====
        if systemctl is-active --quiet pounce-frontend 2>/dev/null; then
            echo 'Restarting frontend via systemd (fast restart)...'
            echo '$SERVER_PASS' | sudo -S systemctl restart pounce-frontend
            sleep 2
        else
            # Manual restart - minimize gap
            echo 'Manual restart - minimizing downtime...'
            
            # Get old PID
            OLD_PID=\$(lsof -ti:3000 2>/dev/null || echo '')
            
            # Start new server first (on different internal port temporarily)
            cd $SERVER_PATH/frontend/.next/standalone
            NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3001 BACKEND_URL=http://127.0.0.1:8000 node server.js &
            NEW_PID=\$!
            sleep 3
            
            # Verify new server is healthy
            if curl -s -o /dev/null -w '%{http_code}' http://localhost:3001 | grep -q '200'; then
                echo 'New server healthy on port 3001'
                
                # Kill old server
                if [ -n \"\$OLD_PID\" ]; then
                    kill -9 \$OLD_PID 2>/dev/null || true
                fi
                
                # Kill new server on temp port and restart on correct port
                kill -9 \$NEW_PID 2>/dev/null || true
                sleep 1
                
                # Start on correct port
                cd $SERVER_PATH/frontend/.next/standalone
                nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node server.js > /tmp/frontend.log 2>&1 &
                sleep 2
                echo 'New server running on port 3000'
            else
                echo '⚠️ New server failed health check, keeping old server'
                kill -9 \$NEW_PID 2>/dev/null || true
                exit 1
            fi
        fi
        
        echo ''
        echo '✅ Zero-downtime deployment complete!'
        echo \"Build ID: \$BUILD_ID\"
    " 1
    
    return $?
}

# Legacy deploy (with downtime) - kept as fallback
deploy_frontend_legacy() {
    log_info "Deploying frontend (legacy mode with downtime)..."
    
    if [ -z "$SSH_HOST" ]; then
        log_warn "SSH not available, cannot build frontend remotely"
        return 1
    fi
    
    remote_exec "
        cd $SERVER_PATH/frontend
        
        # Stop server during build
        echo 'Stopping server for rebuild...'
        if systemctl is-active --quiet pounce-frontend 2>/dev/null; then
            echo '$SERVER_PASS' | sudo -S systemctl stop pounce-frontend
        else
            pkill -f 'node .next/standalone/server.js' 2>/dev/null || true
            lsof -ti:3000 | xargs -r kill -9 2>/dev/null || true
        fi
        
        # Install & build
        npm ci --prefer-offline --no-audit --no-fund
        NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS='--max-old-space-size=2048' npm run build
        
        # Setup standalone
        mkdir -p .next/standalone/.next
        rm -rf .next/standalone/.next/static
        cp -r .next/static .next/standalone/.next/
        rm -rf .next/standalone/public
        cp -r public .next/standalone/public
        
        # Start server
        if systemctl is-active --quiet pounce-frontend 2>/dev/null; then
            echo '$SERVER_PASS' | sudo -S systemctl start pounce-frontend
        else
            cd $SERVER_PATH/frontend/.next/standalone
            nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node server.js > /tmp/frontend.log 2>&1 &
        fi
        sleep 3
        echo 'Frontend deployment complete'
    " 1
    
    return $?
}

# ============================================================================
# GIT FUNCTIONS
# ============================================================================

git_commit_push() {
    local msg="${1:-Deploy: $(date '+%Y-%m-%d %H:%M')}"
    
    log_info "Git operations..."
    
    # Check for changes
    if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
        log_debug "No changes to commit"
    else
        git add -A
        git commit -m "$msg" 2>&1 | tee -a "$LOG_FILE" || true
        log_success "Committed: $msg"
    fi
    
    # Push
    if git push origin main 2>&1 | tee -a "$LOG_FILE"; then
        log_success "Pushed to remote"
    else
        log_warn "Push failed or nothing to push"
    fi
}

# ============================================================================
# MAIN DEPLOY FUNCTION
# ============================================================================

deploy() {
    local mode="${1:-full}"
    local commit_msg="${2:-}"
    
    echo -e "\n${BOLD}${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
    echo -e "${BOLD}${BLUE}║      POUNCE ZERO-DOWNTIME DEPLOY v3.0                         ║${NC}"
    echo -e "${BOLD}${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}\n"
    
    log_info "Mode: ${CYAN}$mode${NC}"
    log_info "Log: ${CYAN}$LOG_FILE${NC}"
    
    local errors=0
    local start_time=$(date +%s)
    
    # Phase 1: Connectivity
    echo -e "\n${BOLD}[1/5] Connectivity${NC}"
    find_server || { log_error "Cannot reach server"; exit 1; }
    find_ssh || log_warn "SSH unavailable - sync-only mode"
    
    # Phase 2: Pre-deploy health check
    echo -e "\n${BOLD}[2/5] Pre-deploy Health Check${NC}"
    check_api_health || ((errors++))
    check_frontend_health || ((errors++))
    
    # Phase 3: Git (skip in quick mode)
    echo -e "\n${BOLD}[3/5] Git${NC}"
    if [ "$mode" = "quick" ] || [ "$mode" = "sync" ]; then
        echo -e " ${GRAY}(skipped)${NC}"
    else
        git_commit_push "$commit_msg"
    fi
    
    # Phase 4: Sync & Deploy
    echo -e "\n${BOLD}[4/5] Sync & Deploy${NC}"
    
    case "$mode" in
        backend)
            sync_backend || ((errors++))
            deploy_backend || ((errors++))
            ;;
        frontend)
            sync_frontend || ((errors++))
            deploy_frontend_zero_downtime || ((errors++))
            ;;
        sync)
            sync_backend || ((errors++))
            sync_frontend || ((errors++))
            ;;
        *)
            # Full or quick deploy
            sync_backend || ((errors++))
            sync_frontend || ((errors++))
            deploy_backend || ((errors++))
            deploy_frontend_zero_downtime || ((errors++))
            ;;
    esac
    
    # Phase 5: Post-deploy health check
    echo -e "\n${BOLD}[5/5] Post-deploy Health Check${NC}"
    sleep 3  # Give services time to start
    check_api_health || ((errors++))
    check_frontend_health || ((errors++))
    
    # Summary
    local end_time=$(date +%s)
    local duration=$((end_time - start_time))
    
    echo -e "\n${BOLD}════════════════════════════════════════════════════════════════${NC}"
    if [ $errors -eq 0 ]; then
        echo -e "${GREEN}${BOLD}✅ ZERO-DOWNTIME DEPLOY SUCCESSFUL${NC} (${duration}s)"
    else
        echo -e "${RED}${BOLD}⚠️  DEPLOY COMPLETED WITH $errors ERROR(S)${NC} (${duration}s)"
    fi
    echo -e "${BOLD}════════════════════════════════════════════════════════════════${NC}\n"
    
    echo -e "  ${CYAN}Frontend:${NC} $FRONTEND_URL"
    echo -e "  ${CYAN}API:${NC}      $API_URL"
    echo -e "  ${CYAN}Log:${NC}      $LOG_FILE"
    echo ""
    
    return $errors
}

# ============================================================================
# CLI INTERFACE
# ============================================================================

show_help() {
    echo "Usage: $0 [command] [options]"
    echo ""
    echo "Commands:"
    echo "  full        Full deploy (default) - git, sync, build, restart"
    echo "  quick       Skip git commit/push"
    echo "  backend     Deploy backend only"
    echo "  frontend    Deploy frontend only"
    echo "  sync        Sync files only (no build/restart)"
    echo "  status      Show server status"
    echo "  health      Run health checks only"
    echo "  legacy      Use legacy deploy (with downtime)"
    echo ""
    echo "Options:"
    echo "  -m MSG      Commit message"
    echo "  -h          Show this help"
    echo ""
    echo "Examples:"
    echo "  $0                    # Full zero-downtime deploy"
    echo "  $0 quick              # Quick deploy (skip git)"
    echo "  $0 frontend           # Frontend only"
    echo "  $0 -m 'feat: new'     # Full with commit message"
}

# Main
main() {
    require_cmd sshpass
    require_cmd rsync
    require_cmd curl
    require_cmd git
    
    local command="full"
    local commit_msg=""
    
    while [[ $# -gt 0 ]]; do
        case $1 in
            full|quick|backend|frontend|sync)
                command="$1"
                shift
                ;;
            legacy)
                # Override frontend deploy function
                deploy_frontend_zero_downtime() { deploy_frontend_legacy; }
                command="full"
                shift
                ;;
            status)
                find_server && find_ssh
                if [ -n "$SSH_HOST" ]; then
                    remote_exec "
                        echo '=== Services ==='
                        systemctl status pounce-backend --no-pager 2>/dev/null | head -5 || echo 'Backend: manual mode'
                        systemctl status pounce-frontend --no-pager 2>/dev/null | head -5 || echo 'Frontend: manual mode'
                        echo ''
                        echo '=== Ports ==='
                        ss -tlnp | grep -E ':(3000|8000)' || echo 'No services on expected ports'
                    "
                fi
                exit 0
                ;;
            health)
                find_server
                check_api_health
                check_frontend_health
                exit 0
                ;;
            -m)
                shift
                commit_msg="$1"
                shift
                ;;
            -h|--help)
                show_help
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                show_help
                exit 1
                ;;
        esac
    done
    
    deploy "$command" "$commit_msg"
}

main "$@"