feat: Robust deploy pipeline v2.0

- Multiple server addresses with automatic fallback
- SSH retry logic with exponential backoff
- Health checks before and after deployment
- HTTP-based deployment endpoint as backup
- Better error handling and logging
- Support for partial deployments (backend/frontend only)

backend/app/api/__init__.py

@@ -30,6 +30,7 @@ from app.api.drops import router as drops_router
 from app.api.llm import router as llm_router
 from app.api.llm_naming import router as llm_naming_router
 from app.api.llm_vision import router as llm_vision_router
+from app.api.deploy import router as deploy_router
 
 api_router = APIRouter()
 
@@ -81,3 +82,6 @@ api_router.include_router(blog_router, prefix="/blog", tags=["Blog"])
 
 # Admin endpoints
 api_router.include_router(admin_router, prefix="/admin", tags=["Admin"])
+
+# Deploy endpoint (internal use only)
+api_router.include_router(deploy_router, tags=["Deploy"])

backend/app/api/deploy.py

@@ -0,0 +1,231 @@
+"""
+Remote Deploy Endpoint
+
+This provides a secure way to trigger deployments remotely when SSH is not available.
+Protected by an internal API key that should be kept secret.
+"""
+
+import asyncio
+import logging
+import os
+import subprocess
+from datetime import datetime
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Header, BackgroundTasks
+from pydantic import BaseModel
+
+from app.config import get_settings
+
+router = APIRouter(prefix="/deploy", tags=["deploy"])
+logger = logging.getLogger(__name__)
+
+settings = get_settings()
+
+
+class DeployStatus(BaseModel):
+    """Response model for deploy status."""
+    status: str
+    message: str
+    timestamp: str
+    details: Optional[dict] = None
+
+
+class DeployRequest(BaseModel):
+    """Request model for deploy trigger."""
+    component: str = "all"  # all, backend, frontend
+    git_pull: bool = True
+
+
+def run_command(cmd: str, cwd: str = None, timeout: int = 300) -> tuple[int, str, str]:
+    """Run a shell command and return exit code, stdout, stderr."""
+    try:
+        result = subprocess.run(
+            cmd,
+            shell=True,
+            cwd=cwd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return result.returncode, result.stdout, result.stderr
+    except subprocess.TimeoutExpired:
+        return -1, "", f"Command timed out after {timeout}s"
+    except Exception as e:
+        return -1, "", str(e)
+
+
+async def run_deploy(component: str, git_pull: bool) -> dict:
+    """
+    Execute deployment steps.
+    
+    This runs in the background to not block the HTTP response.
+    """
+    results = {
+        "started_at": datetime.utcnow().isoformat(),
+        "steps": [],
+    }
+    
+    base_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    
+    try:
+        # Step 1: Git pull (if requested)
+        if git_pull:
+            logger.info("Deploy: Running git pull...")
+            code, stdout, stderr = run_command("git pull origin main", cwd=base_path, timeout=60)
+            results["steps"].append({
+                "step": "git_pull",
+                "success": code == 0,
+                "output": stdout or stderr,
+            })
+            if code != 0:
+                logger.error(f"Git pull failed: {stderr}")
+        
+        # Step 2: Backend deployment
+        if component in ("all", "backend"):
+            logger.info("Deploy: Restarting backend...")
+            
+            # Try systemctl first
+            code, stdout, stderr = run_command("sudo systemctl restart pounce-backend", timeout=30)
+            
+            if code == 0:
+                results["steps"].append({
+                    "step": "backend_restart",
+                    "method": "systemctl",
+                    "success": True,
+                })
+            else:
+                # Fallback: Send SIGHUP to reload
+                code, stdout, stderr = run_command("pkill -HUP -f 'uvicorn app.main:app'", timeout=10)
+                results["steps"].append({
+                    "step": "backend_restart",
+                    "method": "sighup",
+                    "success": code == 0,
+                    "output": stderr if code != 0 else None,
+                })
+        
+        # Step 3: Frontend deployment (more complex)
+        if component in ("all", "frontend"):
+            logger.info("Deploy: Rebuilding frontend...")
+            
+            frontend_path = os.path.join(os.path.dirname(base_path), "frontend")
+            
+            # Build frontend
+            build_cmd = "npm run build"
+            code, stdout, stderr = run_command(
+                f"cd {frontend_path} && {build_cmd}",
+                timeout=300,  # 5 min for build
+            )
+            
+            results["steps"].append({
+                "step": "frontend_build",
+                "success": code == 0,
+                "output": stderr[-500:] if code != 0 else "Build successful",
+            })
+            
+            if code == 0:
+                # Copy public files for standalone
+                run_command(
+                    f"cp -r {frontend_path}/public {frontend_path}/.next/standalone/",
+                    timeout=30,
+                )
+                
+                # Restart frontend
+                code, stdout, stderr = run_command("sudo systemctl restart pounce-frontend", timeout=30)
+                
+                if code != 0:
+                    # Fallback
+                    run_command("pkill -f 'node .next/standalone/server.js'", timeout=10)
+                    run_command(
+                        f"cd {frontend_path}/.next/standalone && nohup node server.js > /tmp/frontend.log 2>&1 &",
+                        timeout=10,
+                    )
+                
+                results["steps"].append({
+                    "step": "frontend_restart",
+                    "success": True,
+                })
+        
+        results["completed_at"] = datetime.utcnow().isoformat()
+        results["success"] = all(s.get("success", False) for s in results["steps"])
+        
+    except Exception as e:
+        logger.exception(f"Deploy failed: {e}")
+        results["error"] = str(e)
+        results["success"] = False
+    
+    logger.info(f"Deploy completed: {results}")
+    return results
+
+
+# Store last deploy result
+_last_deploy_result: Optional[dict] = None
+
+
+@router.post("/trigger", response_model=DeployStatus)
+async def trigger_deploy(
+    request: DeployRequest,
+    background_tasks: BackgroundTasks,
+    x_deploy_key: str = Header(..., alias="X-Deploy-Key"),
+):
+    """
+    Trigger a deployment remotely.
+    
+    Requires X-Deploy-Key header matching the internal_api_key setting.
+    
+    This starts the deployment in the background and returns immediately.
+    Check /deploy/status for results.
+    """
+    global _last_deploy_result
+    
+    # Verify deploy key
+    expected_key = settings.internal_api_key
+    if not expected_key or x_deploy_key != expected_key:
+        raise HTTPException(status_code=403, detail="Invalid deploy key")
+    
+    # Start deployment in background
+    async def do_deploy():
+        global _last_deploy_result
+        _last_deploy_result = await run_deploy(request.component, request.git_pull)
+    
+    background_tasks.add_task(do_deploy)
+    
+    return DeployStatus(
+        status="started",
+        message=f"Deployment started for component: {request.component}",
+        timestamp=datetime.utcnow().isoformat(),
+    )
+
+
+@router.get("/status", response_model=DeployStatus)
+async def get_deploy_status(
+    x_deploy_key: str = Header(..., alias="X-Deploy-Key"),
+):
+    """
+    Get the status of the last deployment.
+    
+    Requires X-Deploy-Key header.
+    """
+    expected_key = settings.internal_api_key
+    if not expected_key or x_deploy_key != expected_key:
+        raise HTTPException(status_code=403, detail="Invalid deploy key")
+    
+    if _last_deploy_result is None:
+        return DeployStatus(
+            status="none",
+            message="No deployments have been triggered",
+            timestamp=datetime.utcnow().isoformat(),
+        )
+    
+    return DeployStatus(
+        status="completed" if _last_deploy_result.get("success") else "failed",
+        message="Last deployment result",
+        timestamp=_last_deploy_result.get("completed_at", "unknown"),
+        details=_last_deploy_result,
+    )
+
+
+@router.get("/health")
+async def deploy_health():
+    """Simple health check for deploy endpoint."""
+    return {"status": "ok", "message": "Deploy endpoint available"}

deploy-http.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# ============================================================================
+# POUNCE HTTP DEPLOY (Backup method when SSH is unavailable)
+#
+# This uses the /api/v1/deploy endpoint to trigger deployments remotely.
+# Requires the internal API key to be configured in the backend.
+# ============================================================================
+
+set -euo pipefail
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# Configuration
+API_URL="https://pounce.ch/api/v1"
+DEPLOY_KEY="${POUNCE_DEPLOY_KEY:-}"
+
+# Check if deploy key is set
+if [ -z "$DEPLOY_KEY" ]; then
+    echo -e "${RED}Error: POUNCE_DEPLOY_KEY environment variable not set${NC}"
+    echo ""
+    echo "Set your deploy key:"
+    echo "  export POUNCE_DEPLOY_KEY=your-internal-api-key"
+    echo ""
+    echo "The key should match the 'internal_api_key' in your backend .env file"
+    exit 1
+fi
+
+show_help() {
+    echo -e "${CYAN}Pounce HTTP Deploy${NC}"
+    echo ""
+    echo "Usage: ./deploy-http.sh [component]"
+    echo ""
+    echo "Components:"
+    echo "  all      Deploy both backend and frontend (default)"
+    echo "  backend  Deploy backend only"
+    echo "  frontend Deploy frontend only"
+    echo "  status   Check last deployment status"
+    echo ""
+}
+
+trigger_deploy() {
+    local component="${1:-all}"
+    
+    echo -e "${CYAN}Triggering deployment for: $component${NC}"
+    
+    response=$(curl -s -X POST "$API_URL/deploy/trigger" \
+        -H "Content-Type: application/json" \
+        -H "X-Deploy-Key: $DEPLOY_KEY" \
+        -d "{\"component\": \"$component\", \"git_pull\": true}")
+    
+    if echo "$response" | grep -q '"status":"started"'; then
+        echo -e "${GREEN}✓ Deployment started${NC}"
+        echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response"
+        
+        echo ""
+        echo -e "${YELLOW}Deployment running in background...${NC}"
+        echo "Check status with: ./deploy-http.sh status"
+    else
+        echo -e "${RED}✗ Failed to trigger deployment${NC}"
+        echo "$response"
+        exit 1
+    fi
+}
+
+check_status() {
+    echo -e "${CYAN}Checking deployment status...${NC}"
+    
+    response=$(curl -s "$API_URL/deploy/status" \
+        -H "X-Deploy-Key: $DEPLOY_KEY")
+    
+    if echo "$response" | grep -q '"status"'; then
+        echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response"
+    else
+        echo -e "${RED}✗ Failed to get status${NC}"
+        echo "$response"
+        exit 1
+    fi
+}
+
+# Main
+case "${1:-all}" in
+    help|-h|--help)
+        show_help
+        ;;
+    status)
+        check_status
+        ;;
+    *)
+        trigger_deploy "$1"
+        ;;
+esac

deploy.sh

@@ -1,14 +1,18 @@
 #!/bin/bash
 
 # ============================================================================
-# POUNCE ZERO-DOWNTIME DEPLOY SCRIPT
-# - Builds locally first (optional)
-# - Syncs only changed files
-# - Hot-reloads backend without full restart
-# - Rebuilds frontend in background
+# POUNCE ROBUST DEPLOY PIPELINE v2.0
+# 
+# Features:
+# - Multiple connection methods (DNS, public IP, internal IP)
+# - Automatic retry with exponential backoff
+# - Health checks before and after deployment
+# - Parallel file sync for speed
+# - Graceful rollback on failure
+# - Detailed logging
 # ============================================================================
 
-set -euo pipefail
+set -uo pipefail
 
 # Colors
 GREEN='\033[0;32m'
@@ -16,96 +20,232 @@ YELLOW='\033[1;33m'
 BLUE='\033[0;34m'
 RED='\033[0;31m'
 CYAN='\033[0;36m'
-NC='\033[0m' # No Color
+GRAY='\033[0;90m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+# ============================================================================
+# CONFIGURATION
+# ============================================================================
 
-# Server config
 SERVER_USER="user"
-SERVER_HOST="10.42.0.73"
-SERVER_PATH="/home/user/pounce"
 SERVER_PASS="user"
-SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-# Force a TTY for password auth + sudo on some hosts
-SSH_CMD="ssh -tt $SSH_OPTS -o PreferredAuthentications=password -o PubkeyAuthentication=no"
+SERVER_PATH="/home/user/pounce"
 
-if ! command -v sshpass >/dev/null 2>&1; then
-    echo -e "${RED}✗ sshpass is required but not installed.${NC}"
-    echo -e "  Install with: ${CYAN}brew install sshpass${NC}"
-    exit 1
-fi
+# Multiple server addresses to try (in order of preference)
+declare -a SERVER_HOSTS=(
+    "pounce.ch"
+    "46.235.147.194"
+    "10.42.0.73"
+)
 
-# Parse flags
-QUICK_MODE=false
-BACKEND_ONLY=false
-FRONTEND_ONLY=false
-COMMIT_MSG=""
+# SSH options
+SSH_TIMEOUT=15
+SSH_RETRIES=3
+SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=$SSH_TIMEOUT -o ServerAliveInterval=10 -o ServerAliveCountMax=3"
 
-while [[ "$#" -gt 0 ]]; do
-    case $1 in
-        -q|--quick) QUICK_MODE=true ;;
-        -b|--backend) BACKEND_ONLY=true ;;
-        -f|--frontend) FRONTEND_ONLY=true ;;
-        *) COMMIT_MSG="$1" ;;
-    esac
-    shift
-done
+# URLs for health checks
+FRONTEND_URL="https://pounce.ch"
+API_URL="https://pounce.ch/api/v1/health"
 
-echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
-echo -e "${BLUE}              POUNCE ZERO-DOWNTIME DEPLOY                      ${NC}"
-echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
+# Log file
+LOG_FILE="/tmp/pounce-deploy-$(date +%Y%m%d-%H%M%S).log"
 
-if $QUICK_MODE; then
-    echo -e "${CYAN}⚡ Quick mode: Skipping git, only syncing changes${NC}"
-fi
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
 
-if $BACKEND_ONLY; then
-    echo -e "${CYAN}🔧 Backend only mode${NC}"
-fi
+log() {
+    local msg="[$(date '+%H:%M:%S')] $1"
+    echo -e "$msg" | tee -a "$LOG_FILE"
+}
 
-if $FRONTEND_ONLY; then
-    echo -e "${CYAN}🎨 Frontend only mode${NC}"
-fi
+log_success() { log "${GREEN}✓ $1${NC}"; }
+log_error() { log "${RED}✗ $1${NC}"; }
+log_warn() { log "${YELLOW}⚠ $1${NC}"; }
+log_info() { log "${BLUE}→ $1${NC}"; }
+log_debug() { log "${GRAY}  $1${NC}"; }
 
-# Step 1: Git (unless quick mode)
-if ! $QUICK_MODE; then
-    echo -e "\n${YELLOW}[1/4] Git operations...${NC}"
-    
-    # Check for changes (including untracked)
-    if [ -z "$(git status --porcelain)" ]; then
-        echo "  No changes to commit"
-    else
-        git add -A
+spinner() {
+    local pid=$1
+    local delay=0.1
+    local spinstr='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
+    while kill -0 "$pid" 2>/dev/null; do
+        local temp=${spinstr#?}
+        printf " %c  " "$spinstr"
+        local spinstr=$temp${spinstr%"$temp"}
+        sleep $delay
+        printf "\b\b\b\b"
+    done
+    printf "    \b\b\b\b"
+}
 
-        if [ -z "$COMMIT_MSG" ]; then
-            COMMIT_MSG="Deploy: $(date '+%Y-%m-%d %H:%M')"
+# Check if command exists
+require_cmd() {
+    if ! command -v "$1" >/dev/null 2>&1; then
+        log_error "$1 is required but not installed"
+        if [ "$1" = "sshpass" ]; then
+            echo -e "  Install with: ${CYAN}brew install hudochenkov/sshpass/sshpass${NC}"
         fi
+        exit 1
+    fi
+}
 
-        git commit -m "$COMMIT_MSG" || true
-        echo "  ✓ Committed: $COMMIT_MSG"
+# ============================================================================
+# CONNECTION FUNCTIONS
+# ============================================================================
+
+# Find working server address
+find_server() {
+    log_info "Finding reachable server..."
+    
+    for host in "${SERVER_HOSTS[@]}"; do
+        log_debug "Trying $host..."
+        
+        # Try HTTP first (faster, more reliable)
+        if curl -s --connect-timeout 5 --max-time 10 "http://$host:8000/health" >/dev/null 2>&1; then
+            log_success "Server reachable via HTTP at $host"
+            ACTIVE_HOST="$host"
+            return 0
+        fi
+        
+        # Try HTTPS
+        if curl -s --connect-timeout 5 --max-time 10 "https://$host/api/v1/health" >/dev/null 2>&1; then
+            log_success "Server reachable via HTTPS at $host"
+            ACTIVE_HOST="$host"
+            return 0
+        fi
+    done
+    
+    log_error "No reachable server found!"
+    return 1
+}
+
+# Test SSH connection
+test_ssh() {
+    local host="$1"
+    sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$host" "echo 'SSH OK'" >/dev/null 2>&1
+    return $?
+}
+
+# Find working SSH connection
+find_ssh() {
+    log_info "Testing SSH connections..."
+    
+    for host in "${SERVER_HOSTS[@]}"; do
+        log_debug "Trying SSH to $host..."
+        
+        for attempt in $(seq 1 $SSH_RETRIES); do
+            if test_ssh "$host"; then
+                log_success "SSH connected to $host"
+                SSH_HOST="$host"
+                return 0
+            fi
+            
+            if [ $attempt -lt $SSH_RETRIES ]; then
+                local wait=$((attempt * 2))
+                log_debug "Retry $attempt/$SSH_RETRIES in ${wait}s..."
+                sleep $wait
+            fi
+        done
+    done
+    
+    log_warn "SSH not available - will use rsync-only mode"
+    SSH_HOST=""
+    return 1
+}
+
+# Execute command on server with retries
+remote_exec() {
+    local cmd="$1"
+    local retries="${2:-3}"
+    
+    if [ -z "$SSH_HOST" ]; then
+        log_error "No SSH connection available"
+        return 1
     fi
     
-    git push origin main 2>/dev/null && echo "  ✓ Pushed to git.6bit.ch" || echo "  ⚠ Push failed or nothing to push"
-else
-    echo -e "\n${YELLOW}[1/4] Skipping git (quick mode)${NC}"
-fi
+    for attempt in $(seq 1 $retries); do
+        if sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$SSH_HOST" "$cmd" 2>&1; then
+            return 0
+        fi
+        
+        if [ $attempt -lt $retries ]; then
+            local wait=$((attempt * 2))
+            log_debug "Command failed, retry $attempt/$retries in ${wait}s..."
+            sleep $wait
+        fi
+    done
+    
+    return 1
+}
 
-# Step 2: Sync files (only changed)
-echo -e "\n${YELLOW}[2/4] Syncing changed files...${NC}"
+# ============================================================================
+# HEALTH CHECK FUNCTIONS
+# ============================================================================
 
-# Using compression (-z), checksum-based detection, and bandwidth throttling for stability
-RSYNC_OPTS="-avz --delete --compress-level=9 --checksum"
+check_api_health() {
+    log_info "Checking API health..."
+    
+    local response
+    response=$(curl -s --connect-timeout 10 --max-time 30 "$API_URL" 2>/dev/null)
+    
+    if echo "$response" | grep -q '"status":"healthy"'; then
+        log_success "API is healthy"
+        return 0
+    else
+        log_error "API health check failed"
+        log_debug "Response: $response"
+        return 1
+    fi
+}
 
-if ! $BACKEND_ONLY; then
-    echo "  Frontend:"
-    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" $RSYNC_OPTS \
-        --exclude 'node_modules' \
-        --exclude '.next' \
-        --exclude '.git' \
-        frontend/ $SERVER_USER@$SERVER_HOST:$SERVER_PATH/frontend/
-fi
+check_frontend_health() {
+    log_info "Checking frontend health..."
+    
+    local status
+    status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "$FRONTEND_URL" 2>/dev/null)
+    
+    if [ "$status" = "200" ]; then
+        log_success "Frontend is healthy (HTTP $status)"
+        return 0
+    else
+        log_error "Frontend health check failed (HTTP $status)"
+        return 1
+    fi
+}
 
-if ! $FRONTEND_ONLY; then
-    echo "  Backend:"
-    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" $RSYNC_OPTS \
+wait_for_healthy() {
+    local service="$1"
+    local max_wait="${2:-60}"
+    local check_func="check_${service}_health"
+    
+    log_info "Waiting for $service to be healthy (max ${max_wait}s)..."
+    
+    for i in $(seq 1 $max_wait); do
+        if $check_func 2>/dev/null; then
+            return 0
+        fi
+        sleep 1
+        printf "."
+    done
+    
+    echo ""
+    log_error "$service did not become healthy within ${max_wait}s"
+    return 1
+}
+
+# ============================================================================
+# SYNC FUNCTIONS
+# ============================================================================
+
+sync_backend() {
+    log_info "Syncing backend files..."
+    
+    local host="${SSH_HOST:-$ACTIVE_HOST}"
+    
+    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \
+        -avz --delete --compress-level=9 --checksum \
         --exclude '__pycache__' \
         --exclude '.pytest_cache' \
         --exclude 'venv' \
@@ -113,163 +253,352 @@ if ! $FRONTEND_ONLY; then
         --exclude '*.pyc' \
         --exclude '.env' \
         --exclude '*.db' \
-        backend/ $SERVER_USER@$SERVER_HOST:$SERVER_PATH/backend/
-fi
+        --exclude 'logs/' \
+        backend/ "$SERVER_USER@$host:$SERVER_PATH/backend/" 2>&1 | tee -a "$LOG_FILE"
+    
+    if [ ${PIPESTATUS[0]} -eq 0 ]; then
+        log_success "Backend files synced"
+        return 0
+    else
+        log_error "Backend sync failed"
+        return 1
+    fi
+}
 
-# Step 3: Reload backend (graceful, no restart)
-if ! $FRONTEND_ONLY; then
-    echo -e "\n${YELLOW}[3/4] Reloading backend (graceful)...${NC}"
-    sshpass -p "$SERVER_PASS" $SSH_CMD $SERVER_USER@$SERVER_HOST << 'BACKEND_EOF'
-        set -e
+sync_frontend() {
+    log_info "Syncing frontend files..."
+    
+    local host="${SSH_HOST:-$ACTIVE_HOST}"
+    
+    sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \
+        -avz --delete --compress-level=9 --checksum \
+        --exclude 'node_modules' \
+        --exclude '.next' \
+        --exclude '.git' \
+        frontend/ "$SERVER_USER@$host:$SERVER_PATH/frontend/" 2>&1 | tee -a "$LOG_FILE"
+    
+    if [ ${PIPESTATUS[0]} -eq 0 ]; then
+        log_success "Frontend files synced"
+        return 0
+    else
+        log_error "Frontend sync failed"
+        return 1
+    fi
+}
 
-        cd ~/pounce/backend
-        if [ -f "venv/bin/activate" ]; then
+# ============================================================================
+# DEPLOY FUNCTIONS
+# ============================================================================
+
+deploy_backend() {
+    log_info "Deploying backend..."
+    
+    if [ -z "$SSH_HOST" ]; then
+        log_warn "SSH not available, backend will use synced files on next restart"
+        return 0
+    fi
+    
+    remote_exec "
+        cd $SERVER_PATH/backend
+        
+        # Activate virtualenv
+        if [ -f 'venv/bin/activate' ]; then
             source venv/bin/activate
-        elif [ -f "../venv/bin/activate" ]; then
-            source ../venv/bin/activate
         else
-            echo "  ✗ venv not found (expected backend/venv or ../venv)"
-            exit 1
+            echo 'venv not found, creating...'
+            python3 -m venv venv
+            source venv/bin/activate
+            pip install -r requirements.txt
         fi
-
-        # Update CZDS credentials if not set
-        if ! grep -q "CZDS_USERNAME=" .env 2>/dev/null; then
-            echo "" >> .env
-            echo "# ICANN CZDS Zone File Service" >> .env
-            echo "CZDS_USERNAME=guggeryves@hotmail.com" >> .env
-            echo "CZDS_PASSWORD=Achiarorocco1278!" >> .env
-            echo "CZDS_DATA_DIR=/home/user/pounce_czds" >> .env
-            echo "  ✓ CZDS credentials added to .env"
-        else
-            echo "  ✓ CZDS credentials already configured"
-        fi
-
-        echo "  Running DB migrations..."
-        python -c "from app.database import init_db; import asyncio; asyncio.run(init_db())"
-        echo "  ✓ DB migrations applied"
-
-        # Restart backend via systemd when available (preferred). Fallback to nohup only if the unit is missing.
-        if [ -f "/etc/systemd/system/pounce-backend.service" ]; then
-            echo "  Restarting backend via systemd..."
-            echo "user" | sudo -S systemctl restart pounce-backend
-            sleep 2
-            if systemctl is-active --quiet pounce-backend; then
-                echo "  ✓ Backend restarted (systemd)"
-            else
-                echo "  ⚠ Backend restart failed (systemd). Check: journalctl -u pounce-backend -n 80"
-            fi
-        else
-            BACKEND_PID=$(pgrep -f 'uvicorn app.main:app' | awk 'NR==1{print; exit}')
-            if [ -n "$BACKEND_PID" ]; then
-                echo "  Restarting backend (PID: $BACKEND_PID)..."
-                kill "$BACKEND_PID" 2>/dev/null || true
-                sleep 1
-            else
-                echo "  ⚠ Backend not running, starting..."
-            fi
-            nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 > /tmp/pounce-backend-nohup.log 2>&1 &
-            sleep 2
-            echo "  ✓ Backend started (nohup fallback)"
-        fi
-BACKEND_EOF
-else
-    echo -e "\n${YELLOW}[3/4] Skipping backend (frontend only)${NC}"
-fi
-
-# Step 4: Rebuild frontend (in background to minimize downtime)
-if ! $BACKEND_ONLY; then
-    echo -e "\n${YELLOW}[4/4] Rebuilding frontend...${NC}"
-    sshpass -p "$SERVER_PASS" $SSH_CMD $SERVER_USER@$SERVER_HOST << 'FRONTEND_EOF'
-        set -e
-        cd ~/pounce/frontend
-
-        # Check if package.json changed (skip npm ci if not)
-        LOCKFILE_HASH=""
-        if [ -f ".lockfile_hash" ]; then
-            LOCKFILE_HASH=$(cat .lockfile_hash)
-        fi
-        CURRENT_HASH=$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo "none")
         
-        if [ "$LOCKFILE_HASH" != "$CURRENT_HASH" ]; then
-            echo "  Installing dependencies (package-lock.json changed)..."
+        # Run migrations
+        echo 'Running database migrations...'
+        python -c 'from app.database import init_db; import asyncio; asyncio.run(init_db())' 2>&1 || true
+        
+        # Restart service
+        if systemctl is-active --quiet pounce-backend 2>/dev/null; then
+            echo 'Restarting backend via systemd...'
+            echo '$SERVER_PASS' | sudo -S systemctl restart pounce-backend
+            sleep 3
+        else
+            echo 'Starting backend with nohup...'
+            pkill -f 'uvicorn app.main:app' 2>/dev/null || true
+            sleep 1
+            cd $SERVER_PATH/backend
+            source venv/bin/activate
+            nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 > /tmp/backend.log 2>&1 &
+            sleep 3
+        fi
+        
+        echo 'Backend deployment complete'
+    " 3
+    
+    return $?
+}
+
+deploy_frontend() {
+    log_info "Deploying frontend (this may take a few minutes)..."
+    
+    if [ -z "$SSH_HOST" ]; then
+        log_warn "SSH not available, cannot build frontend remotely"
+        return 1
+    fi
+    
+    remote_exec "
+        cd $SERVER_PATH/frontend
+        
+        # Check if dependencies need update
+        LOCKFILE_HASH=''
+        if [ -f '.lockfile_hash' ]; then
+            LOCKFILE_HASH=\$(cat .lockfile_hash)
+        fi
+        CURRENT_HASH=\$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo 'none')
+        
+        if [ \"\$LOCKFILE_HASH\" != \"\$CURRENT_HASH\" ]; then
+            echo 'Installing dependencies...'
             npm ci --prefer-offline --no-audit --no-fund
-            echo "$CURRENT_HASH" > .lockfile_hash
+            echo \"\$CURRENT_HASH\" > .lockfile_hash
         else
-            echo "  ✓ Dependencies unchanged, skipping npm ci"
+            echo 'Dependencies up to date'
         fi
         
-        # Build new version (with reduced memory for stability)
-        # Set NEXT_PUBLIC_API_URL for client-side API calls
-        echo "  Building..."
-        NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS="--max-old-space-size=2048" npm run build
-        BUILD_EXIT=$?
+        # Build
+        echo 'Building frontend...'
+        NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS='--max-old-space-size=2048' npm run build
         
-        if [ $BUILD_EXIT -eq 0 ]; then
-            # Next.js standalone output requires public + static inside standalone folder
+        if [ \$? -eq 0 ]; then
+            # Setup standalone
             mkdir -p .next/standalone/.next
             ln -sfn ../../static .next/standalone/.next/static
-            
-            # Copy public folder (symlinks don't work reliably)
             rm -rf .next/standalone/public
             cp -r public .next/standalone/public
-            echo "  ✓ Public files copied to standalone"
-
-            # Restart frontend via systemd when available (preferred). Fallback to nohup only if the unit is missing.
-            if [ -f "/etc/systemd/system/pounce-frontend.service" ]; then
-                echo "  Restarting frontend via systemd..."
-                echo "user" | sudo -S systemctl restart pounce-frontend
-                sleep 2
-                if systemctl is-active --quiet pounce-frontend; then
-                    echo "  ✓ Frontend restarted (systemd)"
-                else
-                    echo "  ⚠ Frontend restart failed (systemd). Check: journalctl -u pounce-frontend -n 80"
-                fi
+            
+            # Restart service
+            if systemctl is-active --quiet pounce-frontend 2>/dev/null; then
+                echo 'Restarting frontend via systemd...'
+                echo '$SERVER_PASS' | sudo -S systemctl restart pounce-frontend
+                sleep 3
             else
-                # Legacy nohup fallback
-                NEXT_PID=$(pgrep -af 'node \\.next/standalone/server\\.js|next start|next-server|next-serv' | awk 'NR==1{print $1; exit}')
-                if [ -n "$NEXT_PID" ]; then
-                    echo "  Restarting Next.js (PID: $NEXT_PID)..."
-                    kill $NEXT_PID 2>/dev/null
-                    sleep 1
-                fi
-                lsof -ti:3000 2>/dev/null | xargs -r kill -9 2>/dev/null || true
+                echo 'Starting frontend with nohup...'
+                pkill -f 'node .next/standalone/server.js' 2>/dev/null || true
+                lsof -ti:3000 | xargs -r kill -9 2>/dev/null || true
                 sleep 1
-                if [ -f ".next/standalone/server.js" ]; then
-                    echo "  Starting Next.js (standalone)..."
-                    nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node .next/standalone/server.js > /tmp/pounce-frontend-nohup.log 2>&1 &
-                else
-                    echo "  Starting Next.js (npm start)..."
-                    nohup env NODE_ENV=production BACKEND_URL=http://127.0.0.1:8000 npm run start > /tmp/pounce-frontend-nohup.log 2>&1 &
-                fi
-                sleep 2
-                NEW_PID=$(pgrep -af 'node \\.next/standalone/server\\.js|next start|next-server|next-serv' | awk 'NR==1{print $1; exit}')
-                if [ -n "$NEW_PID" ]; then
-                    echo "  ✓ Frontend running (nohup fallback, PID: $NEW_PID)"
-                else
-                    echo "  ⚠ Frontend may not have started correctly"
-                    tail -n 80 /tmp/pounce-frontend-nohup.log || true
-                fi
+                cd $SERVER_PATH/frontend
+                nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node .next/standalone/server.js > /tmp/frontend.log 2>&1 &
+                sleep 3
             fi
+            
+            echo 'Frontend deployment complete'
         else
-            echo "  ✗ Build failed, keeping old version"
-            echo "  Last 120 lines of build output (frontend.log):"
-            tail -n 120 frontend.log || true
+            echo 'Build failed!'
+            exit 1
         fi
-FRONTEND_EOF
-else
-    echo -e "\n${YELLOW}[4/4] Skipping frontend (backend only)${NC}"
-fi
+    " 1
+    
+    return $?
+}
 
-# Summary
-echo -e "\n${GREEN}═══════════════════════════════════════════════════════════════${NC}"
-echo -e "${GREEN}                    ✅ DEPLOY COMPLETE!                         ${NC}"
-echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}"
-echo -e ""
-echo -e "  Frontend: ${BLUE}https://pounce.ch${NC} / http://$SERVER_HOST:3000"
-echo -e "  Backend:  ${BLUE}https://pounce.ch/api${NC} / http://$SERVER_HOST:8000"
-echo -e ""
-echo -e "${CYAN}Quick commands:${NC}"
-echo -e "  ./deploy.sh -q          # Quick sync, no git"
-echo -e "  ./deploy.sh -b          # Backend only"
-echo -e "  ./deploy.sh -f          # Frontend only"
-echo -e "  ./deploy.sh \"message\"   # Full deploy with commit message"
+# ============================================================================
+# GIT FUNCTIONS
+# ============================================================================
+
+git_commit_push() {
+    local msg="${1:-Deploy: $(date '+%Y-%m-%d %H:%M')}"
+    
+    log_info "Git operations..."
+    
+    # Check for changes
+    if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
+        log_debug "No changes to commit"
+    else
+        git add -A
+        git commit -m "$msg" 2>&1 | tee -a "$LOG_FILE" || true
+        log_success "Committed: $msg"
+    fi
+    
+    # Push
+    if git push origin main 2>&1 | tee -a "$LOG_FILE"; then
+        log_success "Pushed to remote"
+    else
+        log_warn "Push failed or nothing to push"
+    fi
+}
+
+# ============================================================================
+# MAIN DEPLOY FUNCTION
+# ============================================================================
+
+deploy() {
+    local mode="${1:-full}"
+    local commit_msg="${2:-}"
+    
+    echo -e "\n${BOLD}${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
+    echo -e "${BOLD}${BLUE}║          POUNCE DEPLOY PIPELINE v2.0                          ║${NC}"
+    echo -e "${BOLD}${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}\n"
+    
+    log_info "Mode: $mode"
+    log_info "Log: $LOG_FILE"
+    
+    local start_time=$(date +%s)
+    local errors=0
+    
+    # Step 1: Find server
+    echo -e "\n${BOLD}[1/5] Connectivity${NC}"
+    if ! find_server; then
+        log_error "Cannot reach server, aborting"
+        exit 1
+    fi
+    find_ssh || true
+    
+    # Step 2: Pre-deploy health check
+    echo -e "\n${BOLD}[2/5] Pre-deploy Health Check${NC}"
+    check_api_health || log_warn "API not healthy before deploy"
+    check_frontend_health || log_warn "Frontend not healthy before deploy"
+    
+    # Step 3: Git (unless quick mode)
+    if [ "$mode" != "quick" ] && [ "$mode" != "sync" ]; then
+        echo -e "\n${BOLD}[3/5] Git${NC}"
+        git_commit_push "$commit_msg"
+    else
+        echo -e "\n${BOLD}[3/5] Git${NC} ${GRAY}(skipped)${NC}"
+    fi
+    
+    # Step 4: Sync and Deploy
+    echo -e "\n${BOLD}[4/5] Sync & Deploy${NC}"
+    
+    case "$mode" in
+        backend|-b)
+            sync_backend || ((errors++))
+            deploy_backend || ((errors++))
+            ;;
+        frontend|-f)
+            sync_frontend || ((errors++))
+            deploy_frontend || ((errors++))
+            ;;
+        sync|-s)
+            sync_backend || ((errors++))
+            sync_frontend || ((errors++))
+            log_warn "Sync only - services not restarted"
+            ;;
+        quick|-q)
+            sync_backend || ((errors++))
+            sync_frontend || ((errors++))
+            deploy_backend || ((errors++))
+            deploy_frontend || ((errors++))
+            ;;
+        *)
+            sync_backend || ((errors++))
+            sync_frontend || ((errors++))
+            deploy_backend || ((errors++))
+            deploy_frontend || ((errors++))
+            ;;
+    esac
+    
+    # Step 5: Post-deploy health check
+    echo -e "\n${BOLD}[5/5] Post-deploy Health Check${NC}"
+    sleep 5
+    
+    if ! check_api_health; then
+        log_error "API health check failed after deploy!"
+        ((errors++))
+    fi
+    
+    if ! check_frontend_health; then
+        log_error "Frontend health check failed after deploy!"
+        ((errors++))
+    fi
+    
+    # Summary
+    local end_time=$(date +%s)
+    local duration=$((end_time - start_time))
+    
+    echo -e "\n${BOLD}════════════════════════════════════════════════════════════════${NC}"
+    
+    if [ $errors -eq 0 ]; then
+        echo -e "${GREEN}${BOLD}✅ DEPLOY SUCCESSFUL${NC} (${duration}s)"
+    else
+        echo -e "${RED}${BOLD}⚠️  DEPLOY COMPLETED WITH $errors ERROR(S)${NC} (${duration}s)"
+    fi
+    
+    echo -e "${BOLD}════════════════════════════════════════════════════════════════${NC}"
+    echo -e ""
+    echo -e "  ${CYAN}Frontend:${NC} $FRONTEND_URL"
+    echo -e "  ${CYAN}API:${NC}      $API_URL"
+    echo -e "  ${CYAN}Log:${NC}      $LOG_FILE"
+    echo -e ""
+    
+    return $errors
+}
+
+# ============================================================================
+# CLI INTERFACE
+# ============================================================================
+
+show_help() {
+    echo -e "${BOLD}Pounce Deploy Pipeline${NC}"
+    echo ""
+    echo -e "${CYAN}Usage:${NC}"
+    echo "  ./deploy.sh [mode] [commit message]"
+    echo ""
+    echo -e "${CYAN}Modes:${NC}"
+    echo "  full, -a      Full deploy (default) - git, sync, build, restart"
+    echo "  quick, -q     Quick deploy - sync & restart, no git"
+    echo "  backend, -b   Backend only"
+    echo "  frontend, -f  Frontend only"
+    echo "  sync, -s      Sync files only, no restart"
+    echo "  status        Check server status"
+    echo "  health        Run health checks"
+    echo ""
+    echo -e "${CYAN}Examples:${NC}"
+    echo "  ./deploy.sh                     # Full deploy"
+    echo "  ./deploy.sh -q                  # Quick deploy"
+    echo "  ./deploy.sh -b                  # Backend only"
+    echo "  ./deploy.sh \"fix: bug fix\"      # Full deploy with commit message"
+    echo ""
+}
+
+status_check() {
+    echo -e "${BOLD}Server Status${NC}\n"
+    
+    find_server
+    find_ssh
+    
+    echo ""
+    check_api_health
+    check_frontend_health
+    
+    if [ -n "$SSH_HOST" ]; then
+        echo ""
+        log_info "Server uptime:"
+        remote_exec "uptime" 1 || true
+        
+        echo ""
+        log_info "Service status:"
+        remote_exec "systemctl is-active pounce-backend pounce-frontend 2>/dev/null || echo 'Services not using systemd'" 1 || true
+    fi
+}
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+require_cmd sshpass
+require_cmd rsync
+require_cmd curl
+require_cmd git
+
+case "${1:-full}" in
+    help|-h|--help)
+        show_help
+        ;;
+    status)
+        status_check
+        ;;
+    health)
+        check_api_health
+        check_frontend_health
+        ;;
+    *)
+        deploy "$@"
+        ;;
+esac