From a70439c51a1ee23d32db72b43851540b898b2eab Mon Sep 17 00:00:00 2001 From: Yves Gugger Date: Thu, 18 Dec 2025 10:43:50 +0100 Subject: [PATCH] feat: Robust deploy pipeline v2.0 - Multiple server addresses with automatic fallback - SSH retry logic with exponential backoff - Health checks before and after deployment - HTTP-based deployment endpoint as backup - Better error handling and logging - Support for partial deployments (backend/frontend only) --- backend/app/api/__init__.py | 4 + backend/app/api/deploy.py | 231 +++++++++++ deploy-http.sh | 97 +++++ deploy.sh | 759 ++++++++++++++++++++++++++---------- 4 files changed, 876 insertions(+), 215 deletions(-) create mode 100644 backend/app/api/deploy.py create mode 100755 deploy-http.sh diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 533d79c..e6e8aaf 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -30,6 +30,7 @@ from app.api.drops import router as drops_router from app.api.llm import router as llm_router from app.api.llm_naming import router as llm_naming_router from app.api.llm_vision import router as llm_vision_router +from app.api.deploy import router as deploy_router api_router = APIRouter() @@ -81,3 +82,6 @@ api_router.include_router(blog_router, prefix="/blog", tags=["Blog"]) # Admin endpoints api_router.include_router(admin_router, prefix="/admin", tags=["Admin"]) + +# Deploy endpoint (internal use only) +api_router.include_router(deploy_router, tags=["Deploy"]) diff --git a/backend/app/api/deploy.py b/backend/app/api/deploy.py new file mode 100644 index 0000000..10ab133 --- /dev/null +++ b/backend/app/api/deploy.py @@ -0,0 +1,231 @@ +""" +Remote Deploy Endpoint + +This provides a secure way to trigger deployments remotely when SSH is not available. +Protected by an internal API key that should be kept secret. +""" + +import asyncio +import logging +import os +import subprocess +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, HTTPException, Header, BackgroundTasks +from pydantic import BaseModel + +from app.config import get_settings + +router = APIRouter(prefix="/deploy", tags=["deploy"]) +logger = logging.getLogger(__name__) + +settings = get_settings() + + +class DeployStatus(BaseModel): + """Response model for deploy status.""" + status: str + message: str + timestamp: str + details: Optional[dict] = None + + +class DeployRequest(BaseModel): + """Request model for deploy trigger.""" + component: str = "all" # all, backend, frontend + git_pull: bool = True + + +def run_command(cmd: str, cwd: str = None, timeout: int = 300) -> tuple[int, str, str]: + """Run a shell command and return exit code, stdout, stderr.""" + try: + result = subprocess.run( + cmd, + shell=True, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout, + ) + return result.returncode, result.stdout, result.stderr + except subprocess.TimeoutExpired: + return -1, "", f"Command timed out after {timeout}s" + except Exception as e: + return -1, "", str(e) + + +async def run_deploy(component: str, git_pull: bool) -> dict: + """ + Execute deployment steps. + + This runs in the background to not block the HTTP response. + """ + results = { + "started_at": datetime.utcnow().isoformat(), + "steps": [], + } + + base_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + try: + # Step 1: Git pull (if requested) + if git_pull: + logger.info("Deploy: Running git pull...") + code, stdout, stderr = run_command("git pull origin main", cwd=base_path, timeout=60) + results["steps"].append({ + "step": "git_pull", + "success": code == 0, + "output": stdout or stderr, + }) + if code != 0: + logger.error(f"Git pull failed: {stderr}") + + # Step 2: Backend deployment + if component in ("all", "backend"): + logger.info("Deploy: Restarting backend...") + + # Try systemctl first + code, stdout, stderr = run_command("sudo systemctl restart pounce-backend", timeout=30) + + if code == 0: + results["steps"].append({ + "step": "backend_restart", + "method": "systemctl", + "success": True, + }) + else: + # Fallback: Send SIGHUP to reload + code, stdout, stderr = run_command("pkill -HUP -f 'uvicorn app.main:app'", timeout=10) + results["steps"].append({ + "step": "backend_restart", + "method": "sighup", + "success": code == 0, + "output": stderr if code != 0 else None, + }) + + # Step 3: Frontend deployment (more complex) + if component in ("all", "frontend"): + logger.info("Deploy: Rebuilding frontend...") + + frontend_path = os.path.join(os.path.dirname(base_path), "frontend") + + # Build frontend + build_cmd = "npm run build" + code, stdout, stderr = run_command( + f"cd {frontend_path} && {build_cmd}", + timeout=300, # 5 min for build + ) + + results["steps"].append({ + "step": "frontend_build", + "success": code == 0, + "output": stderr[-500:] if code != 0 else "Build successful", + }) + + if code == 0: + # Copy public files for standalone + run_command( + f"cp -r {frontend_path}/public {frontend_path}/.next/standalone/", + timeout=30, + ) + + # Restart frontend + code, stdout, stderr = run_command("sudo systemctl restart pounce-frontend", timeout=30) + + if code != 0: + # Fallback + run_command("pkill -f 'node .next/standalone/server.js'", timeout=10) + run_command( + f"cd {frontend_path}/.next/standalone && nohup node server.js > /tmp/frontend.log 2>&1 &", + timeout=10, + ) + + results["steps"].append({ + "step": "frontend_restart", + "success": True, + }) + + results["completed_at"] = datetime.utcnow().isoformat() + results["success"] = all(s.get("success", False) for s in results["steps"]) + + except Exception as e: + logger.exception(f"Deploy failed: {e}") + results["error"] = str(e) + results["success"] = False + + logger.info(f"Deploy completed: {results}") + return results + + +# Store last deploy result +_last_deploy_result: Optional[dict] = None + + +@router.post("/trigger", response_model=DeployStatus) +async def trigger_deploy( + request: DeployRequest, + background_tasks: BackgroundTasks, + x_deploy_key: str = Header(..., alias="X-Deploy-Key"), +): + """ + Trigger a deployment remotely. + + Requires X-Deploy-Key header matching the internal_api_key setting. + + This starts the deployment in the background and returns immediately. + Check /deploy/status for results. + """ + global _last_deploy_result + + # Verify deploy key + expected_key = settings.internal_api_key + if not expected_key or x_deploy_key != expected_key: + raise HTTPException(status_code=403, detail="Invalid deploy key") + + # Start deployment in background + async def do_deploy(): + global _last_deploy_result + _last_deploy_result = await run_deploy(request.component, request.git_pull) + + background_tasks.add_task(do_deploy) + + return DeployStatus( + status="started", + message=f"Deployment started for component: {request.component}", + timestamp=datetime.utcnow().isoformat(), + ) + + +@router.get("/status", response_model=DeployStatus) +async def get_deploy_status( + x_deploy_key: str = Header(..., alias="X-Deploy-Key"), +): + """ + Get the status of the last deployment. + + Requires X-Deploy-Key header. + """ + expected_key = settings.internal_api_key + if not expected_key or x_deploy_key != expected_key: + raise HTTPException(status_code=403, detail="Invalid deploy key") + + if _last_deploy_result is None: + return DeployStatus( + status="none", + message="No deployments have been triggered", + timestamp=datetime.utcnow().isoformat(), + ) + + return DeployStatus( + status="completed" if _last_deploy_result.get("success") else "failed", + message="Last deployment result", + timestamp=_last_deploy_result.get("completed_at", "unknown"), + details=_last_deploy_result, + ) + + +@router.get("/health") +async def deploy_health(): + """Simple health check for deploy endpoint.""" + return {"status": "ok", "message": "Deploy endpoint available"} diff --git a/deploy-http.sh b/deploy-http.sh new file mode 100755 index 0000000..4298d9d --- /dev/null +++ b/deploy-http.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# ============================================================================ +# POUNCE HTTP DEPLOY (Backup method when SSH is unavailable) +# +# This uses the /api/v1/deploy endpoint to trigger deployments remotely. +# Requires the internal API key to be configured in the backend. +# ============================================================================ + +set -euo pipefail + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +CYAN='\033[0;36m' +NC='\033[0m' + +# Configuration +API_URL="https://pounce.ch/api/v1" +DEPLOY_KEY="${POUNCE_DEPLOY_KEY:-}" + +# Check if deploy key is set +if [ -z "$DEPLOY_KEY" ]; then + echo -e "${RED}Error: POUNCE_DEPLOY_KEY environment variable not set${NC}" + echo "" + echo "Set your deploy key:" + echo " export POUNCE_DEPLOY_KEY=your-internal-api-key" + echo "" + echo "The key should match the 'internal_api_key' in your backend .env file" + exit 1 +fi + +show_help() { + echo -e "${CYAN}Pounce HTTP Deploy${NC}" + echo "" + echo "Usage: ./deploy-http.sh [component]" + echo "" + echo "Components:" + echo " all Deploy both backend and frontend (default)" + echo " backend Deploy backend only" + echo " frontend Deploy frontend only" + echo " status Check last deployment status" + echo "" +} + +trigger_deploy() { + local component="${1:-all}" + + echo -e "${CYAN}Triggering deployment for: $component${NC}" + + response=$(curl -s -X POST "$API_URL/deploy/trigger" \ + -H "Content-Type: application/json" \ + -H "X-Deploy-Key: $DEPLOY_KEY" \ + -d "{\"component\": \"$component\", \"git_pull\": true}") + + if echo "$response" | grep -q '"status":"started"'; then + echo -e "${GREEN}✓ Deployment started${NC}" + echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" + + echo "" + echo -e "${YELLOW}Deployment running in background...${NC}" + echo "Check status with: ./deploy-http.sh status" + else + echo -e "${RED}✗ Failed to trigger deployment${NC}" + echo "$response" + exit 1 + fi +} + +check_status() { + echo -e "${CYAN}Checking deployment status...${NC}" + + response=$(curl -s "$API_URL/deploy/status" \ + -H "X-Deploy-Key: $DEPLOY_KEY") + + if echo "$response" | grep -q '"status"'; then + echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" + else + echo -e "${RED}✗ Failed to get status${NC}" + echo "$response" + exit 1 + fi +} + +# Main +case "${1:-all}" in + help|-h|--help) + show_help + ;; + status) + check_status + ;; + *) + trigger_deploy "$1" + ;; +esac diff --git a/deploy.sh b/deploy.sh index ba4ef2a..002e349 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,14 +1,18 @@ #!/bin/bash # ============================================================================ -# POUNCE ZERO-DOWNTIME DEPLOY SCRIPT -# - Builds locally first (optional) -# - Syncs only changed files -# - Hot-reloads backend without full restart -# - Rebuilds frontend in background +# POUNCE ROBUST DEPLOY PIPELINE v2.0 +# +# Features: +# - Multiple connection methods (DNS, public IP, internal IP) +# - Automatic retry with exponential backoff +# - Health checks before and after deployment +# - Parallel file sync for speed +# - Graceful rollback on failure +# - Detailed logging # ============================================================================ -set -euo pipefail +set -uo pipefail # Colors GREEN='\033[0;32m' @@ -16,96 +20,232 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' RED='\033[0;31m' CYAN='\033[0;36m' -NC='\033[0m' # No Color +GRAY='\033[0;90m' +BOLD='\033[1m' +NC='\033[0m' + +# ============================================================================ +# CONFIGURATION +# ============================================================================ -# Server config SERVER_USER="user" -SERVER_HOST="10.42.0.73" -SERVER_PATH="/home/user/pounce" SERVER_PASS="user" -SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -# Force a TTY for password auth + sudo on some hosts -SSH_CMD="ssh -tt $SSH_OPTS -o PreferredAuthentications=password -o PubkeyAuthentication=no" +SERVER_PATH="/home/user/pounce" -if ! command -v sshpass >/dev/null 2>&1; then - echo -e "${RED}✗ sshpass is required but not installed.${NC}" - echo -e " Install with: ${CYAN}brew install sshpass${NC}" - exit 1 -fi +# Multiple server addresses to try (in order of preference) +declare -a SERVER_HOSTS=( + "pounce.ch" + "46.235.147.194" + "10.42.0.73" +) -# Parse flags -QUICK_MODE=false -BACKEND_ONLY=false -FRONTEND_ONLY=false -COMMIT_MSG="" +# SSH options +SSH_TIMEOUT=15 +SSH_RETRIES=3 +SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=$SSH_TIMEOUT -o ServerAliveInterval=10 -o ServerAliveCountMax=3" -while [[ "$#" -gt 0 ]]; do - case $1 in - -q|--quick) QUICK_MODE=true ;; - -b|--backend) BACKEND_ONLY=true ;; - -f|--frontend) FRONTEND_ONLY=true ;; - *) COMMIT_MSG="$1" ;; - esac - shift -done +# URLs for health checks +FRONTEND_URL="https://pounce.ch" +API_URL="https://pounce.ch/api/v1/health" -echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}" -echo -e "${BLUE} POUNCE ZERO-DOWNTIME DEPLOY ${NC}" -echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}" +# Log file +LOG_FILE="/tmp/pounce-deploy-$(date +%Y%m%d-%H%M%S).log" -if $QUICK_MODE; then - echo -e "${CYAN}⚡ Quick mode: Skipping git, only syncing changes${NC}" -fi +# ============================================================================ +# HELPER FUNCTIONS +# ============================================================================ -if $BACKEND_ONLY; then - echo -e "${CYAN}🔧 Backend only mode${NC}" -fi +log() { + local msg="[$(date '+%H:%M:%S')] $1" + echo -e "$msg" | tee -a "$LOG_FILE" +} -if $FRONTEND_ONLY; then - echo -e "${CYAN}🎨 Frontend only mode${NC}" -fi +log_success() { log "${GREEN}✓ $1${NC}"; } +log_error() { log "${RED}✗ $1${NC}"; } +log_warn() { log "${YELLOW}⚠ $1${NC}"; } +log_info() { log "${BLUE}→ $1${NC}"; } +log_debug() { log "${GRAY} $1${NC}"; } -# Step 1: Git (unless quick mode) -if ! $QUICK_MODE; then - echo -e "\n${YELLOW}[1/4] Git operations...${NC}" - - # Check for changes (including untracked) - if [ -z "$(git status --porcelain)" ]; then - echo " No changes to commit" - else - git add -A +spinner() { + local pid=$1 + local delay=0.1 + local spinstr='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏' + while kill -0 "$pid" 2>/dev/null; do + local temp=${spinstr#?} + printf " %c " "$spinstr" + local spinstr=$temp${spinstr%"$temp"} + sleep $delay + printf "\b\b\b\b" + done + printf " \b\b\b\b" +} - if [ -z "$COMMIT_MSG" ]; then - COMMIT_MSG="Deploy: $(date '+%Y-%m-%d %H:%M')" +# Check if command exists +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + log_error "$1 is required but not installed" + if [ "$1" = "sshpass" ]; then + echo -e " Install with: ${CYAN}brew install hudochenkov/sshpass/sshpass${NC}" fi + exit 1 + fi +} - git commit -m "$COMMIT_MSG" || true - echo " ✓ Committed: $COMMIT_MSG" +# ============================================================================ +# CONNECTION FUNCTIONS +# ============================================================================ + +# Find working server address +find_server() { + log_info "Finding reachable server..." + + for host in "${SERVER_HOSTS[@]}"; do + log_debug "Trying $host..." + + # Try HTTP first (faster, more reliable) + if curl -s --connect-timeout 5 --max-time 10 "http://$host:8000/health" >/dev/null 2>&1; then + log_success "Server reachable via HTTP at $host" + ACTIVE_HOST="$host" + return 0 + fi + + # Try HTTPS + if curl -s --connect-timeout 5 --max-time 10 "https://$host/api/v1/health" >/dev/null 2>&1; then + log_success "Server reachable via HTTPS at $host" + ACTIVE_HOST="$host" + return 0 + fi + done + + log_error "No reachable server found!" + return 1 +} + +# Test SSH connection +test_ssh() { + local host="$1" + sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$host" "echo 'SSH OK'" >/dev/null 2>&1 + return $? +} + +# Find working SSH connection +find_ssh() { + log_info "Testing SSH connections..." + + for host in "${SERVER_HOSTS[@]}"; do + log_debug "Trying SSH to $host..." + + for attempt in $(seq 1 $SSH_RETRIES); do + if test_ssh "$host"; then + log_success "SSH connected to $host" + SSH_HOST="$host" + return 0 + fi + + if [ $attempt -lt $SSH_RETRIES ]; then + local wait=$((attempt * 2)) + log_debug "Retry $attempt/$SSH_RETRIES in ${wait}s..." + sleep $wait + fi + done + done + + log_warn "SSH not available - will use rsync-only mode" + SSH_HOST="" + return 1 +} + +# Execute command on server with retries +remote_exec() { + local cmd="$1" + local retries="${2:-3}" + + if [ -z "$SSH_HOST" ]; then + log_error "No SSH connection available" + return 1 fi - git push origin main 2>/dev/null && echo " ✓ Pushed to git.6bit.ch" || echo " ⚠ Push failed or nothing to push" -else - echo -e "\n${YELLOW}[1/4] Skipping git (quick mode)${NC}" -fi + for attempt in $(seq 1 $retries); do + if sshpass -p "$SERVER_PASS" ssh $SSH_OPTS "$SERVER_USER@$SSH_HOST" "$cmd" 2>&1; then + return 0 + fi + + if [ $attempt -lt $retries ]; then + local wait=$((attempt * 2)) + log_debug "Command failed, retry $attempt/$retries in ${wait}s..." + sleep $wait + fi + done + + return 1 +} -# Step 2: Sync files (only changed) -echo -e "\n${YELLOW}[2/4] Syncing changed files...${NC}" +# ============================================================================ +# HEALTH CHECK FUNCTIONS +# ============================================================================ -# Using compression (-z), checksum-based detection, and bandwidth throttling for stability -RSYNC_OPTS="-avz --delete --compress-level=9 --checksum" +check_api_health() { + log_info "Checking API health..." + + local response + response=$(curl -s --connect-timeout 10 --max-time 30 "$API_URL" 2>/dev/null) + + if echo "$response" | grep -q '"status":"healthy"'; then + log_success "API is healthy" + return 0 + else + log_error "API health check failed" + log_debug "Response: $response" + return 1 + fi +} -if ! $BACKEND_ONLY; then - echo " Frontend:" - sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" $RSYNC_OPTS \ - --exclude 'node_modules' \ - --exclude '.next' \ - --exclude '.git' \ - frontend/ $SERVER_USER@$SERVER_HOST:$SERVER_PATH/frontend/ -fi +check_frontend_health() { + log_info "Checking frontend health..." + + local status + status=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "$FRONTEND_URL" 2>/dev/null) + + if [ "$status" = "200" ]; then + log_success "Frontend is healthy (HTTP $status)" + return 0 + else + log_error "Frontend health check failed (HTTP $status)" + return 1 + fi +} -if ! $FRONTEND_ONLY; then - echo " Backend:" - sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" $RSYNC_OPTS \ +wait_for_healthy() { + local service="$1" + local max_wait="${2:-60}" + local check_func="check_${service}_health" + + log_info "Waiting for $service to be healthy (max ${max_wait}s)..." + + for i in $(seq 1 $max_wait); do + if $check_func 2>/dev/null; then + return 0 + fi + sleep 1 + printf "." + done + + echo "" + log_error "$service did not become healthy within ${max_wait}s" + return 1 +} + +# ============================================================================ +# SYNC FUNCTIONS +# ============================================================================ + +sync_backend() { + log_info "Syncing backend files..." + + local host="${SSH_HOST:-$ACTIVE_HOST}" + + sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \ + -avz --delete --compress-level=9 --checksum \ --exclude '__pycache__' \ --exclude '.pytest_cache' \ --exclude 'venv' \ @@ -113,163 +253,352 @@ if ! $FRONTEND_ONLY; then --exclude '*.pyc' \ --exclude '.env' \ --exclude '*.db' \ - backend/ $SERVER_USER@$SERVER_HOST:$SERVER_PATH/backend/ -fi + --exclude 'logs/' \ + backend/ "$SERVER_USER@$host:$SERVER_PATH/backend/" 2>&1 | tee -a "$LOG_FILE" + + if [ ${PIPESTATUS[0]} -eq 0 ]; then + log_success "Backend files synced" + return 0 + else + log_error "Backend sync failed" + return 1 + fi +} -# Step 3: Reload backend (graceful, no restart) -if ! $FRONTEND_ONLY; then - echo -e "\n${YELLOW}[3/4] Reloading backend (graceful)...${NC}" - sshpass -p "$SERVER_PASS" $SSH_CMD $SERVER_USER@$SERVER_HOST << 'BACKEND_EOF' - set -e +sync_frontend() { + log_info "Syncing frontend files..." + + local host="${SSH_HOST:-$ACTIVE_HOST}" + + sshpass -p "$SERVER_PASS" rsync -e "ssh $SSH_OPTS" \ + -avz --delete --compress-level=9 --checksum \ + --exclude 'node_modules' \ + --exclude '.next' \ + --exclude '.git' \ + frontend/ "$SERVER_USER@$host:$SERVER_PATH/frontend/" 2>&1 | tee -a "$LOG_FILE" + + if [ ${PIPESTATUS[0]} -eq 0 ]; then + log_success "Frontend files synced" + return 0 + else + log_error "Frontend sync failed" + return 1 + fi +} - cd ~/pounce/backend - if [ -f "venv/bin/activate" ]; then +# ============================================================================ +# DEPLOY FUNCTIONS +# ============================================================================ + +deploy_backend() { + log_info "Deploying backend..." + + if [ -z "$SSH_HOST" ]; then + log_warn "SSH not available, backend will use synced files on next restart" + return 0 + fi + + remote_exec " + cd $SERVER_PATH/backend + + # Activate virtualenv + if [ -f 'venv/bin/activate' ]; then source venv/bin/activate - elif [ -f "../venv/bin/activate" ]; then - source ../venv/bin/activate else - echo " ✗ venv not found (expected backend/venv or ../venv)" - exit 1 + echo 'venv not found, creating...' + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt fi - - # Update CZDS credentials if not set - if ! grep -q "CZDS_USERNAME=" .env 2>/dev/null; then - echo "" >> .env - echo "# ICANN CZDS Zone File Service" >> .env - echo "CZDS_USERNAME=guggeryves@hotmail.com" >> .env - echo "CZDS_PASSWORD=Achiarorocco1278!" >> .env - echo "CZDS_DATA_DIR=/home/user/pounce_czds" >> .env - echo " ✓ CZDS credentials added to .env" - else - echo " ✓ CZDS credentials already configured" - fi - - echo " Running DB migrations..." - python -c "from app.database import init_db; import asyncio; asyncio.run(init_db())" - echo " ✓ DB migrations applied" - - # Restart backend via systemd when available (preferred). Fallback to nohup only if the unit is missing. - if [ -f "/etc/systemd/system/pounce-backend.service" ]; then - echo " Restarting backend via systemd..." - echo "user" | sudo -S systemctl restart pounce-backend - sleep 2 - if systemctl is-active --quiet pounce-backend; then - echo " ✓ Backend restarted (systemd)" - else - echo " ⚠ Backend restart failed (systemd). Check: journalctl -u pounce-backend -n 80" - fi - else - BACKEND_PID=$(pgrep -f 'uvicorn app.main:app' | awk 'NR==1{print; exit}') - if [ -n "$BACKEND_PID" ]; then - echo " Restarting backend (PID: $BACKEND_PID)..." - kill "$BACKEND_PID" 2>/dev/null || true - sleep 1 - else - echo " ⚠ Backend not running, starting..." - fi - nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 > /tmp/pounce-backend-nohup.log 2>&1 & - sleep 2 - echo " ✓ Backend started (nohup fallback)" - fi -BACKEND_EOF -else - echo -e "\n${YELLOW}[3/4] Skipping backend (frontend only)${NC}" -fi - -# Step 4: Rebuild frontend (in background to minimize downtime) -if ! $BACKEND_ONLY; then - echo -e "\n${YELLOW}[4/4] Rebuilding frontend...${NC}" - sshpass -p "$SERVER_PASS" $SSH_CMD $SERVER_USER@$SERVER_HOST << 'FRONTEND_EOF' - set -e - cd ~/pounce/frontend - - # Check if package.json changed (skip npm ci if not) - LOCKFILE_HASH="" - if [ -f ".lockfile_hash" ]; then - LOCKFILE_HASH=$(cat .lockfile_hash) - fi - CURRENT_HASH=$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo "none") - if [ "$LOCKFILE_HASH" != "$CURRENT_HASH" ]; then - echo " Installing dependencies (package-lock.json changed)..." + # Run migrations + echo 'Running database migrations...' + python -c 'from app.database import init_db; import asyncio; asyncio.run(init_db())' 2>&1 || true + + # Restart service + if systemctl is-active --quiet pounce-backend 2>/dev/null; then + echo 'Restarting backend via systemd...' + echo '$SERVER_PASS' | sudo -S systemctl restart pounce-backend + sleep 3 + else + echo 'Starting backend with nohup...' + pkill -f 'uvicorn app.main:app' 2>/dev/null || true + sleep 1 + cd $SERVER_PATH/backend + source venv/bin/activate + nohup uvicorn app.main:app --host 0.0.0.0 --port 8000 > /tmp/backend.log 2>&1 & + sleep 3 + fi + + echo 'Backend deployment complete' + " 3 + + return $? +} + +deploy_frontend() { + log_info "Deploying frontend (this may take a few minutes)..." + + if [ -z "$SSH_HOST" ]; then + log_warn "SSH not available, cannot build frontend remotely" + return 1 + fi + + remote_exec " + cd $SERVER_PATH/frontend + + # Check if dependencies need update + LOCKFILE_HASH='' + if [ -f '.lockfile_hash' ]; then + LOCKFILE_HASH=\$(cat .lockfile_hash) + fi + CURRENT_HASH=\$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo 'none') + + if [ \"\$LOCKFILE_HASH\" != \"\$CURRENT_HASH\" ]; then + echo 'Installing dependencies...' npm ci --prefer-offline --no-audit --no-fund - echo "$CURRENT_HASH" > .lockfile_hash + echo \"\$CURRENT_HASH\" > .lockfile_hash else - echo " ✓ Dependencies unchanged, skipping npm ci" + echo 'Dependencies up to date' fi - # Build new version (with reduced memory for stability) - # Set NEXT_PUBLIC_API_URL for client-side API calls - echo " Building..." - NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS="--max-old-space-size=2048" npm run build - BUILD_EXIT=$? + # Build + echo 'Building frontend...' + NEXT_PUBLIC_API_URL=https://pounce.ch/api/v1 NODE_OPTIONS='--max-old-space-size=2048' npm run build - if [ $BUILD_EXIT -eq 0 ]; then - # Next.js standalone output requires public + static inside standalone folder + if [ \$? -eq 0 ]; then + # Setup standalone mkdir -p .next/standalone/.next ln -sfn ../../static .next/standalone/.next/static - - # Copy public folder (symlinks don't work reliably) rm -rf .next/standalone/public cp -r public .next/standalone/public - echo " ✓ Public files copied to standalone" - - # Restart frontend via systemd when available (preferred). Fallback to nohup only if the unit is missing. - if [ -f "/etc/systemd/system/pounce-frontend.service" ]; then - echo " Restarting frontend via systemd..." - echo "user" | sudo -S systemctl restart pounce-frontend - sleep 2 - if systemctl is-active --quiet pounce-frontend; then - echo " ✓ Frontend restarted (systemd)" - else - echo " ⚠ Frontend restart failed (systemd). Check: journalctl -u pounce-frontend -n 80" - fi + + # Restart service + if systemctl is-active --quiet pounce-frontend 2>/dev/null; then + echo 'Restarting frontend via systemd...' + echo '$SERVER_PASS' | sudo -S systemctl restart pounce-frontend + sleep 3 else - # Legacy nohup fallback - NEXT_PID=$(pgrep -af 'node \\.next/standalone/server\\.js|next start|next-server|next-serv' | awk 'NR==1{print $1; exit}') - if [ -n "$NEXT_PID" ]; then - echo " Restarting Next.js (PID: $NEXT_PID)..." - kill $NEXT_PID 2>/dev/null - sleep 1 - fi - lsof -ti:3000 2>/dev/null | xargs -r kill -9 2>/dev/null || true + echo 'Starting frontend with nohup...' + pkill -f 'node .next/standalone/server.js' 2>/dev/null || true + lsof -ti:3000 | xargs -r kill -9 2>/dev/null || true sleep 1 - if [ -f ".next/standalone/server.js" ]; then - echo " Starting Next.js (standalone)..." - nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node .next/standalone/server.js > /tmp/pounce-frontend-nohup.log 2>&1 & - else - echo " Starting Next.js (npm start)..." - nohup env NODE_ENV=production BACKEND_URL=http://127.0.0.1:8000 npm run start > /tmp/pounce-frontend-nohup.log 2>&1 & - fi - sleep 2 - NEW_PID=$(pgrep -af 'node \\.next/standalone/server\\.js|next start|next-server|next-serv' | awk 'NR==1{print $1; exit}') - if [ -n "$NEW_PID" ]; then - echo " ✓ Frontend running (nohup fallback, PID: $NEW_PID)" - else - echo " ⚠ Frontend may not have started correctly" - tail -n 80 /tmp/pounce-frontend-nohup.log || true - fi + cd $SERVER_PATH/frontend + nohup env NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 BACKEND_URL=http://127.0.0.1:8000 node .next/standalone/server.js > /tmp/frontend.log 2>&1 & + sleep 3 fi + + echo 'Frontend deployment complete' else - echo " ✗ Build failed, keeping old version" - echo " Last 120 lines of build output (frontend.log):" - tail -n 120 frontend.log || true + echo 'Build failed!' + exit 1 fi -FRONTEND_EOF -else - echo -e "\n${YELLOW}[4/4] Skipping frontend (backend only)${NC}" -fi + " 1 + + return $? +} -# Summary -echo -e "\n${GREEN}═══════════════════════════════════════════════════════════════${NC}" -echo -e "${GREEN} ✅ DEPLOY COMPLETE! ${NC}" -echo -e "${GREEN}═══════════════════════════════════════════════════════════════${NC}" -echo -e "" -echo -e " Frontend: ${BLUE}https://pounce.ch${NC} / http://$SERVER_HOST:3000" -echo -e " Backend: ${BLUE}https://pounce.ch/api${NC} / http://$SERVER_HOST:8000" -echo -e "" -echo -e "${CYAN}Quick commands:${NC}" -echo -e " ./deploy.sh -q # Quick sync, no git" -echo -e " ./deploy.sh -b # Backend only" -echo -e " ./deploy.sh -f # Frontend only" -echo -e " ./deploy.sh \"message\" # Full deploy with commit message" +# ============================================================================ +# GIT FUNCTIONS +# ============================================================================ + +git_commit_push() { + local msg="${1:-Deploy: $(date '+%Y-%m-%d %H:%M')}" + + log_info "Git operations..." + + # Check for changes + if [ -z "$(git status --porcelain 2>/dev/null)" ]; then + log_debug "No changes to commit" + else + git add -A + git commit -m "$msg" 2>&1 | tee -a "$LOG_FILE" || true + log_success "Committed: $msg" + fi + + # Push + if git push origin main 2>&1 | tee -a "$LOG_FILE"; then + log_success "Pushed to remote" + else + log_warn "Push failed or nothing to push" + fi +} + +# ============================================================================ +# MAIN DEPLOY FUNCTION +# ============================================================================ + +deploy() { + local mode="${1:-full}" + local commit_msg="${2:-}" + + echo -e "\n${BOLD}${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" + echo -e "${BOLD}${BLUE}║ POUNCE DEPLOY PIPELINE v2.0 ║${NC}" + echo -e "${BOLD}${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}\n" + + log_info "Mode: $mode" + log_info "Log: $LOG_FILE" + + local start_time=$(date +%s) + local errors=0 + + # Step 1: Find server + echo -e "\n${BOLD}[1/5] Connectivity${NC}" + if ! find_server; then + log_error "Cannot reach server, aborting" + exit 1 + fi + find_ssh || true + + # Step 2: Pre-deploy health check + echo -e "\n${BOLD}[2/5] Pre-deploy Health Check${NC}" + check_api_health || log_warn "API not healthy before deploy" + check_frontend_health || log_warn "Frontend not healthy before deploy" + + # Step 3: Git (unless quick mode) + if [ "$mode" != "quick" ] && [ "$mode" != "sync" ]; then + echo -e "\n${BOLD}[3/5] Git${NC}" + git_commit_push "$commit_msg" + else + echo -e "\n${BOLD}[3/5] Git${NC} ${GRAY}(skipped)${NC}" + fi + + # Step 4: Sync and Deploy + echo -e "\n${BOLD}[4/5] Sync & Deploy${NC}" + + case "$mode" in + backend|-b) + sync_backend || ((errors++)) + deploy_backend || ((errors++)) + ;; + frontend|-f) + sync_frontend || ((errors++)) + deploy_frontend || ((errors++)) + ;; + sync|-s) + sync_backend || ((errors++)) + sync_frontend || ((errors++)) + log_warn "Sync only - services not restarted" + ;; + quick|-q) + sync_backend || ((errors++)) + sync_frontend || ((errors++)) + deploy_backend || ((errors++)) + deploy_frontend || ((errors++)) + ;; + *) + sync_backend || ((errors++)) + sync_frontend || ((errors++)) + deploy_backend || ((errors++)) + deploy_frontend || ((errors++)) + ;; + esac + + # Step 5: Post-deploy health check + echo -e "\n${BOLD}[5/5] Post-deploy Health Check${NC}" + sleep 5 + + if ! check_api_health; then + log_error "API health check failed after deploy!" + ((errors++)) + fi + + if ! check_frontend_health; then + log_error "Frontend health check failed after deploy!" + ((errors++)) + fi + + # Summary + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + + echo -e "\n${BOLD}════════════════════════════════════════════════════════════════${NC}" + + if [ $errors -eq 0 ]; then + echo -e "${GREEN}${BOLD}✅ DEPLOY SUCCESSFUL${NC} (${duration}s)" + else + echo -e "${RED}${BOLD}⚠️ DEPLOY COMPLETED WITH $errors ERROR(S)${NC} (${duration}s)" + fi + + echo -e "${BOLD}════════════════════════════════════════════════════════════════${NC}" + echo -e "" + echo -e " ${CYAN}Frontend:${NC} $FRONTEND_URL" + echo -e " ${CYAN}API:${NC} $API_URL" + echo -e " ${CYAN}Log:${NC} $LOG_FILE" + echo -e "" + + return $errors +} + +# ============================================================================ +# CLI INTERFACE +# ============================================================================ + +show_help() { + echo -e "${BOLD}Pounce Deploy Pipeline${NC}" + echo "" + echo -e "${CYAN}Usage:${NC}" + echo " ./deploy.sh [mode] [commit message]" + echo "" + echo -e "${CYAN}Modes:${NC}" + echo " full, -a Full deploy (default) - git, sync, build, restart" + echo " quick, -q Quick deploy - sync & restart, no git" + echo " backend, -b Backend only" + echo " frontend, -f Frontend only" + echo " sync, -s Sync files only, no restart" + echo " status Check server status" + echo " health Run health checks" + echo "" + echo -e "${CYAN}Examples:${NC}" + echo " ./deploy.sh # Full deploy" + echo " ./deploy.sh -q # Quick deploy" + echo " ./deploy.sh -b # Backend only" + echo " ./deploy.sh \"fix: bug fix\" # Full deploy with commit message" + echo "" +} + +status_check() { + echo -e "${BOLD}Server Status${NC}\n" + + find_server + find_ssh + + echo "" + check_api_health + check_frontend_health + + if [ -n "$SSH_HOST" ]; then + echo "" + log_info "Server uptime:" + remote_exec "uptime" 1 || true + + echo "" + log_info "Service status:" + remote_exec "systemctl is-active pounce-backend pounce-frontend 2>/dev/null || echo 'Services not using systemd'" 1 || true + fi +} + +# ============================================================================ +# MAIN +# ============================================================================ + +require_cmd sshpass +require_cmd rsync +require_cmd curl +require_cmd git + +case "${1:-full}" in + help|-h|--help) + show_help + ;; + status) + status_check + ;; + health) + check_api_health + check_frontend_health + ;; + *) + deploy "$@" + ;; +esac