diff --git a/UX_TERMINAL_UX_REPORT.md b/UX_TERMINAL_UX_REPORT.md index 6fc1879..c584b6d 100644 --- a/UX_TERMINAL_UX_REPORT.md +++ b/UX_TERMINAL_UX_REPORT.md @@ -328,3 +328,6 @@ Empfehlungen: + + + diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index cc7af95..b27d9fa 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -1730,57 +1730,6 @@ async def force_activate_listing( # ============== Zone File Sync ============== -@router.get("/zone-stats") -async def get_zone_stats( - db: Database, - admin: User = Depends(require_admin), -): - """ - Get zone file statistics for all tracked TLDs. - Admin only. - """ - from sqlalchemy import func, text - from app.models.zone_file import DroppedDomain, ZoneSnapshot - - # Get drop counts per TLD - drops_query = await db.execute( - text(""" - SELECT - tld, - COUNT(*) FILTER (WHERE dropped_date >= NOW() - INTERVAL '24 hours') as drops_24h, - COUNT(*) FILTER (WHERE dropped_date >= NOW() - INTERVAL '48 hours') as drops_48h, - COUNT(*) as total_drops - FROM dropped_domains - GROUP BY tld - """) - ) - drop_stats = {row[0]: {"drops_24h": row[1], "drops_48h": row[2], "total_drops": row[3]} for row in drops_query.fetchall()} - - # Get latest snapshot per TLD - snapshots_query = await db.execute( - text(""" - SELECT DISTINCT ON (tld) tld, domain_count, created_at - FROM zone_snapshots - ORDER BY tld, created_at DESC - """) - ) - snapshot_stats = {row[0]: {"total_domains": row[1], "last_sync": row[2].isoformat() if row[2] else None} for row in snapshots_query.fetchall()} - - # Combine stats - all_tlds = set(drop_stats.keys()) | set(snapshot_stats.keys()) - zones = [] - for tld in sorted(all_tlds): - zones.append({ - "tld": tld, - "total_domains": snapshot_stats.get(tld, {}).get("total_domains", 0), - "drops_24h": drop_stats.get(tld, {}).get("drops_24h", 0), - "drops_48h": drop_stats.get(tld, {}).get("drops_48h", 0), - "last_sync": snapshot_stats.get(tld, {}).get("last_sync"), - }) - - return {"zones": zones} - - @router.post("/zone-sync/switch") async def trigger_switch_sync( background_tasks: BackgroundTasks, @@ -1825,12 +1774,94 @@ async def trigger_czds_sync( from app.database import AsyncSessionLocal async with AsyncSessionLocal() as session: client = CZDSClient() - result = await client.sync_all_zones(session) + result = await client.sync_all_zones(session, parallel=True) return result background_tasks.add_task(run_sync) return { "status": "started", - "message": "ICANN CZDS zone sync started in background. Check logs for progress.", + "message": "ICANN CZDS zone sync started in background (parallel mode). Check logs for progress.", + } + + +@router.get("/zone-sync/status") +async def get_zone_sync_status( + db: Database, + admin: User = Depends(require_admin), +): + """ + Get zone sync status and statistics. + Admin only. + """ + from app.models.zone_file import ZoneSnapshot, DroppedDomain + from sqlalchemy import func, desc + from datetime import timedelta + + now = datetime.utcnow() + today = now.replace(hour=0, minute=0, second=0, microsecond=0) + yesterday = today - timedelta(days=1) + + # Get latest snapshots per TLD + snapshots_stmt = ( + select( + ZoneSnapshot.tld, + func.max(ZoneSnapshot.created_at).label("last_sync"), + func.max(ZoneSnapshot.domain_count).label("domain_count"), + ) + .group_by(ZoneSnapshot.tld) + ) + result = await db.execute(snapshots_stmt) + snapshots = {row.tld: {"last_sync": row.last_sync, "domain_count": row.domain_count} for row in result.all()} + + # Get drops count per TLD for today + drops_today_stmt = ( + select( + DroppedDomain.tld, + func.count(DroppedDomain.id).label("count"), + ) + .where(DroppedDomain.dropped_date >= today) + .group_by(DroppedDomain.tld) + ) + result = await db.execute(drops_today_stmt) + drops_today = {row.tld: row.count for row in result.all()} + + # Total drops per TLD + total_drops_stmt = ( + select( + DroppedDomain.tld, + func.count(DroppedDomain.id).label("count"), + ) + .group_by(DroppedDomain.tld) + ) + result = await db.execute(total_drops_stmt) + total_drops = {row.tld: row.count for row in result.all()} + + # Build status for each TLD + all_tlds = set(snapshots.keys()) | set(drops_today.keys()) | set(total_drops.keys()) + + zones = [] + for tld in sorted(all_tlds): + snapshot = snapshots.get(tld, {}) + last_sync = snapshot.get("last_sync") + + zones.append({ + "tld": tld, + "last_sync": last_sync.isoformat() if last_sync else None, + "domain_count": snapshot.get("domain_count", 0), + "drops_today": drops_today.get(tld, 0), + "total_drops": total_drops.get(tld, 0), + "status": "healthy" if last_sync and last_sync > yesterday else "stale" if last_sync else "never", + }) + + return { + "zones": zones, + "summary": { + "total_zones": len(zones), + "healthy": sum(1 for z in zones if z["status"] == "healthy"), + "stale": sum(1 for z in zones if z["status"] == "stale"), + "never_synced": sum(1 for z in zones if z["status"] == "never"), + "total_drops_today": sum(drops_today.values()), + "total_drops_all": sum(total_drops.values()), + } } diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 4c134f2..c4bc658 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -1037,8 +1037,11 @@ async def verify_drops(): async def sync_zone_files(): - """Sync zone files from Switch.ch (.ch, .li) and ICANN CZDS (gTLDs).""" - logger.info("Starting zone file sync...") + """Sync zone files from Switch.ch (.ch, .li).""" + logger.info("Starting Switch.ch zone file sync...") + + results = {"ch": None, "li": None} + errors = [] try: from app.services.zone_file import ZoneFileService @@ -1050,14 +1053,41 @@ async def sync_zone_files(): for tld in ["ch", "li"]: try: result = await service.run_daily_sync(db, tld) - logger.info(f".{tld} zone sync: {len(result.get('dropped', []))} dropped, {result.get('new_count', 0)} new") + dropped_count = len(result.get('dropped', [])) + results[tld] = {"status": "success", "dropped": dropped_count, "new": result.get('new_count', 0)} + logger.info(f".{tld} zone sync: {dropped_count} dropped, {result.get('new_count', 0)} new") except Exception as e: logger.error(f".{tld} zone sync failed: {e}") + results[tld] = {"status": "error", "error": str(e)} + errors.append(f".{tld}: {e}") logger.info("Switch.ch zone file sync completed") + # Send alert if any zones failed + if errors: + from app.services.email_service import email_service + await email_service.send_ops_alert( + alert_type="Zone Sync", + title=f"Switch.ch Sync: {len(errors)} zone(s) failed", + details=f"Results:\n" + "\n".join([ + f"- .{tld}: {r.get('status')} ({r.get('dropped', 0)} dropped)" if r else f"- .{tld}: not processed" + for tld, r in results.items() + ]) + f"\n\nErrors:\n" + "\n".join(errors), + severity="error", + ) + except Exception as e: logger.exception(f"Zone file sync failed: {e}") + try: + from app.services.email_service import email_service + await email_service.send_ops_alert( + alert_type="Zone Sync", + title="Switch.ch Sync CRASHED", + details=f"The Switch.ch sync job crashed:\n\n{str(e)}", + severity="critical", + ) + except: + pass async def sync_czds_zones(): @@ -1078,15 +1108,43 @@ async def sync_czds_zones(): client = CZDSClient() async with AsyncSessionLocal() as db: - results = await client.sync_all_zones(db, APPROVED_TLDS) + results = await client.sync_all_zones(db, APPROVED_TLDS, parallel=True) success_count = sum(1 for r in results if r["status"] == "success") + error_count = sum(1 for r in results if r["status"] == "error") total_dropped = sum(r["dropped_count"] for r in results) logger.info(f"CZDS sync complete: {success_count}/{len(APPROVED_TLDS)} zones, {total_dropped:,} dropped") + + # Send alert if any zones failed + if error_count > 0: + from app.services.email_service import email_service + error_details = "\n".join([ + f"- .{r['tld']}: {r.get('error', 'Unknown error')}" + for r in results if r["status"] == "error" + ]) + await email_service.send_ops_alert( + alert_type="Zone Sync", + title=f"CZDS Sync: {error_count} zone(s) failed", + details=f"Successful: {success_count}/{len(APPROVED_TLDS)}\n" + f"Dropped domains: {total_dropped:,}\n\n" + f"Failed zones:\n{error_details}", + severity="error" if error_count > 2 else "warning", + ) except Exception as e: logger.exception(f"CZDS zone file sync failed: {e}") + # Send critical alert for complete failure + try: + from app.services.email_service import email_service + await email_service.send_ops_alert( + alert_type="Zone Sync", + title="CZDS Sync CRASHED", + details=f"The entire CZDS sync job crashed:\n\n{str(e)}", + severity="critical", + ) + except: + pass # Don't fail the error handler async def match_sniper_alerts(): diff --git a/backend/app/services/czds_client.py b/backend/app/services/czds_client.py index a165719..884ef8b 100644 --- a/backend/app/services/czds_client.py +++ b/backend/app/services/czds_client.py @@ -440,29 +440,31 @@ class CZDSClient: db: Database session tlds: Optional list of TLDs to sync. Defaults to APPROVED_TLDS. parallel: If True, download zones in parallel (faster) - max_concurrent: Max concurrent downloads (be nice to ICANN) + max_concurrent: Max concurrent downloads (to be nice to ICANN) Returns: List of sync results for each TLD. """ target_tlds = tlds or APPROVED_TLDS + start_time = datetime.utcnow() # Get available zones with their download URLs available_zones = await self.get_available_zones() logger.info(f"Starting CZDS sync for {len(target_tlds)} zones: {target_tlds}") logger.info(f"Available zones: {list(available_zones.keys())}") + logger.info(f"Mode: {'PARALLEL' if parallel else 'SEQUENTIAL'} (max {max_concurrent} concurrent)") - # Build list of TLDs with URLs - sync_tasks = [] - results = [] + # Prepare tasks with their download URLs + tasks_to_run = [] + unavailable_results = [] for tld in target_tlds: download_url = available_zones.get(tld) if not download_url: logger.warning(f"No download URL available for .{tld}") - results.append({ + unavailable_results.append({ "tld": tld, "status": "not_available", "current_count": 0, @@ -471,27 +473,30 @@ class CZDSClient: "new_count": 0, "error": f"No access to .{tld} zone" }) - continue - - sync_tasks.append((tld, download_url)) + else: + tasks_to_run.append((tld, download_url)) - if parallel and len(sync_tasks) > 1: - # Parallel sync with semaphore to limit concurrency + results = unavailable_results.copy() + + if parallel and len(tasks_to_run) > 1: + # Parallel execution with semaphore for rate limiting semaphore = asyncio.Semaphore(max_concurrent) async def sync_with_semaphore(tld: str, url: str) -> dict: async with semaphore: return await self.sync_zone(db, tld, url) - # Run all syncs in parallel + # Run all tasks in parallel parallel_results = await asyncio.gather( - *[sync_with_semaphore(tld, url) for tld, url in sync_tasks], + *[sync_with_semaphore(tld, url) for tld, url in tasks_to_run], return_exceptions=True ) - for (tld, _), result in zip(sync_tasks, parallel_results): + # Process results + for i, result in enumerate(parallel_results): + tld = tasks_to_run[i][0] if isinstance(result, Exception): - logger.error(f"Sync failed for .{tld}: {result}") + logger.error(f"Parallel sync failed for .{tld}: {result}") results.append({ "tld": tld, "status": "error", @@ -504,66 +509,24 @@ class CZDSClient: else: results.append(result) else: - # Sequential sync (original behavior) - for tld, download_url in sync_tasks: + # Sequential execution (fallback) + for tld, download_url in tasks_to_run: result = await self.sync_zone(db, tld, download_url) results.append(result) await asyncio.sleep(2) # Summary + elapsed = (datetime.utcnow() - start_time).total_seconds() success_count = sum(1 for r in results if r["status"] == "success") total_dropped = sum(r["dropped_count"] for r in results) - error_count = sum(1 for r in results if r["status"] == "error") logger.info( - f"CZDS sync complete: " + f"CZDS sync complete in {elapsed:.1f}s: " f"{success_count}/{len(target_tlds)} zones successful, " - f"{error_count} errors, " f"{total_dropped:,} total dropped domains" ) - # Send alert on errors - if error_count > 0: - await self._send_sync_alert(results) - return results - - async def _send_sync_alert(self, results: list[dict]): - """Send email alert when sync has errors.""" - try: - from app.services.email_service import email_service - - errors = [r for r in results if r["status"] == "error"] - if not errors: - return - - error_details = "\n".join([ - f" • .{r['tld']}: {r.get('error', 'Unknown error')}" - for r in errors - ]) - - success_count = sum(1 for r in results if r["status"] == "success") - total_dropped = sum(r["dropped_count"] for r in results) - - await email_service.send_email( - to_email=settings.smtp_from_email, # Send to admin - subject=f"⚠️ CZDS Zone Sync Alert - {len(errors)} errors", - html_content=f""" -
Status: {success_count} success, {len(errors)} errors
-Total Drops: {total_dropped:,}
- -{error_details}
-
- - Time: {datetime.utcnow().isoformat()} -
- """ - ) - logger.info("Sent sync error alert email") - except Exception as e: - logger.error(f"Failed to send sync alert: {e}") # ============================================================================ diff --git a/backend/app/services/email_service.py b/backend/app/services/email_service.py index 73dfbdd..b3cbb38 100644 --- a/backend/app/services/email_service.py +++ b/backend/app/services/email_service.py @@ -727,5 +727,63 @@ class EmailService: ) + @staticmethod + async def send_ops_alert( + alert_type: str, + title: str, + details: str, + severity: str = "warning", # info, warning, error, critical + ) -> bool: + """ + Send operational alert to admin email. + + Used for: + - Zone sync failures + - Database connection issues + - Scheduler job failures + - Security incidents + """ + settings = get_settings() + admin_email = settings.smtp_from_email # Send to ourselves for now + + # Build HTML content + severity_colors = { + "info": "#3b82f6", + "warning": "#f59e0b", + "error": "#ef4444", + "critical": "#dc2626", + } + color = severity_colors.get(severity, "#6b7280") + + html = f""" +Timestamp: {datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")}
+Server: pounce.ch
+.ch and .li domains via AXFR
-gTLD zone files (.xyz, .org, .info, etc.)
-Automatic Sync Schedule
-Switch.ch (.ch, .li): Daily at 05:00 UTC
-ICANN CZDS (gTLDs): Daily at 06:00 UTC
-Zone files are processed with 24 parallel workers for maximum speed.
-