diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 810f93f..cc7af95 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -1730,6 +1730,57 @@ async def force_activate_listing( # ============== Zone File Sync ============== +@router.get("/zone-stats") +async def get_zone_stats( + db: Database, + admin: User = Depends(require_admin), +): + """ + Get zone file statistics for all tracked TLDs. + Admin only. + """ + from sqlalchemy import func, text + from app.models.zone_file import DroppedDomain, ZoneSnapshot + + # Get drop counts per TLD + drops_query = await db.execute( + text(""" + SELECT + tld, + COUNT(*) FILTER (WHERE dropped_date >= NOW() - INTERVAL '24 hours') as drops_24h, + COUNT(*) FILTER (WHERE dropped_date >= NOW() - INTERVAL '48 hours') as drops_48h, + COUNT(*) as total_drops + FROM dropped_domains + GROUP BY tld + """) + ) + drop_stats = {row[0]: {"drops_24h": row[1], "drops_48h": row[2], "total_drops": row[3]} for row in drops_query.fetchall()} + + # Get latest snapshot per TLD + snapshots_query = await db.execute( + text(""" + SELECT DISTINCT ON (tld) tld, domain_count, created_at + FROM zone_snapshots + ORDER BY tld, created_at DESC + """) + ) + snapshot_stats = {row[0]: {"total_domains": row[1], "last_sync": row[2].isoformat() if row[2] else None} for row in snapshots_query.fetchall()} + + # Combine stats + all_tlds = set(drop_stats.keys()) | set(snapshot_stats.keys()) + zones = [] + for tld in sorted(all_tlds): + zones.append({ + "tld": tld, + "total_domains": snapshot_stats.get(tld, {}).get("total_domains", 0), + "drops_24h": drop_stats.get(tld, {}).get("drops_24h", 0), + "drops_48h": drop_stats.get(tld, {}).get("drops_48h", 0), + "last_sync": snapshot_stats.get(tld, {}).get("last_sync"), + }) + + return {"zones": zones} + + @router.post("/zone-sync/switch") async def trigger_switch_sync( background_tasks: BackgroundTasks, diff --git a/backend/app/services/czds_client.py b/backend/app/services/czds_client.py index 0a0ca6c..a165719 100644 --- a/backend/app/services/czds_client.py +++ b/backend/app/services/czds_client.py @@ -429,7 +429,9 @@ class CZDSClient: async def sync_all_zones( self, db: AsyncSession, - tlds: Optional[list[str]] = None + tlds: Optional[list[str]] = None, + parallel: bool = True, + max_concurrent: int = 3 ) -> list[dict]: """ Sync all approved zone files. @@ -437,6 +439,8 @@ class CZDSClient: Args: db: Database session tlds: Optional list of TLDs to sync. Defaults to APPROVED_TLDS. + parallel: If True, download zones in parallel (faster) + max_concurrent: Max concurrent downloads (be nice to ICANN) Returns: List of sync results for each TLD. @@ -449,9 +453,11 @@ class CZDSClient: logger.info(f"Starting CZDS sync for {len(target_tlds)} zones: {target_tlds}") logger.info(f"Available zones: {list(available_zones.keys())}") + # Build list of TLDs with URLs + sync_tasks = [] results = [] + for tld in target_tlds: - # Get the actual download URL for this TLD download_url = available_zones.get(tld) if not download_url: @@ -467,23 +473,97 @@ class CZDSClient: }) continue - result = await self.sync_zone(db, tld, download_url) - results.append(result) + sync_tasks.append((tld, download_url)) + + if parallel and len(sync_tasks) > 1: + # Parallel sync with semaphore to limit concurrency + semaphore = asyncio.Semaphore(max_concurrent) - # Small delay between zones to be nice to ICANN servers - await asyncio.sleep(2) + async def sync_with_semaphore(tld: str, url: str) -> dict: + async with semaphore: + return await self.sync_zone(db, tld, url) + + # Run all syncs in parallel + parallel_results = await asyncio.gather( + *[sync_with_semaphore(tld, url) for tld, url in sync_tasks], + return_exceptions=True + ) + + for (tld, _), result in zip(sync_tasks, parallel_results): + if isinstance(result, Exception): + logger.error(f"Sync failed for .{tld}: {result}") + results.append({ + "tld": tld, + "status": "error", + "current_count": 0, + "previous_count": 0, + "dropped_count": 0, + "new_count": 0, + "error": str(result) + }) + else: + results.append(result) + else: + # Sequential sync (original behavior) + for tld, download_url in sync_tasks: + result = await self.sync_zone(db, tld, download_url) + results.append(result) + await asyncio.sleep(2) # Summary success_count = sum(1 for r in results if r["status"] == "success") total_dropped = sum(r["dropped_count"] for r in results) + error_count = sum(1 for r in results if r["status"] == "error") logger.info( f"CZDS sync complete: " f"{success_count}/{len(target_tlds)} zones successful, " + f"{error_count} errors, " f"{total_dropped:,} total dropped domains" ) + # Send alert on errors + if error_count > 0: + await self._send_sync_alert(results) + return results + + async def _send_sync_alert(self, results: list[dict]): + """Send email alert when sync has errors.""" + try: + from app.services.email_service import email_service + + errors = [r for r in results if r["status"] == "error"] + if not errors: + return + + error_details = "\n".join([ + f" • .{r['tld']}: {r.get('error', 'Unknown error')}" + for r in errors + ]) + + success_count = sum(1 for r in results if r["status"] == "success") + total_dropped = sum(r["dropped_count"] for r in results) + + await email_service.send_email( + to_email=settings.smtp_from_email, # Send to admin + subject=f"⚠️ CZDS Zone Sync Alert - {len(errors)} errors", + html_content=f""" +

Zone Sync Report

+

Status: {success_count} success, {len(errors)} errors

+

Total Drops: {total_dropped:,}

+ +

Errors:

+ 
{error_details}
+ +

+ Time: {datetime.utcnow().isoformat()} +

+ """ + ) + logger.info("Sent sync error alert email") + except Exception as e: + logger.error(f"Failed to send sync alert: {e}") # ============================================================================ diff --git a/frontend/src/app/admin/page.tsx b/frontend/src/app/admin/page.tsx index 3a6040b..183f2f5 100644 --- a/frontend/src/app/admin/page.tsx +++ b/frontend/src/app/admin/page.tsx @@ -5,6 +5,7 @@ import { useRouter } from 'next/navigation' import { useStore } from '@/lib/store' import { api } from '@/lib/api' import { EarningsTab } from '@/components/admin/EarningsTab' +import { ZoneSyncTab } from '@/components/admin/ZoneSyncTab' import { PremiumTable, Badge, TableActionButton, StatCard } from '@/components/PremiumTable' import { Users, @@ -56,7 +57,7 @@ import Image from 'next/image' // TYPES // ============================================================================ -type TabType = 'overview' | 'earnings' | 'telemetry' | 'users' | 'alerts' | 'newsletter' | 'tld' | 'auctions' | 'blog' | 'system' | 'activity' +type TabType = 'overview' | 'earnings' | 'telemetry' | 'users' | 'alerts' | 'newsletter' | 'tld' | 'auctions' | 'blog' | 'system' | 'activity' | 'zones' interface AdminStats { users: { total: number; active: number; verified: number; new_this_week: number } @@ -94,6 +95,7 @@ const TABS: Array<{ id: TabType; label: string; icon: any; shortLabel?: string } { id: 'tld', label: 'TLD Data', icon: Globe, shortLabel: 'TLD' }, { id: 'auctions', label: 'Auctions', icon: Gavel, shortLabel: 'Auctions' }, { id: 'system', label: 'System', icon: Database, shortLabel: 'System' }, + { id: 'zones', label: 'Zone Sync', icon: Download, shortLabel: 'Zones' }, { id: 'activity', label: 'Activity', icon: History, shortLabel: 'Log' }, ] @@ -957,6 +959,8 @@ export default function AdminPage() { )} {/* Activity Tab */} + {activeTab === 'zones' && } + {activeTab === 'activity' && ( ([]) + const [loading, setLoading] = useState(true) + const [syncingSwitch, setSyncingSwitch] = useState(false) + const [syncingCzds, setSyncingCzds] = useState(false) + const [result, setResult] = useState<{ type: 'success' | 'error'; message: string } | null>(null) + + const loadStats = useCallback(async () => { + try { + const data = await api.getZoneStats() + setStats(data.zones || []) + } catch (e) { + console.error('Failed to load zone stats:', e) + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + loadStats() + }, [loadStats]) + + const triggerSwitchSync = async () => { + setSyncingSwitch(true) + setResult(null) + try { + const res = await api.triggerSwitchSync() + setResult({ type: 'success', message: res.message || 'Switch.ch sync started!' }) + // Reload stats after a delay + setTimeout(loadStats, 5000) + } catch (e) { + setResult({ type: 'error', message: e instanceof Error ? e.message : 'Sync failed' }) + } finally { + setSyncingSwitch(false) + } + } + + const triggerCzdsSync = async () => { + setSyncingCzds(true) + setResult(null) + try { + const res = await api.triggerCzdsSync() + setResult({ type: 'success', message: res.message || 'CZDS sync started!' }) + // Reload stats after a delay + setTimeout(loadStats, 10000) + } catch (e) { + setResult({ type: 'error', message: e instanceof Error ? e.message : 'Sync failed' }) + } finally { + setSyncingCzds(false) + } + } + + const switchTlds = stats.filter(s => ['ch', 'li'].includes(s.tld)) + const czdsTlds = stats.filter(s => !['ch', 'li'].includes(s.tld)) + + const totalDrops24h = stats.reduce((sum, s) => sum + (s.drops_24h || 0), 0) + const totalDomains = stats.reduce((sum, s) => sum + (s.total_domains || 0), 0) + + if (loading) { + return ( +
+ +
+ ) + } + + return ( +
+ {/* Stats Overview */} +
+
+
+ + TLDs Tracked +
+
{stats.length}
+
+
+
+ + Total Domains +
+
{(totalDomains / 1000000).toFixed(1)}M
+
+
+
+ + Drops (24h) +
+
{totalDrops24h.toLocaleString()}
+
+
+
+ + Next Sync +
+
05:00 UTC
+
+
+ + {/* Result Message */} + {result && ( +
+ {result.type === 'success' ? : } + {result.message} +
+ )} + + {/* Switch.ch Section */} +
+
+
+
+ 🇨🇭 +
+
+

Switch.ch Zone Sync

+

.ch and .li domains via AXFR

+
+
+ +
+
+ {switchTlds.map(zone => ( +
+
+ .{zone.tld} + {zone.total_domains?.toLocaleString()} domains +
+
+ {zone.drops_24h || 0} drops + + {zone.last_sync ? new Date(zone.last_sync).toLocaleString() : 'Never'} + +
+
+ ))} +
+
+ + {/* ICANN CZDS Section */} +
+
+
+
+ +
+
+

ICANN CZDS Zone Sync

+

gTLD zone files (.xyz, .org, .info, etc.)

+
+
+ +
+
+ {czdsTlds.map(zone => ( +
+
+ .{zone.tld} + {zone.total_domains?.toLocaleString()} domains +
+
+ {zone.drops_24h || 0} drops + + {zone.last_sync ? new Date(zone.last_sync).toLocaleString() : 'Never'} + +
+
+ ))} + {czdsTlds.length === 0 && ( +
+ No CZDS zones synced yet. Click "Sync Now" to start. +
+ )} +
+
+ + {/* Info Box */} +
+ +
+

Automatic Sync Schedule

+

Switch.ch (.ch, .li): Daily at 05:00 UTC

+

ICANN CZDS (gTLDs): Daily at 06:00 UTC

+

Zone files are processed with 24 parallel workers for maximum speed.

+
+
+
+ ) +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 7bf7ab6..ea400ff 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1616,6 +1616,28 @@ class AdminApiClient extends ApiClient { }>(`/admin/activity-log?limit=${limit}&offset=${offset}`) } + // ============== Zone Sync ============== + + async getZoneStats() { + return this.request<{ + zones: Array<{ + tld: string + total_domains: number + drops_24h: number + drops_48h: number + last_sync: string | null + }> + }>('/admin/zone-stats') + } + + async triggerSwitchSync() { + return this.request<{ status: string; message: string }>('/admin/zone-sync/switch', { method: 'POST' }) + } + + async triggerCzdsSync() { + return this.request<{ status: string; message: string }>('/admin/zone-sync/czds', { method: 'POST' }) + } + // ============== Blog ============== async getBlogPosts(limit = 10, offset = 0, category?: string, tag?: string) {