feat: add ICANN CZDS zone file integration for gTLDs

backend/app/api/drops.py

@@ -1,7 +1,9 @@
 """
 Drops API - Zone File Analysis Endpoints
 =========================================
-API endpoints for accessing freshly dropped .ch and .li domains.
+API endpoints for accessing freshly dropped domains from:
+- Switch.ch zone files (.ch, .li)
+- ICANN CZDS zone files (.com, .net, .org, .xyz, .info, .dev, .app, .online)
 """
 
 from datetime import datetime
@@ -20,6 +22,12 @@ from app.services.zone_file import (
 
 router = APIRouter(prefix="/drops", tags=["drops"])
 
+# All supported TLDs
+SWITCH_TLDS = ["ch", "li"]
+CZDS_TLDS = ["xyz", "org", "online", "info", "dev", "app"]  # Approved
+CZDS_PENDING = ["com", "net", "club", "biz"]  # Pending approval
+ALL_TLDS = SWITCH_TLDS + CZDS_TLDS
+
 
 # ============================================================================
 # PUBLIC ENDPOINTS (for stats)
@@ -46,7 +54,7 @@ async def api_get_zone_stats(
 
 @router.get("")
 async def api_get_drops(
-    tld: Optional[str] = Query(None, description="Filter by TLD (ch or li)"),
+    tld: Optional[str] = Query(None, description="Filter by TLD"),
     days: int = Query(7, ge=1, le=30, description="Days to look back"),
     min_length: Optional[int] = Query(None, ge=1, le=63, description="Minimum domain length"),
     max_length: Optional[int] = Query(None, ge=1, le=63, description="Maximum domain length"),
@@ -59,13 +67,20 @@ async def api_get_drops(
     current_user = Depends(get_current_user)
 ):
     """
-    Get recently dropped domains from .ch and .li zone files.
+    Get recently dropped domains from zone files.
+    
+    Supports:
+    - Switch.ch zones: .ch, .li
+    - ICANN CZDS zones: .xyz, .org, .online, .info, .dev, .app
     
     Domains are detected by comparing daily zone file snapshots.
     Only available for authenticated users.
     """
-    if tld and tld not in ["ch", "li"]:
+    if tld and tld not in ALL_TLDS:
-        raise HTTPException(status_code=400, detail="TLD must be 'ch' or 'li'")
+        raise HTTPException(
+            status_code=400, 
+            detail=f"Unsupported TLD. Supported: {', '.join(ALL_TLDS)}"
+        )
     
     try:
         result = await get_dropped_domains(
@@ -102,19 +117,28 @@ async def api_trigger_sync(
     if not getattr(current_user, 'is_admin', False):
         raise HTTPException(status_code=403, detail="Admin access required")
     
-    if tld not in ["ch", "li"]:
+    if tld not in ALL_TLDS:
-        raise HTTPException(status_code=400, detail="TLD must be 'ch' or 'li'")
+        raise HTTPException(
-    
+            status_code=400, 
-    # Run sync in background
+            detail=f"Unsupported TLD. Supported: {', '.join(ALL_TLDS)}"
-    service = ZoneFileService()
+        )
     
     async def run_sync():
-        async for session in get_db():
+        from app.database import AsyncSessionLocal
+        
+        async with AsyncSessionLocal() as session:
             try:
+                if tld in SWITCH_TLDS:
+                    # Use Switch.ch zone transfer
+                    service = ZoneFileService()
                     await service.run_daily_sync(session, tld)
+                else:
+                    # Use ICANN CZDS
+                    from app.services.czds_client import CZDSClient
+                    client = CZDSClient()
+                    await client.sync_zone(session, tld)
             except Exception as e:
                 print(f"Zone sync failed for .{tld}: {e}")
-            break
     
     background_tasks.add_task(run_sync)
     
@@ -132,17 +156,22 @@ async def api_get_supported_tlds():
     """
     return {
         "tlds": [
-            {
+            # Switch.ch zones
-                "tld": "ch",
+            {"tld": "ch", "name": "Switzerland", "flag": "🇨🇭", "registry": "Switch", "source": "switch"},
-                "name": "Switzerland",
+            {"tld": "li", "name": "Liechtenstein", "flag": "🇱🇮", "registry": "Switch", "source": "switch"},
-                "flag": "🇨🇭",
+            # ICANN CZDS zones (approved)
-                "registry": "Switch"
+            {"tld": "xyz", "name": "XYZ", "flag": "🌐", "registry": "XYZ.COM LLC", "source": "czds"},
-            },
+            {"tld": "org", "name": "Organization", "flag": "🏛️", "registry": "PIR", "source": "czds"},
-            {
+            {"tld": "online", "name": "Online", "flag": "💻", "registry": "Radix", "source": "czds"},
-                "tld": "li",
+            {"tld": "info", "name": "Information", "flag": "ℹ️", "registry": "Afilias", "source": "czds"},
-                "name": "Liechtenstein", 
+            {"tld": "dev", "name": "Developer", "flag": "👨‍💻", "registry": "Google", "source": "czds"},
-                "flag": "🇱🇮",
+            {"tld": "app", "name": "Application", "flag": "📱", "registry": "Google", "source": "czds"},
-                "registry": "Switch"
+        ],
-            }
+        "pending": [
+            # CZDS pending approval
+            {"tld": "com", "name": "Commercial", "flag": "🏢", "registry": "Verisign", "source": "czds"},
+            {"tld": "net", "name": "Network", "flag": "🌐", "registry": "Verisign", "source": "czds"},
+            {"tld": "club", "name": "Club", "flag": "🎉", "registry": "GoDaddy", "source": "czds"},
+            {"tld": "biz", "name": "Business", "flag": "💼", "registry": "GoDaddy", "source": "czds"},
         ]
     }

backend/app/config.py

@@ -117,6 +117,13 @@ class Settings(BaseSettings):
     moz_access_id: str = ""
     moz_secret_key: str = ""
     
+    # ICANN CZDS (Centralized Zone Data Service)
+    # For downloading gTLD zone files (.com, .net, .org, etc.)
+    # Register at: https://czds.icann.org/
+    czds_username: str = ""
+    czds_password: str = ""
+    czds_data_dir: str = "/tmp/pounce_czds"
+    
     class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"

backend/app/scheduler.py

@@ -662,7 +662,16 @@ def setup_scheduler():
         sync_zone_files,
         CronTrigger(hour=5, minute=0),  # Daily at 05:00 UTC
         id="zone_file_sync",
-        name="Zone File Sync (daily)",
+        name="Zone File Sync - Switch.ch (daily)",
+        replace_existing=True,
+    )
+    
+    # CZDS zone file sync for gTLDs (daily at 06:00 UTC, after Switch sync)
+    scheduler.add_job(
+        sync_czds_zones,
+        CronTrigger(hour=6, minute=0),  # Daily at 06:00 UTC
+        id="czds_zone_sync",
+        name="Zone File Sync - ICANN CZDS (daily)",
         replace_existing=True,
     )
     
@@ -842,7 +851,7 @@ async def scrape_auctions():
 
 
 async def sync_zone_files():
-    """Sync zone files for .ch and .li domains from Switch.ch."""
+    """Sync zone files from Switch.ch (.ch, .li) and ICANN CZDS (gTLDs)."""
     logger.info("Starting zone file sync...")
     
     try:
@@ -851,26 +860,49 @@ async def sync_zone_files():
         service = ZoneFileService()
         
         async with AsyncSessionLocal() as db:
-            # Sync .ch zone
+            # Sync Switch.ch zones (.ch, .li)
+            for tld in ["ch", "li"]:
                 try:
-                ch_result = await service.run_daily_sync(db, "ch")
+                    result = await service.run_daily_sync(db, tld)
-                logger.info(f".ch zone sync: {len(ch_result.get('dropped', []))} dropped, {ch_result.get('new_count', 0)} new")
+                    logger.info(f".{tld} zone sync: {len(result.get('dropped', []))} dropped, {result.get('new_count', 0)} new")
                 except Exception as e:
-                logger.error(f".ch zone sync failed: {e}")
+                    logger.error(f".{tld} zone sync failed: {e}")
         
-            # Sync .li zone
+        logger.info("Switch.ch zone file sync completed")
-            try:
-                li_result = await service.run_daily_sync(db, "li")
-                logger.info(f".li zone sync: {len(li_result.get('dropped', []))} dropped, {li_result.get('new_count', 0)} new")
-            except Exception as e:
-                logger.error(f".li zone sync failed: {e}")
-        
-        logger.info("Zone file sync completed")
         
     except Exception as e:
         logger.exception(f"Zone file sync failed: {e}")
 
 
+async def sync_czds_zones():
+    """Sync zone files from ICANN CZDS (gTLDs like .xyz, .org, .dev, .app)."""
+    logger.info("Starting CZDS zone file sync...")
+    
+    try:
+        from app.services.czds_client import CZDSClient, APPROVED_TLDS
+        from app.config import get_settings
+        
+        settings = get_settings()
+        
+        # Skip if credentials not configured
+        if not settings.czds_username or not settings.czds_password:
+            logger.info("CZDS credentials not configured, skipping CZDS sync")
+            return
+        
+        client = CZDSClient()
+        
+        async with AsyncSessionLocal() as db:
+            results = await client.sync_all_zones(db, APPROVED_TLDS)
+            
+            success_count = sum(1 for r in results if r["status"] == "success")
+            total_dropped = sum(r["dropped_count"] for r in results)
+            
+            logger.info(f"CZDS sync complete: {success_count}/{len(APPROVED_TLDS)} zones, {total_dropped:,} dropped")
+        
+    except Exception as e:
+        logger.exception(f"CZDS zone file sync failed: {e}")
+
+
 async def match_sniper_alerts():
     """Match active sniper alerts against current auctions and notify users."""
     from app.models.sniper_alert import SniperAlert, SniperAlertMatch

backend/app/services/czds_client.py

@@ -0,0 +1,456 @@
+"""
+ICANN CZDS (Centralized Zone Data Service) Client
+==================================================
+Downloads zone files from ICANN CZDS, parses them, and detects dropped domains.
+
+Authentication: OAuth2 with username/password
+Zone Format: Standard DNS zone file format (.txt.gz)
+
+Usage:
+    client = CZDSClient(username, password)
+    await client.sync_all_zones(db)
+"""
+
+import asyncio
+import gzip
+import hashlib
+import logging
+import os
+import re
+import shutil
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional
+
+import httpx
+from sqlalchemy import select, func, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.models.zone_file import ZoneSnapshot, DroppedDomain
+
+logger = logging.getLogger(__name__)
+settings = get_settings()
+
+# ============================================================================
+# CONSTANTS
+# ============================================================================
+
+CZDS_AUTH_URL = "https://account-api.icann.org/api/authenticate"
+CZDS_ZONES_URL = "https://czds-api.icann.org/czds/downloads/links"
+CZDS_DOWNLOAD_BASE = "https://czds-download.icann.org"
+
+# TLDs we have approved access to
+APPROVED_TLDS = ["xyz", "org", "online", "info", "dev", "app"]
+
+# Regex to extract domain names from zone file NS records
+# Format: example.tld.    IN    NS    ns1.example.com.
+NS_RECORD_PATTERN = re.compile(r'^([a-z0-9][-a-z0-9]*)\.[a-z]+\.\s+\d*\s*IN\s+NS\s+', re.IGNORECASE)
+
+
+# ============================================================================
+# CZDS CLIENT
+# ============================================================================
+
+class CZDSClient:
+    """Client for ICANN CZDS zone file downloads."""
+    
+    def __init__(
+        self, 
+        username: Optional[str] = None, 
+        password: Optional[str] = None,
+        data_dir: Optional[Path] = None
+    ):
+        self.username = username or os.getenv("CZDS_USERNAME") or settings.czds_username
+        self.password = password or os.getenv("CZDS_PASSWORD") or settings.czds_password
+        self.data_dir = data_dir or Path(os.getenv("CZDS_DATA_DIR", "/tmp/pounce_czds"))
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        
+        self._token: Optional[str] = None
+        self._token_expires: Optional[datetime] = None
+    
+    async def _authenticate(self) -> str:
+        """Authenticate with ICANN and get access token."""
+        if self._token and self._token_expires and datetime.utcnow() < self._token_expires:
+            return self._token
+        
+        if not self.username or not self.password:
+            raise ValueError("CZDS credentials not configured. Set CZDS_USERNAME and CZDS_PASSWORD.")
+        
+        logger.info("Authenticating with ICANN CZDS...")
+        
+        async with httpx.AsyncClient(timeout=30) as client:
+            response = await client.post(
+                CZDS_AUTH_URL,
+                json={"username": self.username, "password": self.password},
+                headers={"Content-Type": "application/json"}
+            )
+            
+            if response.status_code != 200:
+                logger.error(f"CZDS authentication failed: {response.status_code} {response.text}")
+                raise RuntimeError(f"CZDS authentication failed: {response.status_code}")
+            
+            data = response.json()
+            self._token = data.get("accessToken")
+            
+            # Token expires in 24 hours, refresh after 23 hours
+            self._token_expires = datetime.utcnow() + timedelta(hours=23)
+            
+            logger.info("CZDS authentication successful")
+            return self._token
+    
+    async def get_available_zones(self) -> list[str]:
+        """Get list of zone files available for download."""
+        token = await self._authenticate()
+        
+        async with httpx.AsyncClient(timeout=60) as client:
+            response = await client.get(
+                CZDS_ZONES_URL,
+                headers={"Authorization": f"Bearer {token}"}
+            )
+            
+            if response.status_code != 200:
+                logger.error(f"Failed to get zone list: {response.status_code}")
+                return []
+            
+            # Response is a list of download URLs
+            urls = response.json()
+            
+            # Extract TLDs from URLs
+            tlds = []
+            for url in urls:
+                # URL format: https://czds-download.icann.org/czds/downloads/xyz.zone
+                match = re.search(r'/([a-z0-9-]+)\.zone$', url, re.IGNORECASE)
+                if match:
+                    tlds.append(match.group(1).lower())
+            
+            logger.info(f"Available zones: {tlds}")
+            return tlds
+    
+    async def download_zone(self, tld: str) -> Optional[Path]:
+        """Download a zone file for a specific TLD."""
+        token = await self._authenticate()
+        
+        download_url = f"{CZDS_DOWNLOAD_BASE}/czds/downloads/{tld}.zone"
+        output_path = self.data_dir / f"{tld}.zone.txt.gz"
+        
+        logger.info(f"Downloading zone file for .{tld}...")
+        
+        async with httpx.AsyncClient(timeout=600, follow_redirects=True) as client:
+            try:
+                async with client.stream(
+                    "GET",
+                    download_url,
+                    headers={"Authorization": f"Bearer {token}"}
+                ) as response:
+                    if response.status_code != 200:
+                        logger.error(f"Failed to download .{tld}: {response.status_code}")
+                        return None
+                    
+                    # Stream to file
+                    with open(output_path, "wb") as f:
+                        async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
+                            f.write(chunk)
+                
+                file_size = output_path.stat().st_size / (1024 * 1024)
+                logger.info(f"Downloaded .{tld} zone file: {file_size:.1f} MB")
+                return output_path
+                
+            except Exception as e:
+                logger.error(f"Error downloading .{tld}: {e}")
+                return None
+    
+    def extract_zone_file(self, gz_path: Path) -> Path:
+        """Extract gzipped zone file."""
+        output_path = gz_path.with_suffix('')  # Remove .gz
+        
+        logger.info(f"Extracting {gz_path.name}...")
+        
+        with gzip.open(gz_path, 'rb') as f_in:
+            with open(output_path, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        
+        # Remove gz file to save space
+        gz_path.unlink()
+        
+        return output_path
+    
+    def parse_zone_file(self, zone_path: Path, tld: str) -> set[str]:
+        """
+        Parse zone file and extract unique domain names.
+        
+        Zone files contain various record types. We extract domains from:
+        - NS records (most reliable indicator of active domain)
+        - A/AAAA records
+        
+        Returns set of domain names (without TLD suffix).
+        """
+        logger.info(f"Parsing zone file for .{tld}...")
+        
+        domains = set()
+        line_count = 0
+        
+        with open(zone_path, 'r', encoding='utf-8', errors='ignore') as f:
+            for line in f:
+                line_count += 1
+                
+                # Skip comments and empty lines
+                if line.startswith(';') or not line.strip():
+                    continue
+                
+                # Look for NS records which indicate delegated domains
+                # Format: example.tld.    86400    IN    NS    ns1.registrar.com.
+                parts = line.split()
+                if len(parts) >= 4:
+                    # First column is the domain name
+                    name = parts[0].rstrip('.')
+                    
+                    # Must end with our TLD
+                    if name.lower().endswith(f'.{tld}'):
+                        # Extract just the domain name part
+                        domain_name = name[:-(len(tld) + 1)]
+                        
+                        # Skip the TLD itself and subdomains
+                        if domain_name and '.' not in domain_name:
+                            domains.add(domain_name.lower())
+        
+        logger.info(f"Parsed .{tld}: {len(domains):,} unique domains from {line_count:,} lines")
+        return domains
+    
+    def compute_checksum(self, domains: set[str]) -> str:
+        """Compute SHA256 checksum of sorted domain list."""
+        sorted_domains = "\n".join(sorted(domains))
+        return hashlib.sha256(sorted_domains.encode()).hexdigest()
+    
+    async def get_previous_domains(self, tld: str) -> Optional[set[str]]:
+        """Load previous day's domain set from cache file."""
+        cache_file = self.data_dir / f"{tld}_domains.txt"
+        
+        if cache_file.exists():
+            try:
+                content = cache_file.read_text()
+                return set(line.strip() for line in content.splitlines() if line.strip())
+            except Exception as e:
+                logger.warning(f"Failed to load cache for .{tld}: {e}")
+        
+        return None
+    
+    async def save_domains(self, tld: str, domains: set[str]):
+        """Save current domains to cache file."""
+        cache_file = self.data_dir / f"{tld}_domains.txt"
+        cache_file.write_text("\n".join(sorted(domains)))
+        logger.info(f"Saved {len(domains):,} domains for .{tld}")
+    
+    async def process_drops(
+        self, 
+        db: AsyncSession, 
+        tld: str, 
+        previous: set[str], 
+        current: set[str]
+    ) -> list[dict]:
+        """Find and store dropped domains."""
+        dropped = previous - current
+        
+        if not dropped:
+            logger.info(f"No dropped domains found for .{tld}")
+            return []
+        
+        logger.info(f"Found {len(dropped):,} dropped domains for .{tld}")
+        
+        today = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+        
+        # Batch insert for performance
+        dropped_records = []
+        batch_size = 1000
+        batch = []
+        
+        for name in dropped:
+            record = DroppedDomain(
+                domain=f"{name}.{tld}",
+                tld=tld,
+                dropped_date=today,
+                length=len(name),
+                is_numeric=name.isdigit(),
+                has_hyphen='-' in name
+            )
+            batch.append(record)
+            dropped_records.append({
+                "domain": f"{name}.{tld}",
+                "length": len(name),
+                "is_numeric": name.isdigit(),
+                "has_hyphen": '-' in name
+            })
+            
+            if len(batch) >= batch_size:
+                db.add_all(batch)
+                await db.flush()
+                batch = []
+        
+        # Add remaining
+        if batch:
+            db.add_all(batch)
+        
+        await db.commit()
+        
+        return dropped_records
+    
+    async def sync_zone(self, db: AsyncSession, tld: str) -> dict:
+        """
+        Sync a single zone file:
+        1. Download zone file
+        2. Extract and parse
+        3. Compare with previous snapshot
+        4. Store dropped domains
+        5. Save new snapshot
+        """
+        logger.info(f"Starting sync for .{tld}")
+        
+        result = {
+            "tld": tld,
+            "status": "pending",
+            "current_count": 0,
+            "previous_count": 0,
+            "dropped_count": 0,
+            "new_count": 0,
+            "error": None
+        }
+        
+        try:
+            # Download zone file
+            gz_path = await self.download_zone(tld)
+            if not gz_path:
+                result["status"] = "download_failed"
+                result["error"] = "Failed to download zone file"
+                return result
+            
+            # Extract
+            zone_path = self.extract_zone_file(gz_path)
+            
+            # Parse
+            current_domains = self.parse_zone_file(zone_path, tld)
+            result["current_count"] = len(current_domains)
+            
+            # Clean up zone file (can be very large)
+            zone_path.unlink()
+            
+            # Get previous snapshot
+            previous_domains = await self.get_previous_domains(tld)
+            
+            if previous_domains:
+                result["previous_count"] = len(previous_domains)
+                
+                # Find dropped domains
+                dropped = await self.process_drops(db, tld, previous_domains, current_domains)
+                result["dropped_count"] = len(dropped)
+                result["new_count"] = len(current_domains - previous_domains)
+            
+            # Save current snapshot
+            await self.save_domains(tld, current_domains)
+            
+            # Save snapshot metadata
+            checksum = self.compute_checksum(current_domains)
+            snapshot = ZoneSnapshot(
+                tld=tld,
+                snapshot_date=datetime.utcnow(),
+                domain_count=len(current_domains),
+                checksum=checksum
+            )
+            db.add(snapshot)
+            await db.commit()
+            
+            result["status"] = "success"
+            logger.info(
+                f"Sync complete for .{tld}: "
+                f"{result['current_count']:,} domains, "
+                f"{result['dropped_count']:,} dropped, "
+                f"{result['new_count']:,} new"
+            )
+            
+        except Exception as e:
+            logger.exception(f"Error syncing .{tld}: {e}")
+            result["status"] = "error"
+            result["error"] = str(e)
+        
+        return result
+    
+    async def sync_all_zones(
+        self, 
+        db: AsyncSession, 
+        tlds: Optional[list[str]] = None
+    ) -> list[dict]:
+        """
+        Sync all approved zone files.
+        
+        Args:
+            db: Database session
+            tlds: Optional list of TLDs to sync. Defaults to APPROVED_TLDS.
+        
+        Returns:
+            List of sync results for each TLD.
+        """
+        tlds = tlds or APPROVED_TLDS
+        
+        logger.info(f"Starting CZDS sync for {len(tlds)} zones: {tlds}")
+        
+        results = []
+        for tld in tlds:
+            result = await self.sync_zone(db, tld)
+            results.append(result)
+            
+            # Small delay between zones to be nice to ICANN servers
+            await asyncio.sleep(2)
+        
+        # Summary
+        success_count = sum(1 for r in results if r["status"] == "success")
+        total_dropped = sum(r["dropped_count"] for r in results)
+        
+        logger.info(
+            f"CZDS sync complete: "
+            f"{success_count}/{len(tlds)} zones successful, "
+            f"{total_dropped:,} total dropped domains"
+        )
+        
+        return results
+
+
+# ============================================================================
+# STANDALONE SCRIPT
+# ============================================================================
+
+async def main():
+    """Standalone script to run CZDS sync."""
+    import sys
+    from app.database import AsyncSessionLocal, init_db
+    
+    # Initialize database
+    await init_db()
+    
+    # Parse arguments
+    tlds = sys.argv[1:] if len(sys.argv) > 1 else APPROVED_TLDS
+    
+    print(f"🌐 CZDS Zone File Sync")
+    print(f"📂 TLDs: {', '.join(tlds)}")
+    print("-" * 50)
+    
+    client = CZDSClient()
+    
+    async with AsyncSessionLocal() as db:
+        results = await client.sync_all_zones(db, tlds)
+    
+    print("\n" + "=" * 50)
+    print("📊 RESULTS")
+    print("=" * 50)
+    
+    for r in results:
+        status_icon = "✅" if r["status"] == "success" else "❌"
+        print(f"{status_icon} .{r['tld']}: {r['current_count']:,} domains, "
+              f"{r['dropped_count']:,} dropped, {r['new_count']:,} new")
+        if r["error"]:
+            print(f"   ⚠️ Error: {r['error']}")
+    
+    total_dropped = sum(r["dropped_count"] for r in results)
+    print(f"\n🎯 Total dropped domains: {total_dropped:,}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/scripts/sync_czds.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+CZDS Zone File Sync Script
+==========================
+Manually sync zone files from ICANN CZDS.
+
+Usage:
+    # Sync all approved TLDs
+    python scripts/sync_czds.py
+    
+    # Sync specific TLDs
+    python scripts/sync_czds.py xyz org dev
+    
+Environment:
+    CZDS_USERNAME: ICANN CZDS username (email)
+    CZDS_PASSWORD: ICANN CZDS password
+    CZDS_DATA_DIR: Directory for zone file cache (default: /tmp/pounce_czds)
+"""
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+# Add backend to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+load_dotenv()
+
+
+async def main():
+    from app.database import AsyncSessionLocal, init_db
+    from app.services.czds_client import CZDSClient, APPROVED_TLDS
+    
+    # Parse arguments
+    tlds = sys.argv[1:] if len(sys.argv) > 1 else APPROVED_TLDS
+    
+    # Check credentials
+    username = os.getenv("CZDS_USERNAME")
+    password = os.getenv("CZDS_PASSWORD")
+    
+    if not username or not password:
+        print("❌ CZDS credentials not configured!")
+        print()
+        print("Set these environment variables:")
+        print("  export CZDS_USERNAME='your-icann-email'")
+        print("  export CZDS_PASSWORD='your-password'")
+        print()
+        print("Or add to .env file:")
+        print("  CZDS_USERNAME=your-icann-email")
+        print("  CZDS_PASSWORD=your-password")
+        sys.exit(1)
+    
+    print("🌐 ICANN CZDS Zone File Sync")
+    print("=" * 50)
+    print(f"📧 Username: {username}")
+    print(f"📂 TLDs: {', '.join(tlds)}")
+    print(f"💾 Cache Dir: {os.getenv('CZDS_DATA_DIR', '/tmp/pounce_czds')}")
+    print("=" * 50)
+    print()
+    
+    # Initialize database
+    print("📦 Initializing database...")
+    await init_db()
+    
+    # Create client and sync
+    client = CZDSClient()
+    
+    print("\n🔐 Authenticating with ICANN...")
+    
+    async with AsyncSessionLocal() as db:
+        print("\n📥 Starting zone file downloads...")
+        print("   (This may take a while for large zones)")
+        print()
+        
+        results = await client.sync_all_zones(db, tlds)
+    
+    # Summary
+    print("\n" + "=" * 50)
+    print("📊 SYNC RESULTS")
+    print("=" * 50)
+    
+    total_domains = 0
+    total_dropped = 0
+    total_new = 0
+    
+    for r in results:
+        status_icon = "✅" if r["status"] == "success" else "❌"
+        print(f"\n{status_icon} .{r['tld']}")
+        
+        if r["status"] == "success":
+            print(f"   Domains:  {r['current_count']:>12,}")
+            print(f"   Previous: {r['previous_count']:>12,}")
+            print(f"   Dropped:  {r['dropped_count']:>12,}")
+            print(f"   New:      {r['new_count']:>12,}")
+            
+            total_domains += r['current_count']
+            total_dropped += r['dropped_count']
+            total_new += r['new_count']
+        else:
+            print(f"   Error: {r.get('error', 'Unknown error')}")
+    
+    print("\n" + "-" * 50)
+    print(f"📈 TOTALS")
+    print(f"   Total Domains: {total_domains:>12,}")
+    print(f"   Total Dropped: {total_dropped:>12,}")
+    print(f"   Total New:     {total_new:>12,}")
+    print()
+    
+    if total_dropped > 0:
+        print(f"🎯 {total_dropped:,} dropped domains available in the database!")
+        print("   View them at: https://pounce.ch/terminal/hunt -> Drops tab")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

frontend/src/components/hunt/DropsTab.tsx

@@ -65,8 +65,11 @@ export function DropsTab({ showToast }: DropsTabProps) {
   const [refreshing, setRefreshing] = useState(false)
   const [total, setTotal] = useState(0)
 
+  // All supported TLDs
+  type SupportedTld = 'ch' | 'li' | 'xyz' | 'org' | 'online' | 'info' | 'dev' | 'app'
+  
   // Filter State
-  const [selectedTld, setSelectedTld] = useState<'ch' | 'li' | null>(null)
+  const [selectedTld, setSelectedTld] = useState<SupportedTld | null>(null)
   const [searchQuery, setSearchQuery] = useState('')
   const [searchFocused, setSearchFocused] = useState(false)
   const [days, setDays] = useState(7)
@@ -217,32 +220,6 @@ export function DropsTab({ showToast }: DropsTabProps) {
       {/* Stats Cards */}
       {stats && (
         <div className="grid grid-cols-2 lg:grid-cols-4 gap-3">
-          <div className="border border-white/[0.08] bg-[#020202] p-3">
-            <div className="flex items-center gap-2 mb-1">
-              <span className="text-lg">🇨🇭</span>
-              <span className="text-[10px] font-mono text-white/40 uppercase">.ch Zone</span>
-            </div>
-            <div className="text-xl font-bold text-white font-mono">
-              {stats.ch.domain_count.toLocaleString()}
-            </div>
-            <div className="text-[10px] font-mono text-white/30">
-              {stats.ch.last_sync ? `Last: ${new Date(stats.ch.last_sync).toLocaleDateString()}` : 'Not synced'}
-            </div>
-          </div>
-
-          <div className="border border-white/[0.08] bg-[#020202] p-3">
-            <div className="flex items-center gap-2 mb-1">
-              <span className="text-lg">🇱🇮</span>
-              <span className="text-[10px] font-mono text-white/40 uppercase">.li Zone</span>
-            </div>
-            <div className="text-xl font-bold text-white font-mono">
-              {stats.li.domain_count.toLocaleString()}
-            </div>
-            <div className="text-[10px] font-mono text-white/30">
-              {stats.li.last_sync ? `Last: ${new Date(stats.li.last_sync).toLocaleDateString()}` : 'Not synced'}
-            </div>
-          </div>
-
           <div className="border border-accent/20 bg-accent/[0.03] p-3">
             <div className="flex items-center gap-2 mb-1">
               <Zap className="w-4 h-4 text-accent" />
@@ -258,12 +235,41 @@ export function DropsTab({ showToast }: DropsTabProps) {
 
           <div className="border border-white/[0.08] bg-[#020202] p-3">
             <div className="flex items-center gap-2 mb-1">
-              <Clock className="w-4 h-4 text-white/40" />
+              <span className="text-sm">🇨🇭🇱🇮</span>
-              <span className="text-[10px] font-mono text-white/40 uppercase">Data Source</span>
+              <span className="text-[10px] font-mono text-white/40 uppercase">Switch.ch</span>
+            </div>
+            <div className="text-lg font-bold text-white font-mono">
+              {((stats.ch?.domain_count || 0) + (stats.li?.domain_count || 0)).toLocaleString()}
             </div>
-            <div className="text-sm font-bold text-white">Switch.ch</div>
             <div className="text-[10px] font-mono text-white/30">
-              Official zone files
+              .ch + .li zones
+            </div>
+          </div>
+
+          <div className="border border-white/[0.08] bg-[#020202] p-3">
+            <div className="flex items-center gap-2 mb-1">
+              <Globe className="w-4 h-4 text-white/40" />
+              <span className="text-[10px] font-mono text-white/40 uppercase">ICANN CZDS</span>
+            </div>
+            <div className="text-lg font-bold text-white font-mono">6 TLDs</div>
+            <div className="text-[10px] font-mono text-white/30">
+              .xyz .org .dev .app ...
+            </div>
+          </div>
+
+          <div className="border border-white/[0.08] bg-[#020202] p-3">
+            <div className="flex items-center gap-2 mb-1">
+              <Clock className="w-4 h-4 text-white/40" />
+              <span className="text-[10px] font-mono text-white/40 uppercase">Last Sync</span>
+            </div>
+            <div className="text-sm font-bold text-white">
+              {stats.ch?.last_sync 
+                ? new Date(stats.ch.last_sync).toLocaleDateString() 
+                : 'Pending'
+              }
+            </div>
+            <div className="text-[10px] font-mono text-white/30">
+              Daily @ 05:00 UTC
             </div>
           </div>
         </div>
@@ -299,34 +305,54 @@ export function DropsTab({ showToast }: DropsTabProps) {
         </div>
 
         {/* TLD Quick Filter */}
-        <div className="flex gap-2">
+        <div className="flex gap-2 flex-wrap">
           <button
             onClick={() => setSelectedTld(null)}
             className={clsx(
-              "px-4 py-2 text-xs font-mono uppercase border transition-colors",
+              "px-3 py-1.5 text-[10px] font-mono uppercase border transition-colors",
               selectedTld === null ? "border-accent bg-accent/10 text-accent" : "border-white/[0.08] text-white/40"
             )}
           >
             All
           </button>
+          {/* Switch.ch TLDs */}
+          {[
+            { tld: 'ch', flag: '🇨🇭' },
+            { tld: 'li', flag: '🇱🇮' },
+          ].map(({ tld, flag }) => (
             <button
-            onClick={() => setSelectedTld('ch')}
+              key={tld}
+              onClick={() => setSelectedTld(tld as 'ch' | 'li')}
               className={clsx(
-              "px-4 py-2 text-xs font-mono uppercase border transition-colors flex items-center gap-2",
+                "px-3 py-1.5 text-[10px] font-mono uppercase border transition-colors flex items-center gap-1.5",
-              selectedTld === 'ch' ? "border-accent bg-accent/10 text-accent" : "border-white/[0.08] text-white/40"
+                selectedTld === tld ? "border-accent bg-accent/10 text-accent" : "border-white/[0.08] text-white/40"
               )}
             >
-            🇨🇭 .ch
+              <span>{flag}</span> .{tld}
             </button>
+          ))}
+          {/* Separator */}
+          <div className="w-px h-6 bg-white/10 self-center mx-1" />
+          {/* CZDS TLDs */}
+          {[
+            { tld: 'xyz', flag: '🌐' },
+            { tld: 'org', flag: '🏛️' },
+            { tld: 'online', flag: '💻' },
+            { tld: 'info', flag: 'ℹ️' },
+            { tld: 'dev', flag: '👨‍💻' },
+            { tld: 'app', flag: '📱' },
+          ].map(({ tld, flag }) => (
             <button
-            onClick={() => setSelectedTld('li')}
+              key={tld}
+              onClick={() => setSelectedTld(tld as any)}
               className={clsx(
-              "px-4 py-2 text-xs font-mono uppercase border transition-colors flex items-center gap-2",
+                "px-3 py-1.5 text-[10px] font-mono uppercase border transition-colors flex items-center gap-1.5",
-              selectedTld === 'li' ? "border-accent bg-accent/10 text-accent" : "border-white/[0.08] text-white/40"
+                selectedTld === tld ? "border-accent bg-accent/10 text-accent" : "border-white/[0.08] text-white/40"
               )}
             >
-            🇱🇮 .li
+              <span>{flag}</span> .{tld}
             </button>
+          ))}
         </div>
 
         {/* Filter Toggle */}
@@ -622,11 +648,19 @@ export function DropsTab({ showToast }: DropsTabProps) {
           </div>
           <div>
             <h3 className="text-sm font-bold text-white mb-1">Zone File Analysis</h3>
-            <p className="text-xs text-white/40 leading-relaxed">
+            <p className="text-xs text-white/40 leading-relaxed mb-2">
-              Domains are detected by comparing daily zone file snapshots from Switch.ch.
+              Domains are detected by comparing daily zone file snapshots. Data sources:
-              Data is updated automatically every 24 hours. Only .ch and .li domains are supported
-              as these zones are publicly available from the Swiss registry.
             </p>
+            <div className="grid grid-cols-1 sm:grid-cols-2 gap-2 text-[10px] font-mono">
+              <div className="flex items-center gap-2 text-white/50">
+                <span>🇨🇭</span>
+                <span><strong>.ch/.li</strong> via Switch.ch (AXFR)</span>
+              </div>
+              <div className="flex items-center gap-2 text-white/50">
+                <span>🌐</span>
+                <span><strong>gTLDs</strong> via ICANN CZDS</span>
+              </div>
+            </div>
           </div>
         </div>
       </div>

frontend/src/lib/api.ts

@@ -1844,7 +1844,7 @@ class AdminApiClient extends ApiClient {
   }
 
   async getDrops(params?: {
-    tld?: 'ch' | 'li'
+    tld?: 'ch' | 'li' | 'xyz' | 'org' | 'online' | 'info' | 'dev' | 'app'
     days?: number
     min_length?: number
     max_length?: number