From 90256e60493e8674c5fb8e98b011a4d47aaf7a51 Mon Sep 17 00:00:00 2001 From: Yves Gugger Date: Mon, 15 Dec 2025 22:07:23 +0100 Subject: [PATCH] feat: add ICANN CZDS zone file integration for gTLDs --- backend/app/api/drops.py | 79 ++-- backend/app/config.py | 7 + backend/app/scheduler.py | 64 ++- backend/app/services/czds_client.py | 456 ++++++++++++++++++++++ backend/scripts/sync_czds.py | 117 ++++++ frontend/src/components/hunt/DropsTab.tsx | 144 ++++--- frontend/src/lib/api.ts | 2 +- 7 files changed, 772 insertions(+), 97 deletions(-) create mode 100644 backend/app/services/czds_client.py create mode 100644 backend/scripts/sync_czds.py diff --git a/backend/app/api/drops.py b/backend/app/api/drops.py index dd9703e..b932be3 100644 --- a/backend/app/api/drops.py +++ b/backend/app/api/drops.py @@ -1,7 +1,9 @@ """ Drops API - Zone File Analysis Endpoints ========================================= -API endpoints for accessing freshly dropped .ch and .li domains. +API endpoints for accessing freshly dropped domains from: +- Switch.ch zone files (.ch, .li) +- ICANN CZDS zone files (.com, .net, .org, .xyz, .info, .dev, .app, .online) """ from datetime import datetime @@ -20,6 +22,12 @@ from app.services.zone_file import ( router = APIRouter(prefix="/drops", tags=["drops"]) +# All supported TLDs +SWITCH_TLDS = ["ch", "li"] +CZDS_TLDS = ["xyz", "org", "online", "info", "dev", "app"] # Approved +CZDS_PENDING = ["com", "net", "club", "biz"] # Pending approval +ALL_TLDS = SWITCH_TLDS + CZDS_TLDS + # ============================================================================ # PUBLIC ENDPOINTS (for stats) @@ -46,7 +54,7 @@ async def api_get_zone_stats( @router.get("") async def api_get_drops( - tld: Optional[str] = Query(None, description="Filter by TLD (ch or li)"), + tld: Optional[str] = Query(None, description="Filter by TLD"), days: int = Query(7, ge=1, le=30, description="Days to look back"), min_length: Optional[int] = Query(None, ge=1, le=63, description="Minimum domain length"), max_length: Optional[int] = Query(None, ge=1, le=63, description="Maximum domain length"), @@ -59,13 +67,20 @@ async def api_get_drops( current_user = Depends(get_current_user) ): """ - Get recently dropped domains from .ch and .li zone files. + Get recently dropped domains from zone files. + + Supports: + - Switch.ch zones: .ch, .li + - ICANN CZDS zones: .xyz, .org, .online, .info, .dev, .app Domains are detected by comparing daily zone file snapshots. Only available for authenticated users. """ - if tld and tld not in ["ch", "li"]: - raise HTTPException(status_code=400, detail="TLD must be 'ch' or 'li'") + if tld and tld not in ALL_TLDS: + raise HTTPException( + status_code=400, + detail=f"Unsupported TLD. Supported: {', '.join(ALL_TLDS)}" + ) try: result = await get_dropped_domains( @@ -102,19 +117,28 @@ async def api_trigger_sync( if not getattr(current_user, 'is_admin', False): raise HTTPException(status_code=403, detail="Admin access required") - if tld not in ["ch", "li"]: - raise HTTPException(status_code=400, detail="TLD must be 'ch' or 'li'") - - # Run sync in background - service = ZoneFileService() + if tld not in ALL_TLDS: + raise HTTPException( + status_code=400, + detail=f"Unsupported TLD. Supported: {', '.join(ALL_TLDS)}" + ) async def run_sync(): - async for session in get_db(): + from app.database import AsyncSessionLocal + + async with AsyncSessionLocal() as session: try: - await service.run_daily_sync(session, tld) + if tld in SWITCH_TLDS: + # Use Switch.ch zone transfer + service = ZoneFileService() + await service.run_daily_sync(session, tld) + else: + # Use ICANN CZDS + from app.services.czds_client import CZDSClient + client = CZDSClient() + await client.sync_zone(session, tld) except Exception as e: print(f"Zone sync failed for .{tld}: {e}") - break background_tasks.add_task(run_sync) @@ -132,17 +156,22 @@ async def api_get_supported_tlds(): """ return { "tlds": [ - { - "tld": "ch", - "name": "Switzerland", - "flag": "šŸ‡ØšŸ‡­", - "registry": "Switch" - }, - { - "tld": "li", - "name": "Liechtenstein", - "flag": "šŸ‡±šŸ‡®", - "registry": "Switch" - } + # Switch.ch zones + {"tld": "ch", "name": "Switzerland", "flag": "šŸ‡ØšŸ‡­", "registry": "Switch", "source": "switch"}, + {"tld": "li", "name": "Liechtenstein", "flag": "šŸ‡±šŸ‡®", "registry": "Switch", "source": "switch"}, + # ICANN CZDS zones (approved) + {"tld": "xyz", "name": "XYZ", "flag": "šŸŒ", "registry": "XYZ.COM LLC", "source": "czds"}, + {"tld": "org", "name": "Organization", "flag": "šŸ›ļø", "registry": "PIR", "source": "czds"}, + {"tld": "online", "name": "Online", "flag": "šŸ’»", "registry": "Radix", "source": "czds"}, + {"tld": "info", "name": "Information", "flag": "ā„¹ļø", "registry": "Afilias", "source": "czds"}, + {"tld": "dev", "name": "Developer", "flag": "šŸ‘Øā€šŸ’»", "registry": "Google", "source": "czds"}, + {"tld": "app", "name": "Application", "flag": "šŸ“±", "registry": "Google", "source": "czds"}, + ], + "pending": [ + # CZDS pending approval + {"tld": "com", "name": "Commercial", "flag": "šŸ¢", "registry": "Verisign", "source": "czds"}, + {"tld": "net", "name": "Network", "flag": "šŸŒ", "registry": "Verisign", "source": "czds"}, + {"tld": "club", "name": "Club", "flag": "šŸŽ‰", "registry": "GoDaddy", "source": "czds"}, + {"tld": "biz", "name": "Business", "flag": "šŸ’¼", "registry": "GoDaddy", "source": "czds"}, ] } diff --git a/backend/app/config.py b/backend/app/config.py index fa2115f..a9fc3ea 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -117,6 +117,13 @@ class Settings(BaseSettings): moz_access_id: str = "" moz_secret_key: str = "" + # ICANN CZDS (Centralized Zone Data Service) + # For downloading gTLD zone files (.com, .net, .org, etc.) + # Register at: https://czds.icann.org/ + czds_username: str = "" + czds_password: str = "" + czds_data_dir: str = "/tmp/pounce_czds" + class Config: env_file = ".env" env_file_encoding = "utf-8" diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 8b6a2e6..3874939 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -662,7 +662,16 @@ def setup_scheduler(): sync_zone_files, CronTrigger(hour=5, minute=0), # Daily at 05:00 UTC id="zone_file_sync", - name="Zone File Sync (daily)", + name="Zone File Sync - Switch.ch (daily)", + replace_existing=True, + ) + + # CZDS zone file sync for gTLDs (daily at 06:00 UTC, after Switch sync) + scheduler.add_job( + sync_czds_zones, + CronTrigger(hour=6, minute=0), # Daily at 06:00 UTC + id="czds_zone_sync", + name="Zone File Sync - ICANN CZDS (daily)", replace_existing=True, ) @@ -842,7 +851,7 @@ async def scrape_auctions(): async def sync_zone_files(): - """Sync zone files for .ch and .li domains from Switch.ch.""" + """Sync zone files from Switch.ch (.ch, .li) and ICANN CZDS (gTLDs).""" logger.info("Starting zone file sync...") try: @@ -851,26 +860,49 @@ async def sync_zone_files(): service = ZoneFileService() async with AsyncSessionLocal() as db: - # Sync .ch zone - try: - ch_result = await service.run_daily_sync(db, "ch") - logger.info(f".ch zone sync: {len(ch_result.get('dropped', []))} dropped, {ch_result.get('new_count', 0)} new") - except Exception as e: - logger.error(f".ch zone sync failed: {e}") - - # Sync .li zone - try: - li_result = await service.run_daily_sync(db, "li") - logger.info(f".li zone sync: {len(li_result.get('dropped', []))} dropped, {li_result.get('new_count', 0)} new") - except Exception as e: - logger.error(f".li zone sync failed: {e}") + # Sync Switch.ch zones (.ch, .li) + for tld in ["ch", "li"]: + try: + result = await service.run_daily_sync(db, tld) + logger.info(f".{tld} zone sync: {len(result.get('dropped', []))} dropped, {result.get('new_count', 0)} new") + except Exception as e: + logger.error(f".{tld} zone sync failed: {e}") - logger.info("Zone file sync completed") + logger.info("Switch.ch zone file sync completed") except Exception as e: logger.exception(f"Zone file sync failed: {e}") +async def sync_czds_zones(): + """Sync zone files from ICANN CZDS (gTLDs like .xyz, .org, .dev, .app).""" + logger.info("Starting CZDS zone file sync...") + + try: + from app.services.czds_client import CZDSClient, APPROVED_TLDS + from app.config import get_settings + + settings = get_settings() + + # Skip if credentials not configured + if not settings.czds_username or not settings.czds_password: + logger.info("CZDS credentials not configured, skipping CZDS sync") + return + + client = CZDSClient() + + async with AsyncSessionLocal() as db: + results = await client.sync_all_zones(db, APPROVED_TLDS) + + success_count = sum(1 for r in results if r["status"] == "success") + total_dropped = sum(r["dropped_count"] for r in results) + + logger.info(f"CZDS sync complete: {success_count}/{len(APPROVED_TLDS)} zones, {total_dropped:,} dropped") + + except Exception as e: + logger.exception(f"CZDS zone file sync failed: {e}") + + async def match_sniper_alerts(): """Match active sniper alerts against current auctions and notify users.""" from app.models.sniper_alert import SniperAlert, SniperAlertMatch diff --git a/backend/app/services/czds_client.py b/backend/app/services/czds_client.py new file mode 100644 index 0000000..462f943 --- /dev/null +++ b/backend/app/services/czds_client.py @@ -0,0 +1,456 @@ +""" +ICANN CZDS (Centralized Zone Data Service) Client +================================================== +Downloads zone files from ICANN CZDS, parses them, and detects dropped domains. + +Authentication: OAuth2 with username/password +Zone Format: Standard DNS zone file format (.txt.gz) + +Usage: + client = CZDSClient(username, password) + await client.sync_all_zones(db) +""" + +import asyncio +import gzip +import hashlib +import logging +import os +import re +import shutil +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +import httpx +from sqlalchemy import select, func, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import get_settings +from app.models.zone_file import ZoneSnapshot, DroppedDomain + +logger = logging.getLogger(__name__) +settings = get_settings() + +# ============================================================================ +# CONSTANTS +# ============================================================================ + +CZDS_AUTH_URL = "https://account-api.icann.org/api/authenticate" +CZDS_ZONES_URL = "https://czds-api.icann.org/czds/downloads/links" +CZDS_DOWNLOAD_BASE = "https://czds-download.icann.org" + +# TLDs we have approved access to +APPROVED_TLDS = ["xyz", "org", "online", "info", "dev", "app"] + +# Regex to extract domain names from zone file NS records +# Format: example.tld. IN NS ns1.example.com. +NS_RECORD_PATTERN = re.compile(r'^([a-z0-9][-a-z0-9]*)\.[a-z]+\.\s+\d*\s*IN\s+NS\s+', re.IGNORECASE) + + +# ============================================================================ +# CZDS CLIENT +# ============================================================================ + +class CZDSClient: + """Client for ICANN CZDS zone file downloads.""" + + def __init__( + self, + username: Optional[str] = None, + password: Optional[str] = None, + data_dir: Optional[Path] = None + ): + self.username = username or os.getenv("CZDS_USERNAME") or settings.czds_username + self.password = password or os.getenv("CZDS_PASSWORD") or settings.czds_password + self.data_dir = data_dir or Path(os.getenv("CZDS_DATA_DIR", "/tmp/pounce_czds")) + self.data_dir.mkdir(parents=True, exist_ok=True) + + self._token: Optional[str] = None + self._token_expires: Optional[datetime] = None + + async def _authenticate(self) -> str: + """Authenticate with ICANN and get access token.""" + if self._token and self._token_expires and datetime.utcnow() < self._token_expires: + return self._token + + if not self.username or not self.password: + raise ValueError("CZDS credentials not configured. Set CZDS_USERNAME and CZDS_PASSWORD.") + + logger.info("Authenticating with ICANN CZDS...") + + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post( + CZDS_AUTH_URL, + json={"username": self.username, "password": self.password}, + headers={"Content-Type": "application/json"} + ) + + if response.status_code != 200: + logger.error(f"CZDS authentication failed: {response.status_code} {response.text}") + raise RuntimeError(f"CZDS authentication failed: {response.status_code}") + + data = response.json() + self._token = data.get("accessToken") + + # Token expires in 24 hours, refresh after 23 hours + self._token_expires = datetime.utcnow() + timedelta(hours=23) + + logger.info("CZDS authentication successful") + return self._token + + async def get_available_zones(self) -> list[str]: + """Get list of zone files available for download.""" + token = await self._authenticate() + + async with httpx.AsyncClient(timeout=60) as client: + response = await client.get( + CZDS_ZONES_URL, + headers={"Authorization": f"Bearer {token}"} + ) + + if response.status_code != 200: + logger.error(f"Failed to get zone list: {response.status_code}") + return [] + + # Response is a list of download URLs + urls = response.json() + + # Extract TLDs from URLs + tlds = [] + for url in urls: + # URL format: https://czds-download.icann.org/czds/downloads/xyz.zone + match = re.search(r'/([a-z0-9-]+)\.zone$', url, re.IGNORECASE) + if match: + tlds.append(match.group(1).lower()) + + logger.info(f"Available zones: {tlds}") + return tlds + + async def download_zone(self, tld: str) -> Optional[Path]: + """Download a zone file for a specific TLD.""" + token = await self._authenticate() + + download_url = f"{CZDS_DOWNLOAD_BASE}/czds/downloads/{tld}.zone" + output_path = self.data_dir / f"{tld}.zone.txt.gz" + + logger.info(f"Downloading zone file for .{tld}...") + + async with httpx.AsyncClient(timeout=600, follow_redirects=True) as client: + try: + async with client.stream( + "GET", + download_url, + headers={"Authorization": f"Bearer {token}"} + ) as response: + if response.status_code != 200: + logger.error(f"Failed to download .{tld}: {response.status_code}") + return None + + # Stream to file + with open(output_path, "wb") as f: + async for chunk in response.aiter_bytes(chunk_size=1024 * 1024): + f.write(chunk) + + file_size = output_path.stat().st_size / (1024 * 1024) + logger.info(f"Downloaded .{tld} zone file: {file_size:.1f} MB") + return output_path + + except Exception as e: + logger.error(f"Error downloading .{tld}: {e}") + return None + + def extract_zone_file(self, gz_path: Path) -> Path: + """Extract gzipped zone file.""" + output_path = gz_path.with_suffix('') # Remove .gz + + logger.info(f"Extracting {gz_path.name}...") + + with gzip.open(gz_path, 'rb') as f_in: + with open(output_path, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + # Remove gz file to save space + gz_path.unlink() + + return output_path + + def parse_zone_file(self, zone_path: Path, tld: str) -> set[str]: + """ + Parse zone file and extract unique domain names. + + Zone files contain various record types. We extract domains from: + - NS records (most reliable indicator of active domain) + - A/AAAA records + + Returns set of domain names (without TLD suffix). + """ + logger.info(f"Parsing zone file for .{tld}...") + + domains = set() + line_count = 0 + + with open(zone_path, 'r', encoding='utf-8', errors='ignore') as f: + for line in f: + line_count += 1 + + # Skip comments and empty lines + if line.startswith(';') or not line.strip(): + continue + + # Look for NS records which indicate delegated domains + # Format: example.tld. 86400 IN NS ns1.registrar.com. + parts = line.split() + if len(parts) >= 4: + # First column is the domain name + name = parts[0].rstrip('.') + + # Must end with our TLD + if name.lower().endswith(f'.{tld}'): + # Extract just the domain name part + domain_name = name[:-(len(tld) + 1)] + + # Skip the TLD itself and subdomains + if domain_name and '.' not in domain_name: + domains.add(domain_name.lower()) + + logger.info(f"Parsed .{tld}: {len(domains):,} unique domains from {line_count:,} lines") + return domains + + def compute_checksum(self, domains: set[str]) -> str: + """Compute SHA256 checksum of sorted domain list.""" + sorted_domains = "\n".join(sorted(domains)) + return hashlib.sha256(sorted_domains.encode()).hexdigest() + + async def get_previous_domains(self, tld: str) -> Optional[set[str]]: + """Load previous day's domain set from cache file.""" + cache_file = self.data_dir / f"{tld}_domains.txt" + + if cache_file.exists(): + try: + content = cache_file.read_text() + return set(line.strip() for line in content.splitlines() if line.strip()) + except Exception as e: + logger.warning(f"Failed to load cache for .{tld}: {e}") + + return None + + async def save_domains(self, tld: str, domains: set[str]): + """Save current domains to cache file.""" + cache_file = self.data_dir / f"{tld}_domains.txt" + cache_file.write_text("\n".join(sorted(domains))) + logger.info(f"Saved {len(domains):,} domains for .{tld}") + + async def process_drops( + self, + db: AsyncSession, + tld: str, + previous: set[str], + current: set[str] + ) -> list[dict]: + """Find and store dropped domains.""" + dropped = previous - current + + if not dropped: + logger.info(f"No dropped domains found for .{tld}") + return [] + + logger.info(f"Found {len(dropped):,} dropped domains for .{tld}") + + today = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + + # Batch insert for performance + dropped_records = [] + batch_size = 1000 + batch = [] + + for name in dropped: + record = DroppedDomain( + domain=f"{name}.{tld}", + tld=tld, + dropped_date=today, + length=len(name), + is_numeric=name.isdigit(), + has_hyphen='-' in name + ) + batch.append(record) + dropped_records.append({ + "domain": f"{name}.{tld}", + "length": len(name), + "is_numeric": name.isdigit(), + "has_hyphen": '-' in name + }) + + if len(batch) >= batch_size: + db.add_all(batch) + await db.flush() + batch = [] + + # Add remaining + if batch: + db.add_all(batch) + + await db.commit() + + return dropped_records + + async def sync_zone(self, db: AsyncSession, tld: str) -> dict: + """ + Sync a single zone file: + 1. Download zone file + 2. Extract and parse + 3. Compare with previous snapshot + 4. Store dropped domains + 5. Save new snapshot + """ + logger.info(f"Starting sync for .{tld}") + + result = { + "tld": tld, + "status": "pending", + "current_count": 0, + "previous_count": 0, + "dropped_count": 0, + "new_count": 0, + "error": None + } + + try: + # Download zone file + gz_path = await self.download_zone(tld) + if not gz_path: + result["status"] = "download_failed" + result["error"] = "Failed to download zone file" + return result + + # Extract + zone_path = self.extract_zone_file(gz_path) + + # Parse + current_domains = self.parse_zone_file(zone_path, tld) + result["current_count"] = len(current_domains) + + # Clean up zone file (can be very large) + zone_path.unlink() + + # Get previous snapshot + previous_domains = await self.get_previous_domains(tld) + + if previous_domains: + result["previous_count"] = len(previous_domains) + + # Find dropped domains + dropped = await self.process_drops(db, tld, previous_domains, current_domains) + result["dropped_count"] = len(dropped) + result["new_count"] = len(current_domains - previous_domains) + + # Save current snapshot + await self.save_domains(tld, current_domains) + + # Save snapshot metadata + checksum = self.compute_checksum(current_domains) + snapshot = ZoneSnapshot( + tld=tld, + snapshot_date=datetime.utcnow(), + domain_count=len(current_domains), + checksum=checksum + ) + db.add(snapshot) + await db.commit() + + result["status"] = "success" + logger.info( + f"Sync complete for .{tld}: " + f"{result['current_count']:,} domains, " + f"{result['dropped_count']:,} dropped, " + f"{result['new_count']:,} new" + ) + + except Exception as e: + logger.exception(f"Error syncing .{tld}: {e}") + result["status"] = "error" + result["error"] = str(e) + + return result + + async def sync_all_zones( + self, + db: AsyncSession, + tlds: Optional[list[str]] = None + ) -> list[dict]: + """ + Sync all approved zone files. + + Args: + db: Database session + tlds: Optional list of TLDs to sync. Defaults to APPROVED_TLDS. + + Returns: + List of sync results for each TLD. + """ + tlds = tlds or APPROVED_TLDS + + logger.info(f"Starting CZDS sync for {len(tlds)} zones: {tlds}") + + results = [] + for tld in tlds: + result = await self.sync_zone(db, tld) + results.append(result) + + # Small delay between zones to be nice to ICANN servers + await asyncio.sleep(2) + + # Summary + success_count = sum(1 for r in results if r["status"] == "success") + total_dropped = sum(r["dropped_count"] for r in results) + + logger.info( + f"CZDS sync complete: " + f"{success_count}/{len(tlds)} zones successful, " + f"{total_dropped:,} total dropped domains" + ) + + return results + + +# ============================================================================ +# STANDALONE SCRIPT +# ============================================================================ + +async def main(): + """Standalone script to run CZDS sync.""" + import sys + from app.database import AsyncSessionLocal, init_db + + # Initialize database + await init_db() + + # Parse arguments + tlds = sys.argv[1:] if len(sys.argv) > 1 else APPROVED_TLDS + + print(f"šŸŒ CZDS Zone File Sync") + print(f"šŸ“‚ TLDs: {', '.join(tlds)}") + print("-" * 50) + + client = CZDSClient() + + async with AsyncSessionLocal() as db: + results = await client.sync_all_zones(db, tlds) + + print("\n" + "=" * 50) + print("šŸ“Š RESULTS") + print("=" * 50) + + for r in results: + status_icon = "āœ…" if r["status"] == "success" else "āŒ" + print(f"{status_icon} .{r['tld']}: {r['current_count']:,} domains, " + f"{r['dropped_count']:,} dropped, {r['new_count']:,} new") + if r["error"]: + print(f" āš ļø Error: {r['error']}") + + total_dropped = sum(r["dropped_count"] for r in results) + print(f"\nšŸŽÆ Total dropped domains: {total_dropped:,}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/scripts/sync_czds.py b/backend/scripts/sync_czds.py new file mode 100644 index 0000000..acef0d5 --- /dev/null +++ b/backend/scripts/sync_czds.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +CZDS Zone File Sync Script +========================== +Manually sync zone files from ICANN CZDS. + +Usage: + # Sync all approved TLDs + python scripts/sync_czds.py + + # Sync specific TLDs + python scripts/sync_czds.py xyz org dev + +Environment: + CZDS_USERNAME: ICANN CZDS username (email) + CZDS_PASSWORD: ICANN CZDS password + CZDS_DATA_DIR: Directory for zone file cache (default: /tmp/pounce_czds) +""" + +import asyncio +import os +import sys +from pathlib import Path + +# Add backend to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from dotenv import load_dotenv +load_dotenv() + + +async def main(): + from app.database import AsyncSessionLocal, init_db + from app.services.czds_client import CZDSClient, APPROVED_TLDS + + # Parse arguments + tlds = sys.argv[1:] if len(sys.argv) > 1 else APPROVED_TLDS + + # Check credentials + username = os.getenv("CZDS_USERNAME") + password = os.getenv("CZDS_PASSWORD") + + if not username or not password: + print("āŒ CZDS credentials not configured!") + print() + print("Set these environment variables:") + print(" export CZDS_USERNAME='your-icann-email'") + print(" export CZDS_PASSWORD='your-password'") + print() + print("Or add to .env file:") + print(" CZDS_USERNAME=your-icann-email") + print(" CZDS_PASSWORD=your-password") + sys.exit(1) + + print("šŸŒ ICANN CZDS Zone File Sync") + print("=" * 50) + print(f"šŸ“§ Username: {username}") + print(f"šŸ“‚ TLDs: {', '.join(tlds)}") + print(f"šŸ’¾ Cache Dir: {os.getenv('CZDS_DATA_DIR', '/tmp/pounce_czds')}") + print("=" * 50) + print() + + # Initialize database + print("šŸ“¦ Initializing database...") + await init_db() + + # Create client and sync + client = CZDSClient() + + print("\nšŸ” Authenticating with ICANN...") + + async with AsyncSessionLocal() as db: + print("\nšŸ“„ Starting zone file downloads...") + print(" (This may take a while for large zones)") + print() + + results = await client.sync_all_zones(db, tlds) + + # Summary + print("\n" + "=" * 50) + print("šŸ“Š SYNC RESULTS") + print("=" * 50) + + total_domains = 0 + total_dropped = 0 + total_new = 0 + + for r in results: + status_icon = "āœ…" if r["status"] == "success" else "āŒ" + print(f"\n{status_icon} .{r['tld']}") + + if r["status"] == "success": + print(f" Domains: {r['current_count']:>12,}") + print(f" Previous: {r['previous_count']:>12,}") + print(f" Dropped: {r['dropped_count']:>12,}") + print(f" New: {r['new_count']:>12,}") + + total_domains += r['current_count'] + total_dropped += r['dropped_count'] + total_new += r['new_count'] + else: + print(f" Error: {r.get('error', 'Unknown error')}") + + print("\n" + "-" * 50) + print(f"šŸ“ˆ TOTALS") + print(f" Total Domains: {total_domains:>12,}") + print(f" Total Dropped: {total_dropped:>12,}") + print(f" Total New: {total_new:>12,}") + print() + + if total_dropped > 0: + print(f"šŸŽÆ {total_dropped:,} dropped domains available in the database!") + print(" View them at: https://pounce.ch/terminal/hunt -> Drops tab") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/frontend/src/components/hunt/DropsTab.tsx b/frontend/src/components/hunt/DropsTab.tsx index ca36353..6256a16 100644 --- a/frontend/src/components/hunt/DropsTab.tsx +++ b/frontend/src/components/hunt/DropsTab.tsx @@ -65,8 +65,11 @@ export function DropsTab({ showToast }: DropsTabProps) { const [refreshing, setRefreshing] = useState(false) const [total, setTotal] = useState(0) + // All supported TLDs + type SupportedTld = 'ch' | 'li' | 'xyz' | 'org' | 'online' | 'info' | 'dev' | 'app' + // Filter State - const [selectedTld, setSelectedTld] = useState<'ch' | 'li' | null>(null) + const [selectedTld, setSelectedTld] = useState(null) const [searchQuery, setSearchQuery] = useState('') const [searchFocused, setSearchFocused] = useState(false) const [days, setDays] = useState(7) @@ -217,32 +220,6 @@ export function DropsTab({ showToast }: DropsTabProps) { {/* Stats Cards */} {stats && (
-
-
- šŸ‡ØšŸ‡­ - .ch Zone -
-
- {stats.ch.domain_count.toLocaleString()} -
-
- {stats.ch.last_sync ? `Last: ${new Date(stats.ch.last_sync).toLocaleDateString()}` : 'Not synced'} -
-
- -
-
- šŸ‡±šŸ‡® - .li Zone -
-
- {stats.li.domain_count.toLocaleString()} -
-
- {stats.li.last_sync ? `Last: ${new Date(stats.li.last_sync).toLocaleDateString()}` : 'Not synced'} -
-
-
@@ -258,12 +235,41 @@ export function DropsTab({ showToast }: DropsTabProps) {
- - Data Source + šŸ‡ØšŸ‡­šŸ‡±šŸ‡® + Switch.ch +
+
+ {((stats.ch?.domain_count || 0) + (stats.li?.domain_count || 0)).toLocaleString()}
-
Switch.ch
- Official zone files + .ch + .li zones +
+
+ +
+
+ + ICANN CZDS +
+
6 TLDs
+
+ .xyz .org .dev .app ... +
+
+ +
+
+ + Last Sync +
+
+ {stats.ch?.last_sync + ? new Date(stats.ch.last_sync).toLocaleDateString() + : 'Pending' + } +
+
+ Daily @ 05:00 UTC
@@ -299,34 +305,54 @@ export function DropsTab({ showToast }: DropsTabProps) {
{/* TLD Quick Filter */} -
+
- - + {/* Switch.ch TLDs */} + {[ + { tld: 'ch', flag: 'šŸ‡ØšŸ‡­' }, + { tld: 'li', flag: 'šŸ‡±šŸ‡®' }, + ].map(({ tld, flag }) => ( + + ))} + {/* Separator */} +
+ {/* CZDS TLDs */} + {[ + { tld: 'xyz', flag: 'šŸŒ' }, + { tld: 'org', flag: 'šŸ›ļø' }, + { tld: 'online', flag: 'šŸ’»' }, + { tld: 'info', flag: 'ā„¹ļø' }, + { tld: 'dev', flag: 'šŸ‘Øā€šŸ’»' }, + { tld: 'app', flag: 'šŸ“±' }, + ].map(({ tld, flag }) => ( + + ))}
{/* Filter Toggle */} @@ -622,11 +648,19 @@ export function DropsTab({ showToast }: DropsTabProps) {

Zone File Analysis

-

- Domains are detected by comparing daily zone file snapshots from Switch.ch. - Data is updated automatically every 24 hours. Only .ch and .li domains are supported - as these zones are publicly available from the Swiss registry. +

+ Domains are detected by comparing daily zone file snapshots. Data sources:

+
+
+ šŸ‡ØšŸ‡­ + .ch/.li via Switch.ch (AXFR) +
+
+ šŸŒ + gTLDs via ICANN CZDS +
+
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index b22572c..9a4e371 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1844,7 +1844,7 @@ class AdminApiClient extends ApiClient { } async getDrops(params?: { - tld?: 'ch' | 'li' + tld?: 'ch' | 'li' | 'xyz' | 'org' | 'online' | 'info' | 'dev' | 'app' days?: number min_length?: number max_length?: number