From 06976674d3bcad6bc3d2b8a35247bca5ebf6da61 Mon Sep 17 00:00:00 2001 From: Yves Gugger Date: Fri, 19 Dec 2025 12:20:48 +0100 Subject: [PATCH] Deploy: 2025-12-19 12:20 --- backend/app/api/drops.py | 74 ++++++++++++-- backend/app/models/zone_file.py | 3 +- backend/app/services/drop_status_checker.py | 101 ++++++++++++++++++-- backend/app/services/zone_file.py | 1 + frontend/src/components/hunt/DropsTab.tsx | 57 ++++++++++- frontend/src/lib/api.ts | 11 ++- 6 files changed, 226 insertions(+), 21 deletions(-) diff --git a/backend/app/api/drops.py b/backend/app/api/drops.py index bb6fb2a..bed91ce 100644 --- a/backend/app/api/drops.py +++ b/backend/app/api/drops.py @@ -190,16 +190,9 @@ async def api_check_drop_status( ): """ Check the real-time availability status of a dropped domain. - - Returns: - - available: Domain can be registered NOW - - dropping_soon: Domain is in deletion phase (track it!) - - taken: Domain was re-registered - - unknown: Could not determine status """ from app.services.drop_status_checker import check_drop_status - # Get the drop from DB result = await db.execute( select(DroppedDomain).where(DroppedDomain.id == drop_id) ) @@ -211,7 +204,6 @@ async def api_check_drop_status( full_domain = f"{drop.domain}.{drop.tld}" try: - # Check with dedicated drop status checker status_result = await check_drop_status(full_domain) # Update the drop in DB @@ -221,7 +213,8 @@ async def api_check_drop_status( .values( availability_status=status_result.status, rdap_status=str(status_result.rdap_status) if status_result.rdap_status else None, - last_status_check=datetime.utcnow() + last_status_check=datetime.utcnow(), + deletion_date=status_result.deletion_date, ) ) await db.commit() @@ -230,7 +223,7 @@ async def api_check_drop_status( "id": drop_id, "domain": full_domain, "status": status_result.status, - "rdap_status": status_result.rdap_status, + "deletion_date": status_result.deletion_date, "can_register_now": status_result.can_register_now, "should_track": status_result.should_monitor, "message": status_result.message, @@ -241,6 +234,67 @@ async def api_check_drop_status( raise HTTPException(status_code=500, detail=str(e)) +@router.post("/batch-check") +async def api_batch_check_status( + background_tasks: BackgroundTasks, + limit: int = Query(50, ge=1, le=100, description="Number of drops to check"), + db: AsyncSession = Depends(get_db), + current_user = Depends(get_current_user) +): + """ + Batch check status for drops that haven't been checked yet. + Returns immediately, updates happen in background. + """ + from app.services.drop_status_checker import batch_check_drops + + # Get drops that haven't been checked or were checked > 1 hour ago + one_hour_ago = datetime.utcnow().replace(hour=datetime.utcnow().hour - 1) + + result = await db.execute( + select(DroppedDomain) + .where( + (DroppedDomain.availability_status == 'unknown') | + (DroppedDomain.last_status_check == None) | + (DroppedDomain.last_status_check < one_hour_ago) + ) + .order_by(DroppedDomain.length) # Prioritize short domains + .limit(limit) + ) + drops = result.scalars().all() + + if not drops: + return {"message": "All drops already checked", "checked": 0} + + # Prepare domain list + domains_to_check = [(d.id, f"{d.domain}.{d.tld}") for d in drops] + + async def run_batch_check(): + from app.database import AsyncSessionLocal + + results = await batch_check_drops(domains_to_check) + + async with AsyncSessionLocal() as session: + for drop_id, status in results: + await session.execute( + update(DroppedDomain) + .where(DroppedDomain.id == drop_id) + .values( + availability_status=status.status, + rdap_status=str(status.rdap_status) if status.rdap_status else None, + last_status_check=datetime.utcnow(), + deletion_date=status.deletion_date, + ) + ) + await session.commit() + + background_tasks.add_task(run_batch_check) + + return { + "message": f"Checking {len(drops)} drops in background", + "checking": len(drops), + } + + @router.post("/track/{drop_id}") async def api_track_drop( drop_id: int, diff --git a/backend/app/models/zone_file.py b/backend/app/models/zone_file.py index c3b50e9..55a6fa1 100644 --- a/backend/app/models/zone_file.py +++ b/backend/app/models/zone_file.py @@ -38,10 +38,11 @@ class DroppedDomain(Base): created_at = Column(DateTime, default=datetime.utcnow) # Real-time availability status (checked via RDAP) - # Possible values: 'available', 'pending_delete', 'redemption', 'taken', 'unknown' + # Possible values: 'available', 'dropping_soon', 'taken', 'unknown' availability_status = Column(String(20), default='unknown', index=True) rdap_status = Column(String(255), nullable=True) # Raw RDAP status string last_status_check = Column(DateTime, nullable=True) + deletion_date = Column(String(10), nullable=True) # YYYY-MM-DD when domain will be purged __table_args__ = ( Index('ix_dropped_domains_tld_date', 'tld', 'dropped_date'), diff --git a/backend/app/services/drop_status_checker.py b/backend/app/services/drop_status_checker.py index 9494d5c..c505659 100644 --- a/backend/app/services/drop_status_checker.py +++ b/backend/app/services/drop_status_checker.py @@ -3,11 +3,13 @@ Drop Status Checker ==================== Dedicated RDAP checker for dropped domains. Correctly identifies pending_delete, redemption, and available status. +Also extracts deletion date for countdown display. """ import httpx import logging from dataclasses import dataclass +from datetime import datetime from typing import Optional logger = logging.getLogger(__name__) @@ -31,6 +33,22 @@ RDAP_ENDPOINTS = { 'app': 'https://rdap.nic.google/domain/', } +# Typical deletion periods after "pending delete" status (in days) +TLD_DELETE_PERIODS = { + 'ch': 40, # .ch has ~40 day redemption + 'li': 40, + 'com': 35, # 30 redemption + 5 pending delete + 'net': 35, + 'org': 35, + 'info': 35, + 'biz': 35, + 'online': 35, + 'xyz': 35, + 'club': 35, + 'dev': 35, + 'app': 35, +} + @dataclass class DropStatus: @@ -41,6 +59,24 @@ class DropStatus: can_register_now: bool should_monitor: bool message: str + deletion_date: Optional[str] = None # ISO format date when domain will be deleted + estimated_available: Optional[str] = None # Estimated date when registrable + + +def _parse_date(date_str: str) -> Optional[datetime]: + """Parse ISO date string to datetime.""" + if not date_str: + return None + try: + # Handle various formats + for fmt in ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d']: + try: + return datetime.strptime(date_str[:19].replace('Z', ''), fmt.replace('Z', '')) + except ValueError: + continue + return None + except Exception: + return None async def check_drop_status(domain: str) -> DropStatus: @@ -58,7 +94,6 @@ async def check_drop_status(domain: str) -> DropStatus: endpoint = RDAP_ENDPOINTS.get(tld) if not endpoint: - # Try generic lookup logger.warning(f"No RDAP endpoint for .{tld}, returning unknown") return DropStatus( domain=domain, @@ -83,7 +118,7 @@ async def check_drop_status(domain: str) -> DropStatus: rdap_status=[], can_register_now=True, should_monitor=False, - message="Domain is available for registration!" + message="Available now!" ) # 200 = Domain exists in registry @@ -92,6 +127,19 @@ async def check_drop_status(domain: str) -> DropStatus: rdap_status = data.get('status', []) status_lower = ' '.join(str(s).lower() for s in rdap_status) + # Extract deletion date from events + deletion_date = None + expiration_date = None + events = data.get('events', []) + for event in events: + action = event.get('eventAction', '').lower() + date_str = event.get('eventDate', '') + + if 'deletion' in action: + deletion_date = date_str[:10] if date_str else None + elif 'expiration' in action: + expiration_date = date_str[:10] if date_str else None + # Check for pending delete / redemption status is_pending = any(x in status_lower for x in [ 'pending delete', 'pendingdelete', @@ -101,13 +149,25 @@ async def check_drop_status(domain: str) -> DropStatus: ]) if is_pending: + # Calculate estimated availability + estimated = None + if deletion_date: + try: + del_dt = datetime.strptime(deletion_date, '%Y-%m-%d') + # For most TLDs, domain is available shortly after deletion date + estimated = del_dt.strftime('%Y-%m-%d') + except Exception: + pass + return DropStatus( domain=domain, status='dropping_soon', rdap_status=rdap_status, can_register_now=False, should_monitor=True, - message="Domain is being deleted. Track it to get notified when available!" + message="Dropping soon - track to get notified!", + deletion_date=deletion_date, + estimated_available=estimated, ) # Domain is actively registered @@ -117,7 +177,7 @@ async def check_drop_status(domain: str) -> DropStatus: rdap_status=rdap_status, can_register_now=False, should_monitor=False, - message="Domain was re-registered" + message="Re-registered" ) # Other status code @@ -128,18 +188,17 @@ async def check_drop_status(domain: str) -> DropStatus: rdap_status=[], can_register_now=False, should_monitor=False, - message=f"RDAP returned HTTP {resp.status_code}" + message=f"RDAP {resp.status_code}" ) except httpx.TimeoutException: - logger.warning(f"RDAP timeout for {domain}") return DropStatus( domain=domain, status='unknown', rdap_status=[], can_register_now=False, should_monitor=False, - message="RDAP timeout" + message="Timeout" ) except Exception as e: logger.warning(f"RDAP error for {domain}: {e}") @@ -151,3 +210,31 @@ async def check_drop_status(domain: str) -> DropStatus: should_monitor=False, message=str(e) ) + + +async def batch_check_drops(domains: list[tuple[int, str]]) -> list[tuple[int, DropStatus]]: + """ + Check status for multiple domains in parallel. + + Args: + domains: List of (id, domain_name) tuples + + Returns: + List of (id, DropStatus) tuples + """ + import asyncio + + async def check_one(item: tuple[int, str]) -> tuple[int, DropStatus]: + drop_id, domain = item + status = await check_drop_status(domain) + return (drop_id, status) + + # Limit concurrency to avoid overwhelming RDAP servers + semaphore = asyncio.Semaphore(10) + + async def limited_check(item): + async with semaphore: + return await check_one(item) + + results = await asyncio.gather(*[limited_check(d) for d in domains]) + return results diff --git a/backend/app/services/zone_file.py b/backend/app/services/zone_file.py index 0aecad5..1e0846a 100644 --- a/backend/app/services/zone_file.py +++ b/backend/app/services/zone_file.py @@ -374,6 +374,7 @@ async def get_dropped_domains( "has_hyphen": item.has_hyphen, "availability_status": getattr(item, 'availability_status', 'unknown') or 'unknown', "last_status_check": item.last_status_check.isoformat() if getattr(item, 'last_status_check', None) else None, + "deletion_date": getattr(item, 'deletion_date', None), } for item in items ] diff --git a/frontend/src/components/hunt/DropsTab.tsx b/frontend/src/components/hunt/DropsTab.tsx index 061684a..5a2acf9 100644 --- a/frontend/src/components/hunt/DropsTab.tsx +++ b/frontend/src/components/hunt/DropsTab.tsx @@ -42,6 +42,7 @@ interface DroppedDomain { has_hyphen: boolean availability_status: AvailabilityStatus last_status_check: string | null + deletion_date: string | null } interface ZoneStats { @@ -155,6 +156,29 @@ export function DropsTab({ showToast }: DropsTabProps) { loadDrops(1) }, [loadDrops]) + // Auto batch-check status when drops are loaded + const [batchChecking, setBatchChecking] = useState(false) + + useEffect(() => { + // Check if we have unknown status drops + const unknownCount = items.filter(i => i.availability_status === 'unknown').length + + if (unknownCount > 0 && !batchChecking && items.length > 0) { + setBatchChecking(true) + + // Trigger batch check in background + api.batchCheckDrops(50).then(() => { + // Reload drops after a short delay to get updated status + setTimeout(() => { + loadDrops(page, false) + setBatchChecking(false) + }, 3000) + }).catch(() => { + setBatchChecking(false) + }) + } + }, [items, batchChecking, loadDrops, page]) + const handlePageChange = useCallback((newPage: number) => { setPage(newPage) loadDrops(newPage) @@ -163,6 +187,8 @@ export function DropsTab({ showToast }: DropsTabProps) { const handleRefresh = useCallback(async () => { await loadDrops(page, true) await loadStats() + // Also trigger batch check + api.batchCheckDrops(50).catch(() => {}) }, [loadDrops, loadStats, page]) // Check real-time status of a drop @@ -252,6 +278,24 @@ export function DropsTab({ showToast }: DropsTabProps) { showOnlyAvailable, ].filter(Boolean).length + // Format countdown to deletion date + const formatCountdown = (deletionDate: string | null) => { + if (!deletionDate) return null + + const now = new Date() + const target = new Date(deletionDate + 'T23:59:59') + const diffMs = target.getTime() - now.getTime() + + if (diffMs <= 0) return 'Now' + + const days = Math.floor(diffMs / (1000 * 60 * 60 * 24)) + const hours = Math.floor((diffMs % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60)) + + if (days > 0) return `${days}d ${hours}h` + if (hours > 0) return `${hours}h` + return '<1h' + } + const formatTime = (iso: string) => { const d = new Date(iso) const now = new Date() @@ -479,6 +523,12 @@ export function DropsTab({ showToast }: DropsTabProps) { {availableCount} available now )} + {batchChecking && ( +
+ + Checking status... +
+ )} {totalPages > 1 && !showOnlyAvailable && ( @@ -534,12 +584,15 @@ export function DropsTab({ showToast }: DropsTabProps) { const isTrackingThis = trackingDrop === item.id const status = item.availability_status || 'unknown' + // Countdown for dropping_soon domains + const countdown = formatCountdown(item.deletion_date) + // Simplified status display config const statusConfig = { available: { label: 'Available', color: 'text-accent', bg: 'bg-accent/10', border: 'border-accent/30', icon: CheckCircle2 }, - dropping_soon: { label: 'Dropping Soon', color: 'text-amber-400', bg: 'bg-amber-400/10', border: 'border-amber-400/30', icon: Clock }, + dropping_soon: { label: countdown || 'Soon', color: 'text-amber-400', bg: 'bg-amber-400/10', border: 'border-amber-400/30', icon: Clock }, taken: { label: 'Taken', color: 'text-rose-400', bg: 'bg-rose-400/10', border: 'border-rose-400/30', icon: Ban }, - unknown: { label: 'Check', color: 'text-white/50', bg: 'bg-white/5', border: 'border-white/20', icon: Search }, + unknown: { label: batchChecking ? '...' : 'Check', color: 'text-white/50', bg: 'bg-white/5', border: 'border-white/20', icon: batchChecking ? Loader2 : Search }, }[status] const StatusIcon = statusConfig.icon diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 1502581..7cd2594 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2052,6 +2052,7 @@ class AdminApiClient extends ApiClient { has_hyphen: boolean availability_status: 'available' | 'dropping_soon' | 'taken' | 'unknown' last_status_check: string | null + deletion_date: string | null }> }>(`/drops?${query}`) } @@ -2072,7 +2073,7 @@ class AdminApiClient extends ApiClient { id: number domain: string status: 'available' | 'dropping_soon' | 'taken' | 'unknown' - rdap_status: string[] + deletion_date: string | null can_register_now: boolean should_track: boolean message: string @@ -2086,6 +2087,14 @@ class AdminApiClient extends ApiClient { message: string }>(`/drops/track/${dropId}`, { method: 'POST' }) } + + async batchCheckDrops(limit: number = 50) { + return this.request<{ + message: string + checking?: number + checked?: number + }>(`/drops/batch-check?limit=${limit}`, { method: 'POST' }) + } } // Yield Types