Deploy: 2025-12-19 13:35
Some checks are pending
CI / Frontend Lint & Type Check (push) Waiting to run
CI / Frontend Build (push) Blocked by required conditions
CI / Backend Lint (push) Waiting to run
CI / Backend Tests (push) Blocked by required conditions
CI / Docker Build (push) Blocked by required conditions
CI / Security Scan (push) Waiting to run
Deploy / Build & Push Images (push) Waiting to run
Deploy / Deploy to Server (push) Blocked by required conditions
Deploy / Notify (push) Blocked by required conditions
Some checks are pending
CI / Frontend Lint & Type Check (push) Waiting to run
CI / Frontend Build (push) Blocked by required conditions
CI / Backend Lint (push) Waiting to run
CI / Backend Tests (push) Blocked by required conditions
CI / Docker Build (push) Blocked by required conditions
CI / Security Scan (push) Waiting to run
Deploy / Build & Push Images (push) Waiting to run
Deploy / Deploy to Server (push) Blocked by required conditions
Deploy / Notify (push) Blocked by required conditions
This commit is contained in:
@ -316,3 +316,5 @@ Empfehlungen:
|
|||||||
- Accessibility sweep: Kontrast, Focus states, ARIA, reduced motion.
|
- Accessibility sweep: Kontrast, Focus states, ARIA, reduced motion.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -190,9 +190,16 @@ async def api_check_drop_status(
|
|||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Check the real-time availability status of a dropped domain.
|
Check the real-time availability status of a dropped domain.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
- available: Domain can be registered NOW
|
||||||
|
- dropping_soon: Domain is in deletion phase (track it!)
|
||||||
|
- taken: Domain was re-registered
|
||||||
|
- unknown: Could not determine status
|
||||||
"""
|
"""
|
||||||
from app.services.drop_status_checker import check_drop_status
|
from app.services.drop_status_checker import check_drop_status
|
||||||
|
|
||||||
|
# Get the drop from DB
|
||||||
result = await db.execute(
|
result = await db.execute(
|
||||||
select(DroppedDomain).where(DroppedDomain.id == drop_id)
|
select(DroppedDomain).where(DroppedDomain.id == drop_id)
|
||||||
)
|
)
|
||||||
@ -204,6 +211,7 @@ async def api_check_drop_status(
|
|||||||
full_domain = f"{drop.domain}.{drop.tld}"
|
full_domain = f"{drop.domain}.{drop.tld}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
# Check with dedicated drop status checker
|
||||||
status_result = await check_drop_status(full_domain)
|
status_result = await check_drop_status(full_domain)
|
||||||
|
|
||||||
# Update the drop in DB
|
# Update the drop in DB
|
||||||
@ -213,8 +221,7 @@ async def api_check_drop_status(
|
|||||||
.values(
|
.values(
|
||||||
availability_status=status_result.status,
|
availability_status=status_result.status,
|
||||||
rdap_status=str(status_result.rdap_status) if status_result.rdap_status else None,
|
rdap_status=str(status_result.rdap_status) if status_result.rdap_status else None,
|
||||||
last_status_check=datetime.utcnow(),
|
last_status_check=datetime.utcnow()
|
||||||
deletion_date=status_result.deletion_date,
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
@ -223,10 +230,11 @@ async def api_check_drop_status(
|
|||||||
"id": drop_id,
|
"id": drop_id,
|
||||||
"domain": full_domain,
|
"domain": full_domain,
|
||||||
"status": status_result.status,
|
"status": status_result.status,
|
||||||
"deletion_date": status_result.deletion_date,
|
"rdap_status": status_result.rdap_status,
|
||||||
"can_register_now": status_result.can_register_now,
|
"can_register_now": status_result.can_register_now,
|
||||||
"should_track": status_result.should_monitor,
|
"should_track": status_result.should_monitor,
|
||||||
"message": status_result.message,
|
"message": status_result.message,
|
||||||
|
"deletion_date": status_result.deletion_date.isoformat() if status_result.deletion_date else None,
|
||||||
}
|
}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@ -234,67 +242,6 @@ async def api_check_drop_status(
|
|||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
@router.post("/batch-check")
|
|
||||||
async def api_batch_check_status(
|
|
||||||
background_tasks: BackgroundTasks,
|
|
||||||
limit: int = Query(50, ge=1, le=100, description="Number of drops to check"),
|
|
||||||
db: AsyncSession = Depends(get_db),
|
|
||||||
current_user = Depends(get_current_user)
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Batch check status for drops that haven't been checked yet.
|
|
||||||
Returns immediately, updates happen in background.
|
|
||||||
"""
|
|
||||||
from app.services.drop_status_checker import batch_check_drops
|
|
||||||
|
|
||||||
# Get drops that haven't been checked or were checked > 1 hour ago
|
|
||||||
one_hour_ago = datetime.utcnow().replace(hour=datetime.utcnow().hour - 1)
|
|
||||||
|
|
||||||
result = await db.execute(
|
|
||||||
select(DroppedDomain)
|
|
||||||
.where(
|
|
||||||
(DroppedDomain.availability_status == 'unknown') |
|
|
||||||
(DroppedDomain.last_status_check == None) |
|
|
||||||
(DroppedDomain.last_status_check < one_hour_ago)
|
|
||||||
)
|
|
||||||
.order_by(DroppedDomain.length) # Prioritize short domains
|
|
||||||
.limit(limit)
|
|
||||||
)
|
|
||||||
drops = result.scalars().all()
|
|
||||||
|
|
||||||
if not drops:
|
|
||||||
return {"message": "All drops already checked", "checked": 0}
|
|
||||||
|
|
||||||
# Prepare domain list
|
|
||||||
domains_to_check = [(d.id, f"{d.domain}.{d.tld}") for d in drops]
|
|
||||||
|
|
||||||
async def run_batch_check():
|
|
||||||
from app.database import AsyncSessionLocal
|
|
||||||
|
|
||||||
results = await batch_check_drops(domains_to_check)
|
|
||||||
|
|
||||||
async with AsyncSessionLocal() as session:
|
|
||||||
for drop_id, status in results:
|
|
||||||
await session.execute(
|
|
||||||
update(DroppedDomain)
|
|
||||||
.where(DroppedDomain.id == drop_id)
|
|
||||||
.values(
|
|
||||||
availability_status=status.status,
|
|
||||||
rdap_status=str(status.rdap_status) if status.rdap_status else None,
|
|
||||||
last_status_check=datetime.utcnow(),
|
|
||||||
deletion_date=status.deletion_date,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await session.commit()
|
|
||||||
|
|
||||||
background_tasks.add_task(run_batch_check)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"message": f"Checking {len(drops)} drops in background",
|
|
||||||
"checking": len(drops),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/track/{drop_id}")
|
@router.post("/track/{drop_id}")
|
||||||
async def api_track_drop(
|
async def api_track_drop(
|
||||||
drop_id: int,
|
drop_id: int,
|
||||||
|
|||||||
@ -42,7 +42,7 @@ class DroppedDomain(Base):
|
|||||||
availability_status = Column(String(20), default='unknown', index=True)
|
availability_status = Column(String(20), default='unknown', index=True)
|
||||||
rdap_status = Column(String(255), nullable=True) # Raw RDAP status string
|
rdap_status = Column(String(255), nullable=True) # Raw RDAP status string
|
||||||
last_status_check = Column(DateTime, nullable=True)
|
last_status_check = Column(DateTime, nullable=True)
|
||||||
deletion_date = Column(String(10), nullable=True) # YYYY-MM-DD when domain will be purged
|
deletion_date = Column(DateTime, nullable=True) # When domain will be fully deleted
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index('ix_dropped_domains_tld_date', 'tld', 'dropped_date'),
|
Index('ix_dropped_domains_tld_date', 'tld', 'dropped_date'),
|
||||||
|
|||||||
@ -735,15 +735,6 @@ def setup_scheduler():
|
|||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Drops RDAP status update (hourly - check real status with rate limiting)
|
|
||||||
scheduler.add_job(
|
|
||||||
update_drops_status,
|
|
||||||
CronTrigger(minute=20), # Every hour at :20
|
|
||||||
id="drops_status_update",
|
|
||||||
name="Drops Status Update (hourly)",
|
|
||||||
replace_existing=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Scheduler configured:"
|
f"Scheduler configured:"
|
||||||
f"\n - Scout domain check at {settings.check_hour:02d}:{settings.check_minute:02d} (daily)"
|
f"\n - Scout domain check at {settings.check_hour:02d}:{settings.check_minute:02d} (daily)"
|
||||||
@ -1042,102 +1033,6 @@ async def verify_drops():
|
|||||||
logger.exception(f"Drops verification failed: {e}")
|
logger.exception(f"Drops verification failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
async def update_drops_status():
|
|
||||||
"""
|
|
||||||
Update RDAP status for dropped domains.
|
|
||||||
|
|
||||||
This job runs every hour to check the real status of drops
|
|
||||||
(available, dropping_soon, taken) and store it in the database.
|
|
||||||
Uses rate limiting (0.5s delay) to avoid 429 errors from RDAP servers.
|
|
||||||
|
|
||||||
With 0.5s delay, 50 domains takes ~25 seconds.
|
|
||||||
"""
|
|
||||||
logger.info("Starting drops status update...")
|
|
||||||
|
|
||||||
try:
|
|
||||||
from app.services.drop_status_checker import batch_check_drops
|
|
||||||
from app.models.zone_file import DroppedDomain
|
|
||||||
from sqlalchemy import select, update
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
|
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
# Get drops that haven't been status-checked in the last hour
|
|
||||||
# Or have never been checked
|
|
||||||
one_hour_ago = datetime.utcnow() - timedelta(hours=1)
|
|
||||||
|
|
||||||
# Prioritize .ch and .li (our main focus), then short domains
|
|
||||||
query = (
|
|
||||||
select(DroppedDomain)
|
|
||||||
.where(
|
|
||||||
(DroppedDomain.availability_status == 'unknown') |
|
|
||||||
(DroppedDomain.last_status_check == None) |
|
|
||||||
(DroppedDomain.last_status_check < one_hour_ago)
|
|
||||||
)
|
|
||||||
.order_by(
|
|
||||||
# Prioritize .ch and .li
|
|
||||||
DroppedDomain.tld.desc(),
|
|
||||||
DroppedDomain.length.asc()
|
|
||||||
)
|
|
||||||
.limit(50) # Only 50 per run to avoid rate limiting
|
|
||||||
)
|
|
||||||
|
|
||||||
result = await db.execute(query)
|
|
||||||
drops = result.scalars().all()
|
|
||||||
|
|
||||||
if not drops:
|
|
||||||
logger.info("All drops have been status-checked recently")
|
|
||||||
return
|
|
||||||
|
|
||||||
logger.info(f"Checking status for {len(drops)} drops (with rate limiting)...")
|
|
||||||
|
|
||||||
# Prepare domain list
|
|
||||||
domains_to_check = [(d.id, f"{d.domain}.{d.tld}") for d in drops]
|
|
||||||
|
|
||||||
# Batch check with 0.5s delay between requests
|
|
||||||
results = await batch_check_drops(domains_to_check, delay=0.5)
|
|
||||||
|
|
||||||
# Update database
|
|
||||||
available_count = 0
|
|
||||||
dropping_soon_count = 0
|
|
||||||
taken_count = 0
|
|
||||||
unknown_count = 0
|
|
||||||
|
|
||||||
for drop_id, status in results:
|
|
||||||
await db.execute(
|
|
||||||
update(DroppedDomain)
|
|
||||||
.where(DroppedDomain.id == drop_id)
|
|
||||||
.values(
|
|
||||||
availability_status=status.status,
|
|
||||||
rdap_status=str(status.rdap_status) if status.rdap_status else None,
|
|
||||||
last_status_check=datetime.utcnow(),
|
|
||||||
deletion_date=status.deletion_date,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if status.status == 'available':
|
|
||||||
available_count += 1
|
|
||||||
elif status.status == 'dropping_soon':
|
|
||||||
dropping_soon_count += 1
|
|
||||||
elif status.status == 'taken':
|
|
||||||
taken_count += 1
|
|
||||||
else:
|
|
||||||
unknown_count += 1
|
|
||||||
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
f"Drops status update complete: "
|
|
||||||
f"{len(results)} checked, "
|
|
||||||
f"{available_count} available, "
|
|
||||||
f"{dropping_soon_count} dropping soon, "
|
|
||||||
f"{taken_count} taken, "
|
|
||||||
f"{unknown_count} unknown"
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(f"Drops status update failed: {e}")
|
|
||||||
|
|
||||||
|
|
||||||
async def sync_zone_files():
|
async def sync_zone_files():
|
||||||
"""Sync zone files from Switch.ch (.ch, .li) and ICANN CZDS (gTLDs)."""
|
"""Sync zone files from Switch.ch (.ch, .li) and ICANN CZDS (gTLDs)."""
|
||||||
logger.info("Starting zone file sync...")
|
logger.info("Starting zone file sync...")
|
||||||
|
|||||||
@ -3,9 +3,10 @@ Drop Status Checker
|
|||||||
====================
|
====================
|
||||||
Dedicated RDAP checker for dropped domains.
|
Dedicated RDAP checker for dropped domains.
|
||||||
Correctly identifies pending_delete, redemption, and available status.
|
Correctly identifies pending_delete, redemption, and available status.
|
||||||
Also extracts deletion date for countdown display.
|
Extracts deletion date for countdown display.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import httpx
|
import httpx
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@ -33,22 +34,6 @@ RDAP_ENDPOINTS = {
|
|||||||
'app': 'https://rdap.nic.google/domain/',
|
'app': 'https://rdap.nic.google/domain/',
|
||||||
}
|
}
|
||||||
|
|
||||||
# Typical deletion periods after "pending delete" status (in days)
|
|
||||||
TLD_DELETE_PERIODS = {
|
|
||||||
'ch': 40, # .ch has ~40 day redemption
|
|
||||||
'li': 40,
|
|
||||||
'com': 35, # 30 redemption + 5 pending delete
|
|
||||||
'net': 35,
|
|
||||||
'org': 35,
|
|
||||||
'info': 35,
|
|
||||||
'biz': 35,
|
|
||||||
'online': 35,
|
|
||||||
'xyz': 35,
|
|
||||||
'club': 35,
|
|
||||||
'dev': 35,
|
|
||||||
'app': 35,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class DropStatus:
|
class DropStatus:
|
||||||
@ -59,24 +44,7 @@ class DropStatus:
|
|||||||
can_register_now: bool
|
can_register_now: bool
|
||||||
should_monitor: bool
|
should_monitor: bool
|
||||||
message: str
|
message: str
|
||||||
deletion_date: Optional[str] = None # ISO format date when domain will be deleted
|
deletion_date: Optional[datetime] = None # When domain will be fully deleted
|
||||||
estimated_available: Optional[str] = None # Estimated date when registrable
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_date(date_str: str) -> Optional[datetime]:
|
|
||||||
"""Parse ISO date string to datetime."""
|
|
||||||
if not date_str:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
# Handle various formats
|
|
||||||
for fmt in ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d']:
|
|
||||||
try:
|
|
||||||
return datetime.strptime(date_str[:19].replace('Z', ''), fmt.replace('Z', ''))
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
return None
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
async def check_drop_status(domain: str) -> DropStatus:
|
async def check_drop_status(domain: str) -> DropStatus:
|
||||||
@ -94,6 +62,7 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
|
|
||||||
endpoint = RDAP_ENDPOINTS.get(tld)
|
endpoint = RDAP_ENDPOINTS.get(tld)
|
||||||
if not endpoint:
|
if not endpoint:
|
||||||
|
# Try generic lookup
|
||||||
logger.warning(f"No RDAP endpoint for .{tld}, returning unknown")
|
logger.warning(f"No RDAP endpoint for .{tld}, returning unknown")
|
||||||
return DropStatus(
|
return DropStatus(
|
||||||
domain=domain,
|
domain=domain,
|
||||||
@ -107,11 +76,7 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
url = f"{endpoint}{domain}"
|
url = f"{endpoint}{domain}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
headers = {
|
async with httpx.AsyncClient(timeout=10) as client:
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; PounceBot/1.0; +https://pounce.ch)',
|
|
||||||
'Accept': 'application/rdap+json, application/json',
|
|
||||||
}
|
|
||||||
async with httpx.AsyncClient(timeout=10, headers=headers) as client:
|
|
||||||
resp = await client.get(url)
|
resp = await client.get(url)
|
||||||
|
|
||||||
# 404 = Domain not found = AVAILABLE!
|
# 404 = Domain not found = AVAILABLE!
|
||||||
@ -122,7 +87,7 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
rdap_status=[],
|
rdap_status=[],
|
||||||
can_register_now=True,
|
can_register_now=True,
|
||||||
should_monitor=False,
|
should_monitor=False,
|
||||||
message="Available now!"
|
message="Domain is available for registration!"
|
||||||
)
|
)
|
||||||
|
|
||||||
# 200 = Domain exists in registry
|
# 200 = Domain exists in registry
|
||||||
@ -133,16 +98,16 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
|
|
||||||
# Extract deletion date from events
|
# Extract deletion date from events
|
||||||
deletion_date = None
|
deletion_date = None
|
||||||
expiration_date = None
|
|
||||||
events = data.get('events', [])
|
events = data.get('events', [])
|
||||||
for event in events:
|
for event in events:
|
||||||
action = event.get('eventAction', '').lower()
|
action = event.get('eventAction', '').lower()
|
||||||
date_str = event.get('eventDate', '')
|
date_str = event.get('eventDate', '')
|
||||||
|
if action in ('deletion', 'expiration') and date_str:
|
||||||
if 'deletion' in action:
|
try:
|
||||||
deletion_date = date_str[:10] if date_str else None
|
# Parse ISO date
|
||||||
elif 'expiration' in action:
|
deletion_date = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
|
||||||
expiration_date = date_str[:10] if date_str else None
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
# Check for pending delete / redemption status
|
# Check for pending delete / redemption status
|
||||||
is_pending = any(x in status_lower for x in [
|
is_pending = any(x in status_lower for x in [
|
||||||
@ -153,25 +118,14 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
])
|
])
|
||||||
|
|
||||||
if is_pending:
|
if is_pending:
|
||||||
# Calculate estimated availability
|
|
||||||
estimated = None
|
|
||||||
if deletion_date:
|
|
||||||
try:
|
|
||||||
del_dt = datetime.strptime(deletion_date, '%Y-%m-%d')
|
|
||||||
# For most TLDs, domain is available shortly after deletion date
|
|
||||||
estimated = del_dt.strftime('%Y-%m-%d')
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return DropStatus(
|
return DropStatus(
|
||||||
domain=domain,
|
domain=domain,
|
||||||
status='dropping_soon',
|
status='dropping_soon',
|
||||||
rdap_status=rdap_status,
|
rdap_status=rdap_status,
|
||||||
can_register_now=False,
|
can_register_now=False,
|
||||||
should_monitor=True,
|
should_monitor=True,
|
||||||
message="Dropping soon - track to get notified!",
|
message="Domain is being deleted. Track it to get notified when available!",
|
||||||
deletion_date=deletion_date,
|
deletion_date=deletion_date,
|
||||||
estimated_available=estimated,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Domain is actively registered
|
# Domain is actively registered
|
||||||
@ -181,7 +135,8 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
rdap_status=rdap_status,
|
rdap_status=rdap_status,
|
||||||
can_register_now=False,
|
can_register_now=False,
|
||||||
should_monitor=False,
|
should_monitor=False,
|
||||||
message="Re-registered"
|
message="Domain was re-registered",
|
||||||
|
deletion_date=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Other status code
|
# Other status code
|
||||||
@ -192,17 +147,18 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
rdap_status=[],
|
rdap_status=[],
|
||||||
can_register_now=False,
|
can_register_now=False,
|
||||||
should_monitor=False,
|
should_monitor=False,
|
||||||
message=f"RDAP {resp.status_code}"
|
message=f"RDAP returned HTTP {resp.status_code}"
|
||||||
)
|
)
|
||||||
|
|
||||||
except httpx.TimeoutException:
|
except httpx.TimeoutException:
|
||||||
|
logger.warning(f"RDAP timeout for {domain}")
|
||||||
return DropStatus(
|
return DropStatus(
|
||||||
domain=domain,
|
domain=domain,
|
||||||
status='unknown',
|
status='unknown',
|
||||||
rdap_status=[],
|
rdap_status=[],
|
||||||
can_register_now=False,
|
can_register_now=False,
|
||||||
should_monitor=False,
|
should_monitor=False,
|
||||||
message="Timeout"
|
message="RDAP timeout"
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"RDAP error for {domain}: {e}")
|
logger.warning(f"RDAP error for {domain}: {e}")
|
||||||
@ -216,32 +172,46 @@ async def check_drop_status(domain: str) -> DropStatus:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def batch_check_drops(domains: list[tuple[int, str]], delay: float = 0.5) -> list[tuple[int, DropStatus]]:
|
# Rate limiting: max requests per second per TLD
|
||||||
|
RATE_LIMITS = {
|
||||||
|
'default': 5, # 5 requests per second
|
||||||
|
'ch': 10, # Swiss registry is faster
|
||||||
|
'li': 10,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def check_drops_batch(
|
||||||
|
domains: list[tuple[int, str]], # List of (id, full_domain)
|
||||||
|
delay_between_requests: float = 0.2, # 200ms = 5 req/s
|
||||||
|
) -> list[tuple[int, DropStatus]]:
|
||||||
"""
|
"""
|
||||||
Check status for multiple domains with rate limiting.
|
Check multiple drops with rate limiting.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
domains: List of (id, domain_name) tuples
|
domains: List of (drop_id, full_domain) tuples
|
||||||
delay: Delay between checks in seconds (to avoid rate limiting)
|
delay_between_requests: Seconds to wait between requests (default 200ms)
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of (id, DropStatus) tuples
|
|
||||||
"""
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (drop_id, DropStatus) tuples
|
||||||
|
"""
|
||||||
results = []
|
results = []
|
||||||
|
|
||||||
# Process sequentially with delay to avoid rate limiting (429)
|
for drop_id, domain in domains:
|
||||||
for i, (drop_id, domain) in enumerate(domains):
|
try:
|
||||||
status = await check_drop_status(domain)
|
status = await check_drop_status(domain)
|
||||||
results.append((drop_id, status))
|
results.append((drop_id, status))
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Batch check failed for {domain}: {e}")
|
||||||
|
results.append((drop_id, DropStatus(
|
||||||
|
domain=domain,
|
||||||
|
status='unknown',
|
||||||
|
rdap_status=[],
|
||||||
|
can_register_now=False,
|
||||||
|
should_monitor=False,
|
||||||
|
message=str(e),
|
||||||
|
)))
|
||||||
|
|
||||||
# Add delay between requests to avoid rate limiting
|
# Rate limit
|
||||||
if i < len(domains) - 1:
|
await asyncio.sleep(delay_between_requests)
|
||||||
await asyncio.sleep(delay)
|
|
||||||
|
|
||||||
# Log progress every 50 domains
|
|
||||||
if (i + 1) % 50 == 0:
|
|
||||||
logger.info(f"Checked {i + 1}/{len(domains)} drops...")
|
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|||||||
@ -374,7 +374,7 @@ async def get_dropped_domains(
|
|||||||
"has_hyphen": item.has_hyphen,
|
"has_hyphen": item.has_hyphen,
|
||||||
"availability_status": getattr(item, 'availability_status', 'unknown') or 'unknown',
|
"availability_status": getattr(item, 'availability_status', 'unknown') or 'unknown',
|
||||||
"last_status_check": item.last_status_check.isoformat() if getattr(item, 'last_status_check', None) else None,
|
"last_status_check": item.last_status_check.isoformat() if getattr(item, 'last_status_check', None) else None,
|
||||||
"deletion_date": getattr(item, 'deletion_date', None),
|
"deletion_date": item.deletion_date.isoformat() if getattr(item, 'deletion_date', None) else None,
|
||||||
}
|
}
|
||||||
for item in items
|
for item in items
|
||||||
]
|
]
|
||||||
@ -458,15 +458,16 @@ async def cleanup_old_snapshots(db: AsyncSession, keep_days: int = 7) -> int:
|
|||||||
|
|
||||||
async def verify_drops_availability(
|
async def verify_drops_availability(
|
||||||
db: AsyncSession,
|
db: AsyncSession,
|
||||||
batch_size: int = 100,
|
batch_size: int = 50,
|
||||||
max_checks: int = 500
|
max_checks: int = 200
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""
|
"""
|
||||||
Verify availability of dropped domains and remove those that are no longer available.
|
Verify availability of dropped domains and update their status.
|
||||||
|
|
||||||
This runs periodically to clean up the drops list by checking if domains
|
This runs periodically to check the real RDAP status of drops.
|
||||||
have been re-registered. If a domain is no longer available (taken),
|
Updates availability_status and deletion_date fields.
|
||||||
it's removed from the drops list.
|
|
||||||
|
Rate limited: ~200ms between requests = ~5 req/sec
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
db: Database session
|
db: Database session
|
||||||
@ -474,20 +475,28 @@ async def verify_drops_availability(
|
|||||||
max_checks: Maximum domains to check per run (to avoid overload)
|
max_checks: Maximum domains to check per run (to avoid overload)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
dict with stats: checked, removed, errors
|
dict with stats: checked, available, dropping_soon, taken, errors
|
||||||
"""
|
"""
|
||||||
from sqlalchemy import delete
|
from sqlalchemy import update
|
||||||
from app.services.domain_checker import domain_checker
|
from app.services.drop_status_checker import check_drop_status
|
||||||
|
|
||||||
logger.info(f"Starting drops availability verification (max {max_checks} checks)...")
|
logger.info(f"Starting drops status update (max {max_checks} checks)...")
|
||||||
|
|
||||||
# Get drops from last 24h that haven't been verified recently
|
# Get drops that haven't been checked recently (prioritize unchecked and short domains)
|
||||||
cutoff = datetime.utcnow() - timedelta(hours=24)
|
cutoff = datetime.utcnow() - timedelta(hours=24)
|
||||||
|
check_cutoff = datetime.utcnow() - timedelta(hours=2) # Re-check every 2 hours
|
||||||
|
|
||||||
query = (
|
query = (
|
||||||
select(DroppedDomain)
|
select(DroppedDomain)
|
||||||
.where(DroppedDomain.dropped_date >= cutoff)
|
.where(DroppedDomain.dropped_date >= cutoff)
|
||||||
.order_by(DroppedDomain.length.asc()) # Check short domains first (more valuable)
|
.where(
|
||||||
|
(DroppedDomain.last_status_check == None) | # Never checked
|
||||||
|
(DroppedDomain.last_status_check < check_cutoff) # Not checked recently
|
||||||
|
)
|
||||||
|
.order_by(
|
||||||
|
DroppedDomain.availability_status.desc(), # Unknown first
|
||||||
|
DroppedDomain.length.asc() # Then short domains
|
||||||
|
)
|
||||||
.limit(max_checks)
|
.limit(max_checks)
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -495,59 +504,59 @@ async def verify_drops_availability(
|
|||||||
drops = result.scalars().all()
|
drops = result.scalars().all()
|
||||||
|
|
||||||
if not drops:
|
if not drops:
|
||||||
logger.info("No drops to verify")
|
logger.info("No drops need status update")
|
||||||
return {"checked": 0, "removed": 0, "errors": 0, "available": 0}
|
return {"checked": 0, "available": 0, "dropping_soon": 0, "taken": 0, "errors": 0}
|
||||||
|
|
||||||
checked = 0
|
checked = 0
|
||||||
removed = 0
|
stats = {"available": 0, "dropping_soon": 0, "taken": 0, "unknown": 0}
|
||||||
errors = 0
|
errors = 0
|
||||||
available = 0
|
|
||||||
domains_to_remove = []
|
|
||||||
|
|
||||||
logger.info(f"Verifying {len(drops)} dropped domains...")
|
logger.info(f"Checking {len(drops)} dropped domains...")
|
||||||
|
|
||||||
for i, drop in enumerate(drops):
|
for i, drop in enumerate(drops):
|
||||||
|
full_domain = f"{drop.domain}.{drop.tld}"
|
||||||
try:
|
try:
|
||||||
# Quick DNS-only check for speed
|
status_result = await check_drop_status(full_domain)
|
||||||
result = await domain_checker.check_domain(drop.domain)
|
|
||||||
checked += 1
|
checked += 1
|
||||||
|
stats[status_result.status] = stats.get(status_result.status, 0) + 1
|
||||||
|
|
||||||
if result.is_available:
|
# Update in DB
|
||||||
available += 1
|
await db.execute(
|
||||||
else:
|
update(DroppedDomain)
|
||||||
# Domain is taken - mark for removal
|
.where(DroppedDomain.id == drop.id)
|
||||||
domains_to_remove.append(drop.id)
|
.values(
|
||||||
logger.debug(f"Domain {drop.domain} is now taken, marking for removal")
|
availability_status=status_result.status,
|
||||||
|
rdap_status=str(status_result.rdap_status)[:255] if status_result.rdap_status else None,
|
||||||
|
last_status_check=datetime.utcnow(),
|
||||||
|
deletion_date=status_result.deletion_date,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# Log progress every 50 domains
|
# Log progress every 25 domains
|
||||||
if (i + 1) % 50 == 0:
|
if (i + 1) % 25 == 0:
|
||||||
logger.info(f"Verified {i + 1}/{len(drops)} domains, {len(domains_to_remove)} taken so far")
|
logger.info(f"Checked {i + 1}/{len(drops)}: {stats}")
|
||||||
|
await db.commit() # Commit in batches
|
||||||
|
|
||||||
# Small delay to avoid hammering DNS
|
# Rate limit: 200ms between requests (5 req/sec)
|
||||||
if i % 10 == 0:
|
await asyncio.sleep(0.2)
|
||||||
await asyncio.sleep(0.1)
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
errors += 1
|
errors += 1
|
||||||
logger.warning(f"Error checking {drop.domain}: {e}")
|
logger.warning(f"Error checking {full_domain}: {e}")
|
||||||
|
|
||||||
# Remove taken domains in batch
|
# Final commit
|
||||||
if domains_to_remove:
|
await db.commit()
|
||||||
stmt = delete(DroppedDomain).where(DroppedDomain.id.in_(domains_to_remove))
|
|
||||||
await db.execute(stmt)
|
|
||||||
await db.commit()
|
|
||||||
removed = len(domains_to_remove)
|
|
||||||
logger.info(f"Removed {removed} taken domains from drops list")
|
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Drops verification complete: "
|
f"Drops status update complete: "
|
||||||
f"{checked} checked, {available} still available, "
|
f"{checked} checked, {stats['available']} available, "
|
||||||
f"{removed} removed (taken), {errors} errors"
|
f"{stats['dropping_soon']} dropping_soon, {stats['taken']} taken, {errors} errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"checked": checked,
|
"checked": checked,
|
||||||
"removed": removed,
|
"available": stats['available'],
|
||||||
"errors": errors,
|
"dropping_soon": stats['dropping_soon'],
|
||||||
"available": available
|
"taken": stats['taken'],
|
||||||
|
"errors": errors
|
||||||
}
|
}
|
||||||
|
|||||||
@ -175,7 +175,12 @@ export function DropsTab({ showToast }: DropsTabProps) {
|
|||||||
// Update the item in our list
|
// Update the item in our list
|
||||||
setItems(prev => prev.map(item =>
|
setItems(prev => prev.map(item =>
|
||||||
item.id === dropId
|
item.id === dropId
|
||||||
? { ...item, availability_status: result.status, last_status_check: new Date().toISOString() }
|
? {
|
||||||
|
...item,
|
||||||
|
availability_status: result.status,
|
||||||
|
last_status_check: new Date().toISOString(),
|
||||||
|
deletion_date: result.deletion_date,
|
||||||
|
}
|
||||||
: item
|
: item
|
||||||
))
|
))
|
||||||
|
|
||||||
@ -186,6 +191,25 @@ export function DropsTab({ showToast }: DropsTabProps) {
|
|||||||
setCheckingStatus(null)
|
setCheckingStatus(null)
|
||||||
}
|
}
|
||||||
}, [checkingStatus, showToast])
|
}, [checkingStatus, showToast])
|
||||||
|
|
||||||
|
// Format countdown from deletion date
|
||||||
|
const formatCountdown = useCallback((deletionDate: string | null): string | null => {
|
||||||
|
if (!deletionDate) return null
|
||||||
|
|
||||||
|
const del = new Date(deletionDate)
|
||||||
|
const now = new Date()
|
||||||
|
const diff = del.getTime() - now.getTime()
|
||||||
|
|
||||||
|
if (diff <= 0) return 'Now'
|
||||||
|
|
||||||
|
const days = Math.floor(diff / (1000 * 60 * 60 * 24))
|
||||||
|
const hours = Math.floor((diff % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60))
|
||||||
|
const mins = Math.floor((diff % (1000 * 60 * 60)) / (1000 * 60))
|
||||||
|
|
||||||
|
if (days > 0) return `${days}d ${hours}h`
|
||||||
|
if (hours > 0) return `${hours}h ${mins}m`
|
||||||
|
return `${mins}m`
|
||||||
|
}, [])
|
||||||
|
|
||||||
// Track a drop (add to watchlist)
|
// Track a drop (add to watchlist)
|
||||||
const trackDrop = useCallback(async (dropId: number, domain: string) => {
|
const trackDrop = useCallback(async (dropId: number, domain: string) => {
|
||||||
@ -253,24 +277,6 @@ export function DropsTab({ showToast }: DropsTabProps) {
|
|||||||
showOnlyAvailable,
|
showOnlyAvailable,
|
||||||
].filter(Boolean).length
|
].filter(Boolean).length
|
||||||
|
|
||||||
// Format countdown to deletion date
|
|
||||||
const formatCountdown = (deletionDate: string | null) => {
|
|
||||||
if (!deletionDate) return null
|
|
||||||
|
|
||||||
const now = new Date()
|
|
||||||
const target = new Date(deletionDate + 'T23:59:59')
|
|
||||||
const diffMs = target.getTime() - now.getTime()
|
|
||||||
|
|
||||||
if (diffMs <= 0) return 'Now'
|
|
||||||
|
|
||||||
const days = Math.floor(diffMs / (1000 * 60 * 60 * 24))
|
|
||||||
const hours = Math.floor((diffMs % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60))
|
|
||||||
|
|
||||||
if (days > 0) return `${days}d ${hours}h`
|
|
||||||
if (hours > 0) return `${hours}h`
|
|
||||||
return '<1h'
|
|
||||||
}
|
|
||||||
|
|
||||||
const formatTime = (iso: string) => {
|
const formatTime = (iso: string) => {
|
||||||
const d = new Date(iso)
|
const d = new Date(iso)
|
||||||
const now = new Date()
|
const now = new Date()
|
||||||
@ -553,15 +559,12 @@ export function DropsTab({ showToast }: DropsTabProps) {
|
|||||||
const isTrackingThis = trackingDrop === item.id
|
const isTrackingThis = trackingDrop === item.id
|
||||||
const status = item.availability_status || 'unknown'
|
const status = item.availability_status || 'unknown'
|
||||||
|
|
||||||
// Countdown for dropping_soon domains
|
|
||||||
const countdown = formatCountdown(item.deletion_date)
|
|
||||||
|
|
||||||
// Simplified status display config
|
// Simplified status display config
|
||||||
const statusConfig = {
|
const statusConfig = {
|
||||||
available: { label: 'Available', color: 'text-accent', bg: 'bg-accent/10', border: 'border-accent/30', icon: CheckCircle2 },
|
available: { label: 'Available', color: 'text-accent', bg: 'bg-accent/10', border: 'border-accent/30', icon: CheckCircle2 },
|
||||||
dropping_soon: { label: countdown || 'Dropping', color: 'text-amber-400', bg: 'bg-amber-400/10', border: 'border-amber-400/30', icon: Clock },
|
dropping_soon: { label: 'Dropping Soon', color: 'text-amber-400', bg: 'bg-amber-400/10', border: 'border-amber-400/30', icon: Clock },
|
||||||
taken: { label: 'Taken', color: 'text-rose-400', bg: 'bg-rose-400/10', border: 'border-rose-400/30', icon: Ban },
|
taken: { label: 'Taken', color: 'text-rose-400', bg: 'bg-rose-400/10', border: 'border-rose-400/30', icon: Ban },
|
||||||
unknown: { label: 'Pending', color: 'text-white/50', bg: 'bg-white/5', border: 'border-white/20', icon: Clock },
|
unknown: { label: 'Check', color: 'text-white/50', bg: 'bg-white/5', border: 'border-white/20', icon: Search },
|
||||||
}[status]
|
}[status]
|
||||||
|
|
||||||
const StatusIcon = statusConfig.icon
|
const StatusIcon = statusConfig.icon
|
||||||
@ -596,7 +599,9 @@ export function DropsTab({ showToast }: DropsTabProps) {
|
|||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
{isChecking ? <Loader2 className="w-3 h-3 animate-spin" /> : <StatusIcon className="w-3 h-3" />}
|
{isChecking ? <Loader2 className="w-3 h-3 animate-spin" /> : <StatusIcon className="w-3 h-3" />}
|
||||||
{statusConfig.label}
|
{status === 'dropping_soon' && item.deletion_date
|
||||||
|
? formatCountdown(item.deletion_date)
|
||||||
|
: statusConfig.label}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@ -689,7 +694,9 @@ export function DropsTab({ showToast }: DropsTabProps) {
|
|||||||
title="Click to check real-time status"
|
title="Click to check real-time status"
|
||||||
>
|
>
|
||||||
{isChecking ? <Loader2 className="w-3 h-3 animate-spin" /> : <StatusIcon className="w-3 h-3" />}
|
{isChecking ? <Loader2 className="w-3 h-3 animate-spin" /> : <StatusIcon className="w-3 h-3" />}
|
||||||
{statusConfig.label}
|
{status === 'dropping_soon' && item.deletion_date
|
||||||
|
? formatCountdown(item.deletion_date)
|
||||||
|
: statusConfig.label}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@ -2073,10 +2073,11 @@ class AdminApiClient extends ApiClient {
|
|||||||
id: number
|
id: number
|
||||||
domain: string
|
domain: string
|
||||||
status: 'available' | 'dropping_soon' | 'taken' | 'unknown'
|
status: 'available' | 'dropping_soon' | 'taken' | 'unknown'
|
||||||
deletion_date: string | null
|
rdap_status: string[]
|
||||||
can_register_now: boolean
|
can_register_now: boolean
|
||||||
should_track: boolean
|
should_track: boolean
|
||||||
message: string
|
message: string
|
||||||
|
deletion_date: string | null
|
||||||
}>(`/drops/check-status/${dropId}`, { method: 'POST' })
|
}>(`/drops/check-status/${dropId}`, { method: 'POST' })
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2087,14 +2088,6 @@ class AdminApiClient extends ApiClient {
|
|||||||
message: string
|
message: string
|
||||||
}>(`/drops/track/${dropId}`, { method: 'POST' })
|
}>(`/drops/track/${dropId}`, { method: 'POST' })
|
||||||
}
|
}
|
||||||
|
|
||||||
async batchCheckDrops(limit: number = 50) {
|
|
||||||
return this.request<{
|
|
||||||
message: string
|
|
||||||
checking?: number
|
|
||||||
checked?: number
|
|
||||||
}>(`/drops/batch-check?limit=${limit}`, { method: 'POST' })
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Yield Types
|
// Yield Types
|
||||||
|
|||||||
Reference in New Issue
Block a user