diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index e7cf4d0..c5de331 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -50,6 +50,8 @@ jobs: GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} CZDS_USERNAME: ${{ secrets.CZDS_USERNAME }} CZDS_PASSWORD: ${{ secrets.CZDS_PASSWORD }} + SWITCH_TSIG_CH_SECRET: ${{ secrets.SWITCH_TSIG_CH_SECRET }} + SWITCH_TSIG_LI_SECRET: ${{ secrets.SWITCH_TSIG_LI_SECRET }} run: | python3 - <<'PY' import os @@ -110,6 +112,10 @@ jobs: # CZDS "CZDS_USERNAME": os.environ["CZDS_USERNAME"], "CZDS_PASSWORD": os.environ["CZDS_PASSWORD"], + + # Switch TSIG (AXFR) + "SWITCH_TSIG_CH_SECRET": os.environ["SWITCH_TSIG_CH_SECRET"], + "SWITCH_TSIG_LI_SECRET": os.environ["SWITCH_TSIG_LI_SECRET"], } lines = [] diff --git a/UX_TERMINAL_UX_REPORT.md b/UX_TERMINAL_UX_REPORT.md index c584b6d..e25a37b 100644 --- a/UX_TERMINAL_UX_REPORT.md +++ b/UX_TERMINAL_UX_REPORT.md @@ -331,3 +331,5 @@ Empfehlungen: + + diff --git a/backend/app/config.py b/backend/app/config.py index bace25d..7f26f36 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -133,9 +133,23 @@ class Settings(BaseSettings): # Switch.ch Zone Files (.ch, .li) switch_data_dir: str = "/data/switch" # Persistent storage + + # Switch.ch TSIG (DNS AXFR) credentials + # These should be provided via environment variables in production. + switch_tsig_ch_name: str = "tsig-zonedata-ch-public-21-01" + switch_tsig_ch_algorithm: str = "hmac-sha512" + switch_tsig_ch_secret: str = "" + + switch_tsig_li_name: str = "tsig-zonedata-li-public-21-01" + switch_tsig_li_algorithm: str = "hmac-sha512" + switch_tsig_li_secret: str = "" # Zone File Retention (days to keep historical snapshots) zone_retention_days: int = 3 + + # Domain check scheduler tuning (external I/O heavy; keep conservative defaults) + domain_check_max_concurrent: int = 3 + domain_check_delay_seconds: float = 0.3 class Config: env_file = ".env" diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index c4bc658..8d3c644 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -88,7 +88,6 @@ async def check_domains_by_frequency(frequency: str): tiers_for_frequency.append(tier) # Get domains from users with matching subscription tier - from sqlalchemy.orm import joinedload result = await db.execute( select(Domain) .join(User, Domain.user_id == User.id) @@ -99,56 +98,79 @@ async def check_domains_by_frequency(frequency: str): ) ) domains = result.scalars().all() - + logger.info(f"Checking {len(domains)} domains...") - + checked = 0 errors = 0 newly_available = [] newly_taken = [] # Track domains that became taken status_changes = [] # All status changes for logging - - for domain in domains: - try: - # Check domain availability - check_result = await domain_checker.check_domain(domain.name) - + + # Concurrency control + polite pacing (prevents RDAP/WHOIS bans) + max_concurrent = max(1, int(getattr(settings, "domain_check_max_concurrent", 3) or 3)) + delay = float(getattr(settings, "domain_check_delay_seconds", 0.3) or 0.3) + semaphore = asyncio.Semaphore(max_concurrent) + + async def _check_one(d: Domain) -> tuple[Domain, object | None, Exception | None]: + async with semaphore: + try: + res = await domain_checker.check_domain(d.name) + # small delay after each external request + await asyncio.sleep(delay) + return d, res, None + except Exception as e: + return d, None, e + + # Process in chunks to avoid huge gather lists + chunk_size = 200 + for i in range(0, len(domains), chunk_size): + chunk = domains[i : i + chunk_size] + results = await asyncio.gather(*[_check_one(d) for d in chunk]) + + for domain, check_result, err in results: + if err is not None or check_result is None: + logger.error(f"Error checking domain {domain.name}: {err}") + errors += 1 + continue + # Track status transitions was_available = domain.is_available is_now_available = check_result.is_available - + # Detect transition: taken -> available (domain dropped!) if not was_available and is_now_available: - status_changes.append({ - 'domain': domain.name, - 'change': 'became_available', - 'old_registrar': domain.registrar, - }) + status_changes.append( + { + "domain": domain.name, + "change": "became_available", + "old_registrar": domain.registrar, + } + ) if domain.notify_on_available: newly_available.append(domain) logger.info(f"🎯 Domain AVAILABLE: {domain.name} (was registered by {domain.registrar})") - + # Detect transition: available -> taken (someone registered it!) elif was_available and not is_now_available: - status_changes.append({ - 'domain': domain.name, - 'change': 'became_taken', - 'new_registrar': check_result.registrar, - }) + status_changes.append( + { + "domain": domain.name, + "change": "became_taken", + "new_registrar": check_result.registrar, + } + ) if domain.notify_on_available: # Notify if alerts are on - newly_taken.append({ - 'domain': domain, - 'registrar': check_result.registrar, - }) + newly_taken.append({"domain": domain, "registrar": check_result.registrar}) logger.info(f"⚠️ Domain TAKEN: {domain.name} (now registered by {check_result.registrar})") - + # Update domain with fresh data domain.status = check_result.status domain.is_available = check_result.is_available domain.registrar = check_result.registrar domain.expiration_date = check_result.expiration_date domain.last_checked = datetime.utcnow() - + # Create check record for history check = DomainCheck( domain_id=domain.id, @@ -158,15 +180,7 @@ async def check_domains_by_frequency(frequency: str): checked_at=datetime.utcnow(), ) db.add(check) - checked += 1 - - # Small delay to avoid rate limiting - await asyncio.sleep(0.5) - - except Exception as e: - logger.error(f"Error checking domain {domain.name}: {e}") - errors += 1 await db.commit() diff --git a/backend/app/services/zone_file.py b/backend/app/services/zone_file.py index 0e81d3b..56135f1 100644 --- a/backend/app/services/zone_file.py +++ b/backend/app/services/zone_file.py @@ -17,28 +17,12 @@ from typing import Optional from sqlalchemy import select, func from sqlalchemy.ext.asyncio import AsyncSession +from app.config import get_settings from app.models.zone_file import ZoneSnapshot, DroppedDomain logger = logging.getLogger(__name__) -# ============================================================================ -# TSIG KEYS (from Switch.ch documentation) -# ============================================================================ - -TSIG_KEYS = { - "ch": { - "name": "tsig-zonedata-ch-public-21-01", - "algorithm": "hmac-sha512", - "secret": "stZwEGApYumtXkh73qMLPqfbIDozWKZLkqRvcjKSpRnsor6A6MxixRL6C2HeSVBQNfMW4wer+qjS0ZSfiWiJ3Q==" - }, - "li": { - "name": "tsig-zonedata-li-public-21-01", - "algorithm": "hmac-sha512", - "secret": "t8GgeCn+fhPaj+cRy1epox2Vj4hZ45ax6v3rQCkkfIQNg5fsxuU23QM5mzz+BxJ4kgF/jiQyBDBvL+XWPE6oCQ==" - } -} - ZONE_SERVER = "zonedata.switch.ch" # ============================================================================ @@ -49,18 +33,36 @@ class ZoneFileService: """Service for fetching and analyzing zone files""" def __init__(self, data_dir: Optional[Path] = None): - from app.config import get_settings settings = get_settings() self.data_dir = data_dir or Path(settings.switch_data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) + self._settings = settings + # Store daily snapshots for N days (premium reliability) + self.snapshots_dir = self.data_dir / "snapshots" + self.snapshots_dir.mkdir(parents=True, exist_ok=True) + + def _get_tsig_config(self, tld: str) -> dict: + """Resolve TSIG config from settings/env (no secrets in git).""" + if tld == "ch": + return { + "name": self._settings.switch_tsig_ch_name, + "algorithm": self._settings.switch_tsig_ch_algorithm, + "secret": self._settings.switch_tsig_ch_secret, + } + if tld == "li": + return { + "name": self._settings.switch_tsig_li_name, + "algorithm": self._settings.switch_tsig_li_algorithm, + "secret": self._settings.switch_tsig_li_secret, + } + raise ValueError(f"Unknown TLD: {tld}") def _get_key_file_path(self, tld: str) -> Path: """Generate TSIG key file for dig command""" key_path = self.data_dir / f"{tld}_zonedata.key" - key_info = TSIG_KEYS.get(tld) - - if not key_info: - raise ValueError(f"Unknown TLD: {tld}") + key_info = self._get_tsig_config(tld) + if not (key_info.get("secret") or "").strip(): + raise RuntimeError(f"Missing Switch TSIG secret for .{tld} (set SWITCH_TSIG_{tld.upper()}_SECRET)") # Write TSIG key file in BIND format key_content = f"""key "{key_info['name']}" {{ @@ -76,7 +78,7 @@ class ZoneFileService: Fetch zone file via DNS AXFR transfer. Returns set of domain names (without TLD suffix). """ - if tld not in TSIG_KEYS: + if tld not in ("ch", "li"): raise ValueError(f"Unsupported TLD: {tld}. Only 'ch' and 'li' are supported.") logger.info(f"Starting zone transfer for .{tld}") @@ -143,22 +145,60 @@ class ZoneFileService: async def get_previous_snapshot(self, db: AsyncSession, tld: str) -> Optional[set[str]]: """Load previous day's domain set from cache file""" + # Prefer most recent snapshot file before today (supports N-day retention) + tld_dir = self.snapshots_dir / tld + if tld_dir.exists(): + candidates = sorted([p for p in tld_dir.glob("*.domains.txt") if p.is_file()]) + if candidates: + # Pick the latest snapshot file (by name sort = date sort) + latest = candidates[-1] + try: + content = latest.read_text() + return set(line.strip() for line in content.splitlines() if line.strip()) + except Exception as e: + logger.warning(f"Failed to load snapshot for .{tld} from {latest.name}: {e}") + + # Fallback: legacy cache file cache_file = self.data_dir / f"{tld}_domains.txt" - if cache_file.exists(): try: content = cache_file.read_text() return set(line.strip() for line in content.splitlines() if line.strip()) except Exception as e: logger.warning(f"Failed to load cache for .{tld}: {e}") - return None + + def _cleanup_snapshot_files(self, tld: str) -> None: + """Delete snapshot files older than retention window (best-effort).""" + keep_days = int(self._settings.zone_retention_days or 3) + cutoff = datetime.utcnow().date() - timedelta(days=keep_days) + tld_dir = self.snapshots_dir / tld + if not tld_dir.exists(): + return + for p in tld_dir.glob("*.domains.txt"): + try: + # filename: YYYY-MM-DD.domains.txt + date_part = p.name.split(".")[0] + snap_date = datetime.fromisoformat(date_part).date() + if snap_date < cutoff: + p.unlink(missing_ok=True) + except Exception: + # Don't let cleanup break sync + continue async def save_snapshot(self, db: AsyncSession, tld: str, domains: set[str]): """Save current snapshot to cache and database""" - # Save to cache file + # Save to legacy cache file (fast path) cache_file = self.data_dir / f"{tld}_domains.txt" cache_file.write_text("\n".join(sorted(domains))) + + # Save a daily snapshot file for retention/debugging + tld_dir = self.snapshots_dir / tld + tld_dir.mkdir(parents=True, exist_ok=True) + today_str = datetime.utcnow().date().isoformat() + snapshot_file = tld_dir / f"{today_str}.domains.txt" + snapshot_file.write_text("\n".join(sorted(domains))) + self._cleanup_snapshot_files(tld) # Save metadata to database checksum = self.compute_checksum(domains)