fix: Disable RDAP verification to prevent bans, improve drops UI

- Disabled verify_drops scheduler job (caused RDAP rate limit bans)
- Zone files now saved without RDAP verification (zone diff is reliable)
- Added date-based zone file snapshots with 3-day retention
- Improved DropsTab UI with better status display:
  - "In Transition" with countdown timer for dropping_soon
  - "Available Now" with Buy button
  - "Re-registered" for taken domains
  - Track button for dropping_soon domains
- Added --shm-size=8g to backend container for multiprocessing
- Removed duplicate host cron job (scheduler handles everything)

.gitea/workflows/deploy.yml

@@ -57,6 +57,8 @@ jobs:
           STRIPE_WEBHOOK_SECRET: ${{ secrets.STRIPE_WEBHOOK_SECRET }}
           GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
           GITHUB_CLIENT_SECRET: ${{ secrets.GITHUB_CLIENT_SECRET }}
+          CZDS_USERNAME: ${{ secrets.CZDS_USERNAME }}
+          CZDS_PASSWORD: ${{ secrets.CZDS_PASSWORD }}
         run: |
           # Stop existing container
           docker stop pounce-backend 2>/dev/null || true
@@ -71,10 +73,13 @@ jobs:
             --name pounce-backend \
             --network n0488s44osgoow4wgo04ogg0 \
             --restart unless-stopped \
+            --shm-size=8g \
             -v /data/pounce/zones/czds:/data/czds \
             -v /data/pounce/zones/switch:/data/switch \
             -v /data/pounce/logs:/data/logs \
             -e CZDS_DATA_DIR="/data/czds" \
+            -e CZDS_USERNAME="${CZDS_USERNAME}" \
+            -e CZDS_PASSWORD="${CZDS_PASSWORD}" \
             -e SWITCH_DATA_DIR="/data/switch" \
             -e ZONE_RETENTION_DAYS="3" \
             -e DATABASE_URL="${DATABASE_URL}" \

UX_TERMINAL_UX_REPORT.md

@@ -318,3 +318,9 @@ Empfehlungen:
 
 
 
+
+
+
+
+
+

backend/app/scheduler.py

@@ -726,14 +726,16 @@ def setup_scheduler():
         replace_existing=True,
     )
     
-    # Drops availability verification (every 10 minutes - remove taken domains)
+    # Drops availability verification - DISABLED to prevent RDAP bans
-    scheduler.add_job(
+    # The domains from zone files are already verified as "dropped" by the zone diff
-        verify_drops,
+    # We don't need to double-check via RDAP - this causes rate limiting!
-        CronTrigger(minute='*/10'),  # Every 10 minutes
+    # scheduler.add_job(
-        id="drops_verification",
+    #     verify_drops,
-        name="Drops Availability Check (10-min)",
+    #     CronTrigger(hour=12, minute=0),  # Once a day at noon if needed
-        replace_existing=True,
+    #     id="drops_verification",
-    )
+    #     name="Drops Availability Check (daily)",
+    #     replace_existing=True,
+    # )
     
     logger.info(
         f"Scheduler configured:"
@@ -743,10 +745,11 @@ def setup_scheduler():
         f"\n  - TLD price scrape 2x daily at 03:00 & 15:00 UTC"
         f"\n  - Price change alerts at 04:00 & 16:00 UTC"
         f"\n  - Auction scrape every 2 hours at :30"
-        f"\n  - Expired auction cleanup every 15 minutes"
+        f"\n  - Expired auction cleanup every 5 minutes"
         f"\n  - Sniper alert matching every 30 minutes"
-        f"\n  - Zone file sync daily at 05:00 UTC"
+        f"\n  - Switch.ch zone sync daily at 05:00 UTC (.ch, .li)"
-        f"\n  - Drops availability check every 10 minutes"
+        f"\n  - ICANN CZDS zone sync daily at 06:00 UTC (gTLDs)"
+        f"\n  - Zone cleanup hourly at :45"
     )
 
 

backend/app/services/czds_client.py

@@ -227,11 +227,43 @@ class CZDSClient:
         return None
     
     async def save_domains(self, tld: str, domains: set[str]):
-        """Save current domains to cache file."""
+        """Save current domains to cache file with date-based retention."""
+        from app.config import get_settings
+        settings = get_settings()
+        
+        # Save current file (for next sync comparison)
         cache_file = self.data_dir / f"{tld}_domains.txt"
         cache_file.write_text("\n".join(sorted(domains)))
+        
+        # Also save dated snapshot for retention
+        today = datetime.now().strftime("%Y-%m-%d")
+        dated_file = self.data_dir / f"{tld}_domains_{today}.txt"
+        if not dated_file.exists():
+            dated_file.write_text("\n".join(sorted(domains)))
+            logger.info(f"Saved snapshot: {dated_file.name}")
+        
+        # Cleanup old snapshots (keep last N days)
+        retention_days = getattr(settings, 'zone_retention_days', 3)
+        await self._cleanup_old_snapshots(tld, retention_days)
+        
         logger.info(f"Saved {len(domains):,} domains for .{tld}")
     
+    async def _cleanup_old_snapshots(self, tld: str, keep_days: int = 3):
+        """Remove zone file snapshots older than keep_days."""
+        import re
+        from datetime import timedelta
+        
+        cutoff = datetime.now() - timedelta(days=keep_days)
+        pattern = re.compile(rf"^{tld}_domains_(\d{{4}}-\d{{2}}-\d{{2}})\.txt$")
+        
+        for file in self.data_dir.glob(f"{tld}_domains_*.txt"):
+            match = pattern.match(file.name)
+            if match:
+                file_date = datetime.strptime(match.group(1), "%Y-%m-%d")
+                if file_date < cutoff:
+                    file.unlink()
+                    logger.info(f"Deleted old snapshot: {file.name}")
+    
     async def process_drops(
         self, 
         db: AsyncSession, 
@@ -240,87 +272,66 @@ class CZDSClient:
         current: set[str]
     ) -> list[dict]:
         """
-        Find dropped domains and verify they are ACTUALLY available before storing.
+        Find dropped domains and store them directly.
         
-        Zone file drops are often immediately re-registered by drop-catching services,
+        NOTE: We do NOT verify availability here to avoid RDAP rate limits/bans.
-        so we must verify availability before storing to avoid showing unavailable domains.
+        Verification happens separately in the 'verify_drops' scheduler job
+        which runs in small batches throughout the day.
         """
-        from app.services.domain_checker import domain_checker
-        
         dropped = previous - current
         
         if not dropped:
             logger.info(f"No dropped domains found for .{tld}")
             return []
         
-        logger.info(f"Found {len(dropped):,} potential drops for .{tld}, verifying availability...")
+        logger.info(f"Found {len(dropped):,} dropped domains for .{tld}, saving to database...")
         
         today = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
         
-        # Filter to valuable domains first (short, no numbers, no hyphens)
+        # Store all drops - availability will be verified separately
-        valuable_drops = [
-            name for name in dropped
-            if len(name) <= 10 and not name.isdigit() and '-' not in name
-        ]
-        
-        # Also include some longer domains (up to 500 total)
-        other_drops = [
-            name for name in dropped
-            if name not in valuable_drops and len(name) <= 15
-        ][:max(0, 500 - len(valuable_drops))]
-        
-        candidates = valuable_drops + other_drops
-        logger.info(f"Checking availability of {len(candidates)} candidates (of {len(dropped):,} total drops)")
-        
-        # Verify availability and only store truly available domains
         dropped_records = []
-        available_count = 0
+        batch_size = 1000
-        checked_count = 0
+        dropped_list = list(dropped)
         
-        for i, name in enumerate(candidates):
+        for i in range(0, len(dropped_list), batch_size):
-            full_domain = f"{name}.{tld}"
+            batch = dropped_list[i:i + batch_size]
             
-            try:
+            for name in batch:
-                # Quick DNS check
+                try:
-                result = await domain_checker.check_domain(full_domain)
-                checked_count += 1
-                
-                if result.is_available:
-                    available_count += 1
                     record = DroppedDomain(
-                        domain=full_domain,
+                        domain=name,  # Just the name, not full domain!
                         tld=tld,
                         dropped_date=today,
                         length=len(name),
                         is_numeric=name.isdigit(),
-                        has_hyphen='-' in name
+                        has_hyphen='-' in name,
+                        availability_status='unknown'  # Will be verified later
                     )
                     db.add(record)
                     dropped_records.append({
-                        "domain": full_domain,
+                        "domain": f"{name}.{tld}",
                         "length": len(name),
-                        "is_numeric": name.isdigit(),
-                        "has_hyphen": '-' in name
                     })
-                
+                except Exception as e:
-                # Progress log every 100 domains
+                    # Duplicate or other error - skip
-                if (i + 1) % 100 == 0:
+                    pass
-                    logger.info(f"Verified {i + 1}/{len(candidates)}: {available_count} available so far")
+            
-                
+            # Commit batch
-                # Small delay to avoid rate limiting
+            try:
-                if i % 20 == 0:
+                await db.commit()
-                    await asyncio.sleep(0.1)
+            except Exception:
-                    
+                await db.rollback()
-            except Exception as e:
+            
-                logger.warning(f"Error checking {full_domain}: {e}")
+            if (i + batch_size) % 5000 == 0:
+                logger.info(f"Saved {min(i + batch_size, len(dropped_list)):,}/{len(dropped_list):,} drops")
         
-        await db.commit()
+        # Final commit
+        try:
+            await db.commit()
+        except Exception:
+            await db.rollback()
         
-        logger.info(
+        logger.info(f"CZDS drops for .{tld}: {len(dropped_records):,} saved (verification pending)")
-            f"CZDS drops for .{tld}: "
-            f"{checked_count} verified, {available_count} actually available, "
-            f"{len(dropped_records)} stored"
-        )
         
         return dropped_records
     
@@ -371,7 +382,9 @@ class CZDSClient:
             result["current_count"] = len(current_domains)
             
             # Clean up zone file (can be very large)
-            zone_path.unlink()
+            # Note: Parser may have already deleted the file during cleanup_ram_drive()
+            if zone_path.exists():
+                zone_path.unlink()
             
             # Get previous snapshot
             previous_domains = await self.get_previous_domains(tld)

backend/app/services/zone_file.py

@@ -181,88 +181,65 @@ class ZoneFileService:
         current: set[str]
     ) -> list[dict]:
         """
-        Find dropped domains and verify they are ACTUALLY available before storing.
+        Find dropped domains and store them directly.
         
-        Zone file drops are often immediately re-registered by drop-catching services,
+        NOTE: We do NOT verify availability via RDAP here to avoid rate limits/bans.
-        so we must verify availability before storing to avoid showing unavailable domains.
+        Zone file diff is already a reliable signal that the domain was dropped.
         """
-        from app.services.domain_checker import domain_checker
-        
         dropped = previous - current
         
         if not dropped:
             logger.info(f"No dropped domains found for .{tld}")
             return []
         
-        logger.info(f"Found {len(dropped)} potential drops for .{tld}, verifying availability...")
+        logger.info(f"Found {len(dropped):,} dropped domains for .{tld}, saving to database...")
         
         today = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
         
-        # Filter to valuable domains first (short, no numbers, no hyphens)
+        # Store all drops - no RDAP verification (prevents bans!)
-        # This reduces the number of availability checks needed
-        valuable_drops = [
-            name for name in dropped
-            if len(name) <= 10 and not name.isdigit() and '-' not in name
-        ]
-        
-        # Also include some longer domains (up to 500 total)
-        other_drops = [
-            name for name in dropped
-            if name not in valuable_drops and len(name) <= 15
-        ][:max(0, 500 - len(valuable_drops))]
-        
-        candidates = valuable_drops + other_drops
-        logger.info(f"Checking availability of {len(candidates)} candidates (of {len(dropped)} total drops)")
-        
-        # Verify availability and only store truly available domains
         dropped_records = []
-        available_count = 0
+        batch_size = 1000
-        checked_count = 0
+        dropped_list = list(dropped)
         
-        for i, name in enumerate(candidates):
+        for i in range(0, len(dropped_list), batch_size):
-            full_domain = f"{name}.{tld}"
+            batch = dropped_list[i:i + batch_size]
             
-            try:
+            for name in batch:
-                # Quick DNS check
+                try:
-                result = await domain_checker.check_domain(full_domain)
-                checked_count += 1
-                
-                if result.is_available:
-                    available_count += 1
                     record = DroppedDomain(
-                        domain=full_domain,
+                        domain=name,  # Just the name, not full domain!
                         tld=tld,
                         dropped_date=today,
                         length=len(name),
                         is_numeric=name.isdigit(),
-                        has_hyphen='-' in name
+                        has_hyphen='-' in name,
+                        availability_status='unknown'
                     )
                     db.add(record)
                     dropped_records.append({
-                        "domain": full_domain,
+                        "domain": f"{name}.{tld}",
                         "length": len(name),
-                        "is_numeric": name.isdigit(),
-                        "has_hyphen": '-' in name
                     })
-                
+                except Exception:
-                # Progress log every 100 domains
+                    # Duplicate or other error - skip
-                if (i + 1) % 100 == 0:
+                    pass
-                    logger.info(f"Verified {i + 1}/{len(candidates)}: {available_count} available so far")
+            
-                
+            # Commit batch
-                # Small delay to avoid rate limiting
+            try:
-                if i % 20 == 0:
+                await db.commit()
-                    await asyncio.sleep(0.1)
+            except Exception:
-                    
+                await db.rollback()
-            except Exception as e:
+            
-                logger.warning(f"Error checking {full_domain}: {e}")
+            if (i + batch_size) % 5000 == 0:
+                logger.info(f"Saved {min(i + batch_size, len(dropped_list)):,}/{len(dropped_list):,} drops")
         
-        await db.commit()
+        # Final commit
+        try:
+            await db.commit()
+        except Exception:
+            await db.rollback()
         
-        logger.info(
+        logger.info(f"Zone drops for .{tld}: {len(dropped_records):,} saved (verification pending)")
-            f"Zone file drops for .{tld}: "
-            f"{checked_count} verified, {available_count} actually available, "
-            f"{len(dropped_records)} stored"
-        )
         
         return dropped_records
     

backend/app/services/zone_file_parser.py

@@ -44,16 +44,34 @@ def get_optimal_workers() -> int:
 
 def get_ram_drive_path() -> Optional[Path]:
     """
-    Get path to RAM drive if available.
+    Get path for temporary zone file processing.
-    Linux: /dev/shm (typically 50% of RAM)
+    
-    macOS: /tmp is often memory-backed
+    Priority:
+    1. CZDS_DATA_DIR environment variable (persistent storage)
+    2. /data/czds (Docker volume mount)
+    3. /tmp fallback
+    
+    Note: We avoid /dev/shm in Docker as it's typically limited to 64MB.
+    With 1.7TB disk and NVMe, disk-based processing is fast enough.
     """
-    # Linux RAM drive
+    from app.config import get_settings
-    if os.path.exists("/dev/shm"):
+    
-        shm_path = Path("/dev/shm/pounce_zones")
+    # Use configured data directory (mounted volume)
+    settings = get_settings()
+    if settings.czds_data_dir:
+        data_path = Path(settings.czds_data_dir) / "tmp"
         try:
-            shm_path.mkdir(parents=True, exist_ok=True)
+            data_path.mkdir(parents=True, exist_ok=True)
-            return shm_path
+            return data_path
+        except PermissionError:
+            pass
+    
+    # Docker volume mount
+    if os.path.exists("/data/czds"):
+        data_path = Path("/data/czds/tmp")
+        try:
+            data_path.mkdir(parents=True, exist_ok=True)
+            return data_path
         except PermissionError:
             pass
     

frontend/Dockerfile

@@ -15,7 +15,9 @@ COPY . .
 
 # Build arguments
 ARG NEXT_PUBLIC_API_URL
+ARG BACKEND_URL
 ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL}
+ENV BACKEND_URL=${BACKEND_URL}
 ENV NODE_OPTIONS="--max-old-space-size=2048"
 ENV NEXT_TELEMETRY_DISABLED=1
 

frontend/src/components/hunt/DropsTab.tsx

@@ -559,12 +559,41 @@ export function DropsTab({ showToast }: DropsTabProps) {
                 const isTrackingThis = trackingDrop === item.id
                 const status = item.availability_status || 'unknown'
                 
-                // Simplified status display config
+                // Status display config with better labels
+                const countdown = item.deletion_date ? formatCountdown(item.deletion_date) : null
                 const statusConfig = {
-                  available: { label: 'Available', color: 'text-accent', bg: 'bg-accent/10', border: 'border-accent/30', icon: CheckCircle2 },
+                  available: { 
-                  dropping_soon: { label: 'Dropping Soon', color: 'text-amber-400', bg: 'bg-amber-400/10', border: 'border-amber-400/30', icon: Clock },
+                    label: 'Available Now', 
-                  taken: { label: 'Taken', color: 'text-rose-400', bg: 'bg-rose-400/10', border: 'border-rose-400/30', icon: Ban },
+                    color: 'text-accent', 
-                  unknown: { label: 'Check', color: 'text-white/50', bg: 'bg-white/5', border: 'border-white/20', icon: Search },
+                    bg: 'bg-accent/10', 
+                    border: 'border-accent/30', 
+                    icon: CheckCircle2,
+                    showBuy: true,
+                  },
+                  dropping_soon: { 
+                    label: countdown ? `In Transition • ${countdown}` : 'In Transition', 
+                    color: 'text-amber-400', 
+                    bg: 'bg-amber-400/10', 
+                    border: 'border-amber-400/30', 
+                    icon: Clock,
+                    showBuy: false,
+                  },
+                  taken: { 
+                    label: 'Re-registered', 
+                    color: 'text-rose-400/60', 
+                    bg: 'bg-rose-400/5', 
+                    border: 'border-rose-400/20', 
+                    icon: Ban,
+                    showBuy: false,
+                  },
+                  unknown: { 
+                    label: 'Check Status', 
+                    color: 'text-white/50', 
+                    bg: 'bg-white/5', 
+                    border: 'border-white/20', 
+                    icon: Search,
+                    showBuy: false,
+                  },
                 }[status]
                 
                 const StatusIcon = statusConfig.icon
@@ -594,14 +623,12 @@ export function DropsTab({ showToast }: DropsTabProps) {
                               onClick={() => checkStatus(item.id, fullDomain)}
                               disabled={isChecking}
                               className={clsx(
-                                "text-[10px] font-mono font-bold px-2.5 py-1 border flex items-center gap-1",
+                                "text-[10px] font-mono font-bold px-2.5 py-1 border flex items-center gap-1.5",
                                 statusConfig.color, statusConfig.bg, statusConfig.border
                               )}
                             >
                               {isChecking ? <Loader2 className="w-3 h-3 animate-spin" /> : <StatusIcon className="w-3 h-3" />}
-                              {status === 'dropping_soon' && item.deletion_date 
+                              {statusConfig.label}
-                                ? formatCountdown(item.deletion_date) 
-                                : statusConfig.label}
                             </button>
                           </div>
                         </div>
@@ -630,10 +657,15 @@ export function DropsTab({ showToast }: DropsTabProps) {
                             Buy Now
                           </a>
                         ) : status === 'dropping_soon' ? (
-                          <span className="flex-1 h-12 border border-amber-400/30 text-amber-400 bg-amber-400/5 text-xs font-bold uppercase tracking-widest flex items-center justify-center gap-2">
+                          <div className="flex-1 h-12 border border-amber-400/30 text-amber-400 bg-amber-400/5 text-xs font-bold uppercase tracking-widest flex flex-col items-center justify-center">
-                            <Clock className="w-4 h-4" />
+                            <span className="flex items-center gap-1.5">
-                            Dropping Soon
+                              <Clock className="w-3 h-3" />
-                          </span>
+                              In Transition
+                            </span>
+                            {countdown && (
+                              <span className="text-[9px] text-amber-400/70 font-mono">{countdown} until drop</span>
+                            )}
+                          </div>
                         ) : status === 'taken' ? (
                           <span className="flex-1 h-12 border border-rose-400/20 text-rose-400/60 text-xs font-bold uppercase tracking-widest flex items-center justify-center gap-2 bg-rose-400/5">
                             <Ban className="w-4 h-4" />
@@ -688,15 +720,13 @@ export function DropsTab({ showToast }: DropsTabProps) {
                           onClick={() => checkStatus(item.id, fullDomain)}
                           disabled={isChecking}
                           className={clsx(
-                            "text-[10px] font-mono font-bold px-2.5 py-1 border inline-flex items-center gap-1.5 transition-all hover:opacity-80",
+                            "text-[10px] font-mono font-bold px-2.5 py-1.5 border inline-flex items-center gap-1.5 transition-all hover:opacity-80",
                             statusConfig.color, statusConfig.bg, statusConfig.border
                           )}
                           title="Click to check real-time status"
                         >
                           {isChecking ? <Loader2 className="w-3 h-3 animate-spin" /> : <StatusIcon className="w-3 h-3" />}
-                          {status === 'dropping_soon' && item.deletion_date 
+                          <span className="max-w-[100px] truncate">{statusConfig.label}</span>
-                            ? formatCountdown(item.deletion_date) 
-                            : statusConfig.label}
                         </button>
                       </div>
 
@@ -736,13 +766,18 @@ export function DropsTab({ showToast }: DropsTabProps) {
                             title="Register this domain now!"
                           >
                             <Zap className="w-3 h-3" />
-                            Buy Now
+                            Buy
                           </a>
                         ) : status === 'dropping_soon' ? (
-                          <span className="h-9 px-3 text-amber-400 text-[10px] font-bold uppercase tracking-widest flex items-center gap-1.5 border border-amber-400/30 bg-amber-400/5">
+                          <button
-                            <Clock className="w-3 h-3" />
+                            onClick={() => trackDrop(item.id, fullDomain)}
-                            Soon
+                            disabled={isTrackingThis}
-                          </span>
+                            className="h-9 px-3 text-amber-400 text-[10px] font-bold uppercase tracking-widest flex items-center gap-1.5 border border-amber-400/30 bg-amber-400/5 hover:bg-amber-400/10 transition-all"
+                            title={countdown ? `Drops in ${countdown} - Track to get notified!` : 'Track to get notified when available'}
+                          >
+                            {isTrackingThis ? <Loader2 className="w-3 h-3 animate-spin" /> : <Eye className="w-3 h-3" />}
+                            Track
+                          </button>
                         ) : status === 'taken' ? (
                           <span className="h-9 px-3 text-rose-400/50 text-[10px] font-bold uppercase tracking-widest flex items-center gap-1.5 border border-rose-400/20 bg-rose-400/5">
                             <Ban className="w-3 h-3" />