Add retry logic (3 attempts with backoff) to CZDS zone downloads
Some checks failed
CI / Frontend Lint & Type Check (push) Has been cancelled
CI / Frontend Build (push) Has been cancelled
CI / Backend Lint (push) Has been cancelled
CI / Backend Tests (push) Has been cancelled
CI / Docker Build (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
Deploy / Build & Push Images (push) Has been cancelled
Deploy / Deploy to Server (push) Has been cancelled
Deploy / Notify (push) Has been cancelled
Some checks failed
CI / Frontend Lint & Type Check (push) Has been cancelled
CI / Frontend Build (push) Has been cancelled
CI / Backend Lint (push) Has been cancelled
CI / Backend Tests (push) Has been cancelled
CI / Docker Build (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
Deploy / Build & Push Images (push) Has been cancelled
Deploy / Deploy to Server (push) Has been cancelled
Deploy / Notify (push) Has been cancelled
Large zone files (100-200MB) were failing due to connection interruptions. Now retries up to 3 times with 30s/60s/90s backoff between attempts.
This commit is contained in:
@ -92,60 +92,71 @@ async def get_db_session():
|
||||
return async_session()
|
||||
|
||||
|
||||
def download_czds_zone(tld: str) -> Optional[Path]:
|
||||
"""Download a single CZDS zone file using pyCZDS"""
|
||||
try:
|
||||
from pyczds.client import CZDSClient
|
||||
def download_czds_zone(tld: str, max_retries: int = 3) -> Optional[Path]:
|
||||
"""Download a single CZDS zone file using pyCZDS with retry logic"""
|
||||
import time
|
||||
|
||||
# Read credentials from .env
|
||||
env_file = Path(__file__).parent.parent / ".env"
|
||||
if not env_file.exists():
|
||||
env_file = Path("/home/user/pounce/backend/.env")
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
from pyczds.client import CZDSClient
|
||||
|
||||
env_content = env_file.read_text()
|
||||
username = password = None
|
||||
# Read credentials from .env
|
||||
env_file = Path(__file__).parent.parent / ".env"
|
||||
if not env_file.exists():
|
||||
env_file = Path("/home/user/pounce/backend/.env")
|
||||
|
||||
for line in env_content.splitlines():
|
||||
if line.startswith("CZDS_USERNAME="):
|
||||
username = line.split("=", 1)[1].strip()
|
||||
elif line.startswith("CZDS_PASSWORD="):
|
||||
password = line.split("=", 1)[1].strip()
|
||||
env_content = env_file.read_text()
|
||||
username = password = None
|
||||
|
||||
for line in env_content.splitlines():
|
||||
if line.startswith("CZDS_USERNAME="):
|
||||
username = line.split("=", 1)[1].strip()
|
||||
elif line.startswith("CZDS_PASSWORD="):
|
||||
password = line.split("=", 1)[1].strip()
|
||||
|
||||
if not username or not password:
|
||||
logger.error(f"CZDS credentials not found in .env")
|
||||
return None
|
||||
|
||||
client = CZDSClient(username, password)
|
||||
urls = client.get_zonefiles_list()
|
||||
|
||||
# Find URL for this TLD
|
||||
target_url = None
|
||||
for url in urls:
|
||||
if f"{tld}.zone" in url or f"/{tld}." in url:
|
||||
target_url = url
|
||||
break
|
||||
|
||||
if not target_url:
|
||||
logger.warning(f"No access to .{tld} zone file")
|
||||
return None
|
||||
|
||||
logger.info(f"Downloading .{tld} from CZDS... (attempt {attempt + 1}/{max_retries})")
|
||||
result = client.get_zonefile(target_url, download_dir=str(CZDS_DIR))
|
||||
|
||||
# Find the downloaded file
|
||||
gz_file = CZDS_DIR / f"{tld}.txt.gz"
|
||||
if gz_file.exists():
|
||||
return gz_file
|
||||
|
||||
# Try alternative naming
|
||||
for f in CZDS_DIR.glob(f"*{tld}*.gz"):
|
||||
return f
|
||||
|
||||
if not username or not password:
|
||||
logger.error(f"CZDS credentials not found in .env")
|
||||
return None
|
||||
|
||||
client = CZDSClient(username, password)
|
||||
urls = client.get_zonefiles_list()
|
||||
except Exception as e:
|
||||
logger.warning(f"CZDS download attempt {attempt + 1} failed for .{tld}: {e}")
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = (attempt + 1) * 30 # 30s, 60s, 90s backoff
|
||||
logger.info(f"Retrying in {wait_time}s...")
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
logger.error(f"CZDS download failed for .{tld} after {max_retries} attempts")
|
||||
return None
|
||||
|
||||
# Find URL for this TLD
|
||||
target_url = None
|
||||
for url in urls:
|
||||
if f"{tld}.zone" in url or f"/{tld}." in url:
|
||||
target_url = url
|
||||
break
|
||||
|
||||
if not target_url:
|
||||
logger.warning(f"No access to .{tld} zone file")
|
||||
return None
|
||||
|
||||
logger.info(f"Downloading .{tld} from CZDS...")
|
||||
result = client.get_zonefile(target_url, download_dir=str(CZDS_DIR))
|
||||
|
||||
# Find the downloaded file
|
||||
gz_file = CZDS_DIR / f"{tld}.txt.gz"
|
||||
if gz_file.exists():
|
||||
return gz_file
|
||||
|
||||
# Try alternative naming
|
||||
for f in CZDS_DIR.glob(f"*{tld}*.gz"):
|
||||
return f
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"CZDS download failed for .{tld}: {e}")
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def download_switch_zone(tld: str) -> Optional[Path]:
|
||||
|
||||
Reference in New Issue
Block a user