Add retry logic (3 attempts with backoff) to CZDS zone downloads
Some checks failed
CI / Frontend Lint & Type Check (push) Has been cancelled
CI / Frontend Build (push) Has been cancelled
CI / Backend Lint (push) Has been cancelled
CI / Backend Tests (push) Has been cancelled
CI / Docker Build (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
Deploy / Build & Push Images (push) Has been cancelled
Deploy / Deploy to Server (push) Has been cancelled
Deploy / Notify (push) Has been cancelled
Some checks failed
CI / Frontend Lint & Type Check (push) Has been cancelled
CI / Frontend Build (push) Has been cancelled
CI / Backend Lint (push) Has been cancelled
CI / Backend Tests (push) Has been cancelled
CI / Docker Build (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
Deploy / Build & Push Images (push) Has been cancelled
Deploy / Deploy to Server (push) Has been cancelled
Deploy / Notify (push) Has been cancelled
Large zone files (100-200MB) were failing due to connection interruptions. Now retries up to 3 times with 30s/60s/90s backoff between attempts.
This commit is contained in:
@ -92,8 +92,11 @@ async def get_db_session():
|
|||||||
return async_session()
|
return async_session()
|
||||||
|
|
||||||
|
|
||||||
def download_czds_zone(tld: str) -> Optional[Path]:
|
def download_czds_zone(tld: str, max_retries: int = 3) -> Optional[Path]:
|
||||||
"""Download a single CZDS zone file using pyCZDS"""
|
"""Download a single CZDS zone file using pyCZDS with retry logic"""
|
||||||
|
import time
|
||||||
|
|
||||||
|
for attempt in range(max_retries):
|
||||||
try:
|
try:
|
||||||
from pyczds.client import CZDSClient
|
from pyczds.client import CZDSClient
|
||||||
|
|
||||||
@ -129,7 +132,7 @@ def download_czds_zone(tld: str) -> Optional[Path]:
|
|||||||
logger.warning(f"No access to .{tld} zone file")
|
logger.warning(f"No access to .{tld} zone file")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
logger.info(f"Downloading .{tld} from CZDS...")
|
logger.info(f"Downloading .{tld} from CZDS... (attempt {attempt + 1}/{max_retries})")
|
||||||
result = client.get_zonefile(target_url, download_dir=str(CZDS_DIR))
|
result = client.get_zonefile(target_url, download_dir=str(CZDS_DIR))
|
||||||
|
|
||||||
# Find the downloaded file
|
# Find the downloaded file
|
||||||
@ -144,7 +147,15 @@ def download_czds_zone(tld: str) -> Optional[Path]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"CZDS download failed for .{tld}: {e}")
|
logger.warning(f"CZDS download attempt {attempt + 1} failed for .{tld}: {e}")
|
||||||
|
if attempt < max_retries - 1:
|
||||||
|
wait_time = (attempt + 1) * 30 # 30s, 60s, 90s backoff
|
||||||
|
logger.info(f"Retrying in {wait_time}s...")
|
||||||
|
time.sleep(wait_time)
|
||||||
|
else:
|
||||||
|
logger.error(f"CZDS download failed for .{tld} after {max_retries} attempts")
|
||||||
|
return None
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user