f0cc69ac95
Major changes: - Add TLD price scraper with Porkbun API (886+ TLDs, no API key needed) - Fix .ch domain checker using rdap.nic.ch custom RDAP - Integrate database for TLD price history tracking - Add admin endpoints for manual scrape and stats - Extend scheduler with daily TLD price scrape job (03:00 UTC) - Update API to use DB data with static fallback - Update README with complete documentation New files: - backend/app/services/tld_scraper/ (scraper package) - TLD_TRACKING_PLAN.md (implementation plan) API changes: - POST /admin/scrape-tld-prices - trigger manual scrape - GET /admin/tld-prices/stats - database statistics - GET /tld-prices/overview now uses DB data
41 lines
1.2 KiB
Python
41 lines
1.2 KiB
Python
"""TLD-List.com scraper (placeholder - site blocks automated requests)."""
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
from app.services.tld_scraper.base import BaseTLDScraper, TLDPriceData, ScraperError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TLDListScraper(BaseTLDScraper):
|
|
"""
|
|
Scraper for TLD-List.com.
|
|
|
|
NOTE: TLD-List.com currently blocks automated requests (403).
|
|
This scraper is a placeholder for future implementation if they
|
|
open up access or we find a workaround.
|
|
|
|
For now, use PorkbunScraper as the primary source.
|
|
"""
|
|
|
|
name = "tld-list"
|
|
base_url = "https://tld-list.com"
|
|
|
|
async def scrape(self) -> list[TLDPriceData]:
|
|
"""
|
|
Attempt to scrape TLD-List.com.
|
|
|
|
Currently returns empty list as the site blocks automated requests.
|
|
"""
|
|
logger.warning(
|
|
"TLD-List.com blocks automated requests. "
|
|
"Use PorkbunScraper as primary source instead."
|
|
)
|
|
return []
|
|
|
|
async def health_check(self) -> bool:
|
|
"""Check if TLD-List.com is accessible."""
|
|
# Currently always returns False due to blocking
|
|
return False
|
|
|