pounce/backend/app/api/tld_prices.py
yves.gugger c2bb48db19 fix: Ensure 100% price consistency across all TLD pages
ROOT CAUSE:
- Overview API prioritized database data (only Porkbun prices)
- Compare API prioritized static data (5+ registrars)
- This caused price mismatches between Overview table and Detail page

SOLUTION:
- Changed Overview API to prioritize static data for TLDs with rich
  multi-registrar pricing (like Compare API does)
- Database data now used only for TLDs NOT in static data
- This ensures all APIs return identical prices for same TLDs

VERIFICATION:
- .com: Overview=$10.75, Compare=$10.75 ✓
- .io: Overview=$32.33, Compare=$32.33 ✓
- .ai: Overview=$71.63, Compare=$71.63 ✓
- .xyz: Overview=$6.86, Compare=$6.86 ✓

DATA FLOW (now consistent):
1. Static TLDs (18): Use rich multi-registrar data
2. DB-only TLDs (869): Use Porkbun scraped data
3. Total: 887 TLDs with consistent pricing
2025-12-08 12:03:08 +01:00

741 lines
27 KiB
Python

"""TLD Price API endpoints with real market data from database + static fallback."""
from datetime import datetime, timedelta
from typing import Optional, List
from fastapi import APIRouter, Query, HTTPException
from pydantic import BaseModel
from sqlalchemy import select, func, desc
from app.api.deps import Database
from app.models.tld_price import TLDPrice, TLDInfo
router = APIRouter()
async def get_db_prices(db, tld: str = None) -> dict:
"""Get latest prices from database."""
# Subquery to get latest record per TLD/registrar
subq = (
select(
TLDPrice.tld,
TLDPrice.registrar,
func.max(TLDPrice.recorded_at).label("max_date")
)
.group_by(TLDPrice.tld, TLDPrice.registrar)
.subquery()
)
query = (
select(TLDPrice)
.join(
subq,
(TLDPrice.tld == subq.c.tld) &
(TLDPrice.registrar == subq.c.registrar) &
(TLDPrice.recorded_at == subq.c.max_date)
)
)
if tld:
query = query.where(TLDPrice.tld == tld.lower().lstrip("."))
result = await db.execute(query)
prices = result.scalars().all()
# Group by TLD
tld_prices = {}
for p in prices:
if p.tld not in tld_prices:
tld_prices[p.tld] = {
"registrars": {},
"prices": []
}
tld_prices[p.tld]["registrars"][p.registrar] = {
"register": p.registration_price,
"renew": p.renewal_price or p.registration_price,
"transfer": p.transfer_price or p.registration_price,
}
tld_prices[p.tld]["prices"].append(p.registration_price)
return tld_prices
async def get_db_price_count(db) -> int:
"""Get count of TLDs in database."""
result = await db.execute(select(func.count(func.distinct(TLDPrice.tld))))
return result.scalar() or 0
# Real TLD price data based on current market research (December 2024)
# Prices in USD, sourced from major registrars: Namecheap, Cloudflare, Porkbun, Google Domains
TLD_DATA = {
# Generic TLDs
"com": {
"type": "generic",
"description": "Commercial - Most popular TLD worldwide",
"registry": "Verisign",
"introduced": 1985,
"registrars": {
"Cloudflare": {"register": 10.44, "renew": 10.44, "transfer": 10.44},
"Namecheap": {"register": 9.58, "renew": 14.58, "transfer": 9.48},
"Porkbun": {"register": 9.73, "renew": 10.91, "transfer": 9.73},
"Google Domains": {"register": 12.00, "renew": 12.00, "transfer": 12.00},
"GoDaddy": {"register": 11.99, "renew": 22.99, "transfer": 11.99},
},
"trend": "stable",
"trend_reason": "Stable registry pricing, slight increase in 2024",
},
"net": {
"type": "generic",
"description": "Network - Popular for tech and infrastructure",
"registry": "Verisign",
"introduced": 1985,
"registrars": {
"Cloudflare": {"register": 11.94, "renew": 11.94, "transfer": 11.94},
"Namecheap": {"register": 12.88, "renew": 16.88, "transfer": 12.78},
"Porkbun": {"register": 11.52, "renew": 12.77, "transfer": 11.52},
"Google Domains": {"register": 15.00, "renew": 15.00, "transfer": 15.00},
},
"trend": "stable",
"trend_reason": "Mature market, predictable pricing",
},
"org": {
"type": "generic",
"description": "Organization - Non-profits and communities",
"registry": "Public Interest Registry",
"introduced": 1985,
"registrars": {
"Cloudflare": {"register": 10.11, "renew": 10.11, "transfer": 10.11},
"Namecheap": {"register": 10.98, "renew": 15.98, "transfer": 10.88},
"Porkbun": {"register": 10.19, "renew": 11.44, "transfer": 10.19},
"Google Domains": {"register": 12.00, "renew": 12.00, "transfer": 12.00},
},
"trend": "stable",
"trend_reason": "Non-profit pricing commitment",
},
"io": {
"type": "ccTLD",
"description": "British Indian Ocean Territory - Popular for tech startups",
"registry": "Internet Computer Bureau",
"introduced": 1997,
"registrars": {
"Cloudflare": {"register": 33.98, "renew": 33.98, "transfer": 33.98},
"Namecheap": {"register": 32.88, "renew": 38.88, "transfer": 32.78},
"Porkbun": {"register": 32.47, "renew": 36.47, "transfer": 32.47},
"Google Domains": {"register": 30.00, "renew": 30.00, "transfer": 30.00},
},
"trend": "up",
"trend_reason": "High demand from tech/startup sector, +8% in 2024",
},
"co": {
"type": "ccTLD",
"description": "Colombia - Popular as 'Company' alternative",
"registry": ".CO Internet S.A.S",
"introduced": 1991,
"registrars": {
"Cloudflare": {"register": 11.02, "renew": 11.02, "transfer": 11.02},
"Namecheap": {"register": 11.98, "renew": 29.98, "transfer": 11.88},
"Porkbun": {"register": 10.77, "renew": 27.03, "transfer": 10.77},
},
"trend": "stable",
"trend_reason": "Steady adoption as .com alternative",
},
"ai": {
"type": "ccTLD",
"description": "Anguilla - Extremely popular for AI companies",
"registry": "Government of Anguilla",
"introduced": 1995,
"registrars": {
"Namecheap": {"register": 74.98, "renew": 74.98, "transfer": 74.88},
"Porkbun": {"register": 59.93, "renew": 79.93, "transfer": 59.93},
"GoDaddy": {"register": 79.99, "renew": 99.99, "transfer": 79.99},
},
"trend": "up",
"trend_reason": "AI boom driving massive demand, +35% since 2023",
},
"dev": {
"type": "generic",
"description": "Developer - For software developers",
"registry": "Google",
"introduced": 2019,
"registrars": {
"Cloudflare": {"register": 11.94, "renew": 11.94, "transfer": 11.94},
"Namecheap": {"register": 14.98, "renew": 17.98, "transfer": 14.88},
"Porkbun": {"register": 13.33, "renew": 15.65, "transfer": 13.33},
"Google Domains": {"register": 14.00, "renew": 14.00, "transfer": 14.00},
},
"trend": "stable",
"trend_reason": "Growing developer adoption",
},
"app": {
"type": "generic",
"description": "Application - For apps and software",
"registry": "Google",
"introduced": 2018,
"registrars": {
"Cloudflare": {"register": 14.94, "renew": 14.94, "transfer": 14.94},
"Namecheap": {"register": 16.98, "renew": 19.98, "transfer": 16.88},
"Porkbun": {"register": 15.45, "renew": 17.77, "transfer": 15.45},
"Google Domains": {"register": 16.00, "renew": 16.00, "transfer": 16.00},
},
"trend": "stable",
"trend_reason": "Steady growth in app ecosystem",
},
"xyz": {
"type": "generic",
"description": "XYZ - Generation XYZ, affordable option",
"registry": "XYZ.COM LLC",
"introduced": 2014,
"registrars": {
"Cloudflare": {"register": 10.44, "renew": 10.44, "transfer": 10.44},
"Namecheap": {"register": 1.00, "renew": 13.98, "transfer": 1.00}, # Promo
"Porkbun": {"register": 9.15, "renew": 10.40, "transfer": 9.15},
},
"trend": "down",
"trend_reason": "Heavy promotional pricing competition",
},
"tech": {
"type": "generic",
"description": "Technology - For tech companies",
"registry": "Radix",
"introduced": 2015,
"registrars": {
"Namecheap": {"register": 5.98, "renew": 49.98, "transfer": 5.88},
"Porkbun": {"register": 4.79, "renew": 44.52, "transfer": 4.79},
"GoDaddy": {"register": 4.99, "renew": 54.99, "transfer": 4.99},
},
"trend": "stable",
"trend_reason": "Low intro pricing, high renewals",
},
"online": {
"type": "generic",
"description": "Online - For online presence",
"registry": "Radix",
"introduced": 2015,
"registrars": {
"Namecheap": {"register": 2.98, "renew": 39.98, "transfer": 2.88},
"Porkbun": {"register": 2.59, "renew": 34.22, "transfer": 2.59},
},
"trend": "stable",
"trend_reason": "Budget-friendly option",
},
"store": {
"type": "generic",
"description": "Store - For e-commerce",
"registry": "Radix",
"introduced": 2016,
"registrars": {
"Namecheap": {"register": 3.88, "renew": 56.88, "transfer": 3.78},
"Porkbun": {"register": 3.28, "renew": 48.95, "transfer": 3.28},
},
"trend": "stable",
"trend_reason": "E-commerce growth sector",
},
"me": {
"type": "ccTLD",
"description": "Montenegro - Popular for personal branding",
"registry": "doMEn",
"introduced": 2007,
"registrars": {
"Cloudflare": {"register": 14.94, "renew": 14.94, "transfer": 14.94},
"Namecheap": {"register": 5.98, "renew": 19.98, "transfer": 5.88},
"Porkbun": {"register": 5.15, "renew": 17.45, "transfer": 5.15},
},
"trend": "stable",
"trend_reason": "Personal branding market",
},
"info": {
"type": "generic",
"description": "Information - For informational sites",
"registry": "Afilias",
"introduced": 2001,
"registrars": {
"Cloudflare": {"register": 11.44, "renew": 11.44, "transfer": 11.44},
"Namecheap": {"register": 4.98, "renew": 22.98, "transfer": 4.88},
"Porkbun": {"register": 4.24, "renew": 19.45, "transfer": 4.24},
},
"trend": "down",
"trend_reason": "Declining popularity vs newer TLDs",
},
"biz": {
"type": "generic",
"description": "Business - Alternative to .com",
"registry": "GoDaddy Registry",
"introduced": 2001,
"registrars": {
"Cloudflare": {"register": 13.44, "renew": 13.44, "transfer": 13.44},
"Namecheap": {"register": 14.98, "renew": 20.98, "transfer": 14.88},
"Porkbun": {"register": 13.96, "renew": 18.45, "transfer": 13.96},
},
"trend": "stable",
"trend_reason": "Mature but declining market",
},
"ch": {
"type": "ccTLD",
"description": "Switzerland - Swiss domains",
"registry": "SWITCH",
"introduced": 1987,
"registrars": {
"Infomaniak": {"register": 9.80, "renew": 9.80, "transfer": 9.80},
"Hostpoint": {"register": 11.90, "renew": 11.90, "transfer": 0.00},
"Namecheap": {"register": 12.98, "renew": 12.98, "transfer": 12.88},
},
"trend": "stable",
"trend_reason": "Stable Swiss market",
},
"de": {
"type": "ccTLD",
"description": "Germany - German domains",
"registry": "DENIC",
"introduced": 1986,
"registrars": {
"United Domains": {"register": 9.90, "renew": 9.90, "transfer": 9.90},
"IONOS": {"register": 0.99, "renew": 12.00, "transfer": 0.00},
"Namecheap": {"register": 9.98, "renew": 11.98, "transfer": 9.88},
},
"trend": "stable",
"trend_reason": "Largest ccTLD in Europe",
},
"uk": {
"type": "ccTLD",
"description": "United Kingdom - British domains",
"registry": "Nominet",
"introduced": 1985,
"registrars": {
"Namecheap": {"register": 8.88, "renew": 10.98, "transfer": 8.78},
"Porkbun": {"register": 8.45, "renew": 9.73, "transfer": 8.45},
"123-reg": {"register": 9.99, "renew": 11.99, "transfer": 9.99},
},
"trend": "stable",
"trend_reason": "Strong local market",
},
}
def get_avg_price(tld_data: dict) -> float:
"""Calculate average registration price across registrars."""
prices = [r["register"] for r in tld_data["registrars"].values()]
return round(sum(prices) / len(prices), 2)
def get_min_price(tld_data: dict) -> float:
"""Get minimum registration price."""
return min(r["register"] for r in tld_data["registrars"].values())
def get_max_price(tld_data: dict) -> float:
"""Get maximum registration price."""
return max(r["register"] for r in tld_data["registrars"].values())
# Top TLDs by popularity (based on actual domain registration volumes)
TOP_TLDS_BY_POPULARITY = [
"com", "net", "org", "de", "uk", "cn", "ru", "nl", "br", "au",
"io", "co", "ai", "app", "dev", "xyz", "online", "site", "tech", "store",
"info", "biz", "me", "tv", "cc", "eu", "fr", "it", "es", "pl",
"ch", "at", "be", "se", "no", "dk", "fi", "ie", "nz", "in",
]
@router.get("/overview")
async def get_tld_overview(
db: Database,
limit: int = Query(25, ge=1, le=100),
offset: int = Query(0, ge=0),
sort_by: str = Query("popularity", enum=["popularity", "price_asc", "price_desc", "name"]),
search: str = Query(None, description="Search TLDs by name"),
source: str = Query("auto", enum=["auto", "db", "static"]),
):
"""Get overview of TLDs with current pricing, pagination, and search.
Data source priority:
- For TLDs with rich static data (multiple registrars): Use static data for consistency
- For TLDs only in database: Use database data
- This ensures Overview and Compare/Detail pages show identical prices
Args:
limit: Number of results per page (default 25)
offset: Skip N results for pagination
search: Filter TLDs by name (e.g., "com", "io")
sort_by: Sort order - popularity (default), price_asc, price_desc, name
source: Data source - "auto" (best available), "db" (only DB), "static" (only static)
"""
tld_list = []
tld_seen = set()
data_source = "combined"
# FIRST: Add all static data TLDs (these have rich multi-registrar data)
# This ensures consistency with /compare endpoint which also uses static data first
if source in ["auto", "static"]:
for tld, data in TLD_DATA.items():
tld_list.append({
"tld": tld,
"type": data["type"],
"description": data["description"],
"avg_registration_price": get_avg_price(data),
"min_registration_price": get_min_price(data),
"max_registration_price": get_max_price(data),
"registrar_count": len(data["registrars"]),
"trend": data["trend"],
"popularity_rank": TOP_TLDS_BY_POPULARITY.index(tld) if tld in TOP_TLDS_BY_POPULARITY else 999,
})
tld_seen.add(tld)
# SECOND: Add TLDs from database that are NOT in static data
# This adds the 800+ TLDs scraped from Porkbun
if source in ["auto", "db"]:
db_count = await get_db_price_count(db)
if db_count > 0:
db_prices = await get_db_prices(db)
for tld, data in db_prices.items():
if tld not in tld_seen: # Only add if not already from static
prices = data["prices"]
tld_list.append({
"tld": tld,
"type": guess_tld_type(tld),
"description": f".{tld} domain extension",
"avg_registration_price": round(sum(prices) / len(prices), 2),
"min_registration_price": min(prices),
"max_registration_price": max(prices),
"registrar_count": len(data["registrars"]),
"trend": "stable",
"popularity_rank": TOP_TLDS_BY_POPULARITY.index(tld) if tld in TOP_TLDS_BY_POPULARITY else 999,
})
tld_seen.add(tld)
# Determine source label
if source == "static":
data_source = "static"
elif source == "db":
data_source = "database"
else:
data_source = "combined"
# Apply search filter
if search:
search_lower = search.lower().lstrip(".")
tld_list = [t for t in tld_list if search_lower in t["tld"].lower()]
# Store total before pagination
total = len(tld_list)
# Sort
if sort_by == "popularity":
tld_list.sort(key=lambda x: (x["popularity_rank"], x["tld"]))
elif sort_by == "price_asc":
tld_list.sort(key=lambda x: x["avg_registration_price"])
elif sort_by == "price_desc":
tld_list.sort(key=lambda x: x["avg_registration_price"], reverse=True)
elif sort_by == "name":
tld_list.sort(key=lambda x: x["tld"])
# Apply pagination
paginated = tld_list[offset:offset + limit]
return {
"tlds": paginated,
"total": total,
"limit": limit,
"offset": offset,
"has_more": offset + limit < total,
"source": data_source,
}
def guess_tld_type(tld: str) -> str:
"""Guess TLD type based on pattern."""
if len(tld) == 2:
return "ccTLD"
if tld in {"com", "net", "org", "info", "biz"}:
return "generic"
return "gTLD"
@router.get("/trending")
async def get_trending_tlds(db: Database):
"""Get trending TLDs based on price changes."""
trending = []
for tld, data in TLD_DATA.items():
if data["trend"] in ["up", "down"]:
# Calculate approximate price change
price_change = 8.5 if data["trend"] == "up" else -5.2
if tld == "ai":
price_change = 35.0 # AI domains have seen massive increase
elif tld == "io":
price_change = 8.0
elif tld == "xyz":
price_change = -12.0
elif tld == "info":
price_change = -8.0
trending.append({
"tld": tld,
"reason": data["trend_reason"],
"price_change": price_change,
"current_price": get_avg_price(data),
})
# Sort by price change magnitude
trending.sort(key=lambda x: abs(x["price_change"]), reverse=True)
return {"trending": trending[:6]}
@router.get("/{tld}/history")
async def get_tld_price_history(
tld: str,
db: Database,
days: int = Query(90, ge=30, le=365),
):
"""Get price history for a specific TLD.
Returns real historical data from database if available,
otherwise generates simulated data based on current price.
"""
import math
tld_clean = tld.lower().lstrip(".")
# Get current price from database
db_prices = await get_db_prices(db, tld_clean)
current_price = 0
if db_prices and tld_clean in db_prices:
prices = db_prices[tld_clean]["prices"]
current_price = round(sum(prices) / len(prices), 2) if prices else 0
# Get static data for metadata and trend info
static_data = TLD_DATA.get(tld_clean, {})
# Determine trend and current price
if not current_price and static_data:
current_price = get_avg_price(static_data)
if not current_price:
raise HTTPException(status_code=404, detail=f"TLD '.{tld_clean}' not found")
# Get trend info
trend = static_data.get("trend", "stable")
trend_reason = static_data.get("trend_reason", "Price tracking available")
# Generate historical data (simulated for now, real when we have more scrapes)
history = []
current_date = datetime.utcnow()
# Calculate trend factor based on known trends
trend_factor = 1.0
if trend == "up":
trend_factor = 0.92 # Prices were ~8% lower
elif trend == "down":
trend_factor = 1.05 # Prices were ~5% higher
# Generate weekly data points
for i in range(days, -1, -7):
date = current_date - timedelta(days=i)
progress = 1 - (i / days)
if trend == "up":
price = current_price * (trend_factor + (1 - trend_factor) * progress)
elif trend == "down":
price = current_price * (trend_factor - (trend_factor - 1) * progress)
else:
# Add small fluctuation for stable prices
fluctuation = math.sin(i * 0.1) * 0.02
price = current_price * (1 + fluctuation)
history.append({
"date": date.strftime("%Y-%m-%d"),
"price": round(price, 2),
})
# Calculate price changes
price_7d_ago = history[-2]["price"] if len(history) >= 2 else current_price
price_30d_ago = history[-5]["price"] if len(history) >= 5 else current_price
price_90d_ago = history[0]["price"] if history else current_price
return {
"tld": tld_clean,
"type": static_data.get("type", guess_tld_type(tld_clean)),
"description": static_data.get("description", f".{tld_clean} domain extension"),
"registry": static_data.get("registry", "Unknown"),
"current_price": current_price,
"price_change_7d": round((current_price - price_7d_ago) / price_7d_ago * 100, 2) if price_7d_ago else 0,
"price_change_30d": round((current_price - price_30d_ago) / price_30d_ago * 100, 2) if price_30d_ago else 0,
"price_change_90d": round((current_price - price_90d_ago) / price_90d_ago * 100, 2) if price_90d_ago else 0,
"trend": trend,
"trend_reason": trend_reason,
"history": history,
"source": "simulated" if not static_data else "static",
}
def calculate_trend(history: list) -> str:
"""Calculate trend from price history."""
if len(history) < 2:
return "stable"
first_price = history[0]["price"]
last_price = history[-1]["price"]
if first_price == 0:
return "stable"
change_percent = (last_price - first_price) / first_price * 100
if change_percent > 5:
return "up"
elif change_percent < -5:
return "down"
return "stable"
@router.get("/{tld}/compare")
async def compare_tld_prices(
tld: str,
db: Database,
):
"""Compare prices across different registrars for a TLD."""
tld_clean = tld.lower().lstrip(".")
# Try static data first
if tld_clean in TLD_DATA:
data = TLD_DATA[tld_clean]
registrars = []
for name, prices in data["registrars"].items():
registrars.append({
"name": name,
"registration_price": prices["register"],
"renewal_price": prices["renew"],
"transfer_price": prices["transfer"],
})
registrars.sort(key=lambda x: x["registration_price"])
return {
"tld": tld_clean,
"type": data["type"],
"description": data["description"],
"registry": data.get("registry", "Unknown"),
"introduced": data.get("introduced"),
"registrars": registrars,
"cheapest_registrar": registrars[0]["name"],
"cheapest_price": registrars[0]["registration_price"],
"price_range": {
"min": get_min_price(data),
"max": get_max_price(data),
"avg": get_avg_price(data),
},
}
# Fall back to database
db_prices = await get_db_prices(db, tld_clean)
if not db_prices:
raise HTTPException(status_code=404, detail=f"TLD '.{tld_clean}' not found")
tld_data = db_prices[tld_clean]
registrars = [
{
"name": name,
"registration_price": prices["register"],
"renewal_price": prices["renew"],
"transfer_price": prices["transfer"],
}
for name, prices in tld_data["registrars"].items()
]
registrars.sort(key=lambda x: x["registration_price"])
prices = tld_data["prices"]
return {
"tld": tld_clean,
"type": guess_tld_type(tld_clean),
"description": f".{tld_clean} domain extension",
"registry": "Unknown",
"introduced": None,
"registrars": registrars,
"cheapest_registrar": registrars[0]["name"] if registrars else "N/A",
"cheapest_price": min(prices) if prices else 0,
"price_range": {
"min": min(prices) if prices else 0,
"max": max(prices) if prices else 0,
"avg": round(sum(prices) / len(prices), 2) if prices else 0,
},
}
@router.get("/{tld}")
async def get_tld_details(
tld: str,
db: Database,
):
"""Get complete details for a specific TLD."""
tld_clean = tld.lower().lstrip(".")
# Try static data first
if tld_clean in TLD_DATA:
data = TLD_DATA[tld_clean]
registrars = []
for name, prices in data["registrars"].items():
registrars.append({
"name": name,
"registration_price": prices["register"],
"renewal_price": prices["renew"],
"transfer_price": prices["transfer"],
})
registrars.sort(key=lambda x: x["registration_price"])
return {
"tld": tld_clean,
"type": data["type"],
"description": data["description"],
"registry": data.get("registry", "Unknown"),
"introduced": data.get("introduced"),
"trend": data["trend"],
"trend_reason": data["trend_reason"],
"pricing": {
"avg": get_avg_price(data),
"min": get_min_price(data),
"max": get_max_price(data),
},
"registrars": registrars,
"cheapest_registrar": registrars[0]["name"],
}
# Fall back to database
db_prices = await get_db_prices(db, tld_clean)
if not db_prices:
raise HTTPException(status_code=404, detail=f"TLD '.{tld_clean}' not found")
tld_data = db_prices[tld_clean]
registrars = [
{
"name": name,
"registration_price": prices["register"],
"renewal_price": prices["renew"],
"transfer_price": prices["transfer"],
}
for name, prices in tld_data["registrars"].items()
]
registrars.sort(key=lambda x: x["registration_price"])
prices = tld_data["prices"]
return {
"tld": tld_clean,
"type": guess_tld_type(tld_clean),
"description": f".{tld_clean} domain extension",
"registry": "Unknown",
"introduced": None,
"trend": "stable",
"trend_reason": "Price tracking started recently",
"pricing": {
"avg": round(sum(prices) / len(prices), 2) if prices else 0,
"min": min(prices) if prices else 0,
"max": max(prices) if prices else 0,
},
"registrars": registrars,
"cheapest_registrar": registrars[0]["name"] if registrars else "N/A",
}