feat: add TLD price export/import/seed scripts

- export_tld_prices.py: Export DB prices to JSON
- import_tld_prices.py: Import prices from JSON
- seed_tld_prices.py: Initial scrape from Porkbun API
- tld_prices_export.json: Current price data (886 TLDs)

Usage:
  python scripts/seed_tld_prices.py     # Initial scrape
  python scripts/export_tld_prices.py   # Backup to JSON
  python scripts/import_tld_prices.py   # Restore from JSON
This commit is contained in:
yves.gugger
2025-12-08 09:16:33 +01:00
parent f0cc69ac95
commit 5ac5643cd2
4 changed files with 9028 additions and 0 deletions

View File

@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Export TLD prices from database to JSON file for backup/versioning."""
import asyncio
import json
import sys
from datetime import datetime
from pathlib import Path
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from sqlalchemy import select
from app.database import AsyncSessionLocal, init_db
from app.models.tld_price import TLDPrice
async def export_prices():
"""Export all TLD prices to JSON."""
await init_db()
async with AsyncSessionLocal() as db:
result = await db.execute(select(TLDPrice).order_by(TLDPrice.tld))
prices = result.scalars().all()
data = {
"exported_at": datetime.utcnow().isoformat(),
"total_records": len(prices),
"prices": [
{
"tld": p.tld,
"registrar": p.registrar,
"registration_price": p.registration_price,
"renewal_price": p.renewal_price,
"transfer_price": p.transfer_price,
"currency": p.currency,
"promo_price": p.promo_price,
"recorded_at": p.recorded_at.isoformat() if p.recorded_at else None,
}
for p in prices
]
}
# Save to file
output_path = Path(__file__).parent / "tld_prices_export.json"
with open(output_path, "w") as f:
json.dump(data, f, indent=2)
print(f"Exported {len(prices)} TLD prices to {output_path}")
return output_path
if __name__ == "__main__":
asyncio.run(export_prices())

View File

@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Import TLD prices from JSON file to database."""
import asyncio
import json
import sys
from datetime import datetime
from pathlib import Path
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from app.database import AsyncSessionLocal, init_db
from app.models.tld_price import TLDPrice
async def import_prices(json_path: str = None):
"""Import TLD prices from JSON."""
await init_db()
# Default path
if json_path is None:
json_path = Path(__file__).parent / "tld_prices_export.json"
else:
json_path = Path(json_path)
if not json_path.exists():
print(f"File not found: {json_path}")
return
with open(json_path) as f:
data = json.load(f)
prices = data.get("prices", [])
print(f"Importing {len(prices)} TLD prices from {json_path}")
async with AsyncSessionLocal() as db:
imported = 0
for p in prices:
try:
price = TLDPrice(
tld=p["tld"],
registrar=p["registrar"],
registration_price=p["registration_price"],
renewal_price=p.get("renewal_price"),
transfer_price=p.get("transfer_price"),
currency=p.get("currency", "USD"),
promo_price=p.get("promo_price"),
recorded_at=datetime.fromisoformat(p["recorded_at"]) if p.get("recorded_at") else datetime.utcnow(),
)
db.add(price)
imported += 1
except Exception as e:
print(f"Error importing {p.get('tld')}: {e}")
await db.commit()
print(f"Successfully imported {imported} TLD prices")
if __name__ == "__main__":
json_file = sys.argv[1] if len(sys.argv) > 1 else None
asyncio.run(import_prices(json_file))

View File

@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Seed TLD prices by running a scrape from Porkbun API."""
import asyncio
import sys
from pathlib import Path
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from app.database import AsyncSessionLocal, init_db
from app.services.tld_scraper.aggregator import tld_aggregator
async def seed_prices():
"""Run initial TLD price scrape."""
print("Initializing database...")
await init_db()
print("Starting TLD price scrape from Porkbun API...")
print("This may take 15-30 seconds...")
async with AsyncSessionLocal() as db:
result = await tld_aggregator.run_scrape(db)
print()
print("=" * 50)
print("Seed Results:")
print(f" Status: {result.status}")
print(f" TLDs scraped: {result.tlds_scraped}")
print(f" Prices saved: {result.prices_saved}")
print(f" Sources: {result.sources_succeeded}/{result.sources_attempted}")
if result.errors:
print(f" Errors: {result.errors}")
print("=" * 50)
if result.status == "success":
print("\n✅ TLD prices seeded successfully!")
else:
print("\n⚠️ Seed completed with warnings")
if __name__ == "__main__":
asyncio.run(seed_prices())

File diff suppressed because it is too large Load Diff