Yves Gugger
5e0d4c6590
- Remove seed/demo auction endpoint + scripts (no mock data) - Rebuild AuctionScraper: strict validation (no -- bids, requires end_time) - Add robust sources: - ExpiredDomains provider auction pages (GoDaddy/Namecheap/Sedo) - Park.io auctions table - Sav load_domains_ajax table - Simplify hidden API scrapers to Dynadot only - Add unique index on (platform, domain) + safe upsert - Update deployment/docs to reflect real scraping
92 lines
4.1 KiB
Python
92 lines
4.1 KiB
Python
"""Auction database models for storing scraped auction data."""
|
|
from datetime import datetime
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String, Text, Index
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from app.database import Base
|
|
|
|
|
|
class DomainAuction(Base):
|
|
"""
|
|
Stores domain auction data scraped from various platforms.
|
|
|
|
Platforms supported:
|
|
- GoDaddy Auctions (auctions.godaddy.com)
|
|
- Sedo (sedo.com)
|
|
- NameJet (namejet.com)
|
|
- Afternic (afternic.com)
|
|
- DropCatch (dropcatch.com)
|
|
|
|
Data is scraped periodically and cached here.
|
|
"""
|
|
__tablename__ = "domain_auctions"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
|
|
|
# Domain info
|
|
domain: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
|
tld: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
|
|
|
# Platform info
|
|
platform: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
|
|
platform_auction_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
|
auction_url: Mapped[str] = mapped_column(Text, nullable=False)
|
|
|
|
# Pricing
|
|
current_bid: Mapped[float] = mapped_column(Float, nullable=False)
|
|
currency: Mapped[str] = mapped_column(String(10), default="USD")
|
|
min_bid: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
|
buy_now_price: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
|
reserve_price: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
|
|
reserve_met: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
|
|
|
|
# Auction details
|
|
num_bids: Mapped[int] = mapped_column(Integer, default=0)
|
|
num_watchers: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
|
end_time: Mapped[datetime] = mapped_column(DateTime, nullable=False, index=True)
|
|
auction_type: Mapped[str] = mapped_column(String(50), default="auction") # auction, buy_now, offer
|
|
|
|
# Domain metrics (if available from platform)
|
|
traffic: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
|
age_years: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
|
backlinks: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
|
domain_authority: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
|
|
|
# Scraping metadata
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
scrape_source: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
|
|
|
# Indexes for common queries
|
|
__table_args__ = (
|
|
# Enforce de-duplication at the database level.
|
|
Index('ux_auctions_platform_domain', 'platform', 'domain', unique=True),
|
|
Index('ix_auctions_end_time_active', 'end_time', 'is_active'),
|
|
Index('ix_auctions_tld_bid', 'tld', 'current_bid'),
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<DomainAuction(domain='{self.domain}', platform='{self.platform}', bid=${self.current_bid})>"
|
|
|
|
|
|
class AuctionScrapeLog(Base):
|
|
"""Logs scraping activity for monitoring and debugging."""
|
|
__tablename__ = "auction_scrape_logs"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
|
platform: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
started_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
|
|
completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
|
|
status: Mapped[str] = mapped_column(String(50), default="running") # running, success, failed
|
|
auctions_found: Mapped[int] = mapped_column(Integer, default=0)
|
|
auctions_updated: Mapped[int] = mapped_column(Integer, default=0)
|
|
auctions_new: Mapped[int] = mapped_column(Integer, default=0)
|
|
error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
|
|
|
def __repr__(self):
|
|
return f"<AuctionScrapeLog(platform='{self.platform}', status='{self.status}')>"
|
|
|