#!/usr/bin/env python3 """ Automated Auction Scraper Script This script runs all auction scrapers and saves results to the database. Designed to be run via cron job every 30 minutes. Usage: python scripts/scrape_auctions.py Cron example (every 30 minutes): */30 * * * * cd /home/user/pounce/backend && ./venv/bin/python scripts/scrape_auctions.py >> /var/log/pounce/scraper.log 2>&1 """ import sys import os import asyncio import logging from datetime import datetime from pathlib import Path # Add parent directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent)) from app.services.auction_scraper import auction_scraper from app.database import AsyncSessionLocal # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) async def run_scrapers(): """Run all auction scrapers.""" start_time = datetime.utcnow() logger.info(f"๐Ÿš€ Starting auction scrape at {start_time.isoformat()}") try: async with AsyncSessionLocal() as db: result = await auction_scraper.scrape_all_platforms(db) # Log results total_found = result.get("total_found", 0) total_new = result.get("total_new", 0) logger.info(f"โœ… Scrape complete!") logger.info(f" Total Found: {total_found}") logger.info(f" New Added: {total_new}") # Log platform breakdown platforms = result.get("platforms", {}) for platform, data in platforms.items(): if isinstance(data, dict) and data.get("found", 0) > 0: logger.info(f" {platform}: {data.get('found', 0)} found, {data.get('new', 0)} new") # Log errors (but don't fail) errors = result.get("errors", []) if errors: logger.warning(f"โš ๏ธ {len(errors)} errors occurred:") for err in errors[:5]: logger.warning(f" - {str(err)[:100]}") elapsed = (datetime.utcnow() - start_time).total_seconds() logger.info(f"โฑ๏ธ Completed in {elapsed:.1f} seconds") return result except Exception as e: logger.exception(f"โŒ Scrape failed: {e}") return {"error": str(e)} async def cleanup_old_auctions(): """Remove expired/old auctions from database.""" try: async with AsyncSessionLocal() as db: from sqlalchemy import delete, and_ from datetime import timedelta from app.models.auction import DomainAuction cutoff = datetime.utcnow() - timedelta(days=7) # Mark expired auctions as inactive from sqlalchemy import update stmt = update(DomainAuction).where( and_( DomainAuction.end_time < datetime.utcnow(), DomainAuction.is_active == True ) ).values(is_active=False) result = await db.execute(stmt) await db.commit() if result.rowcount > 0: logger.info(f"๐Ÿงน Marked {result.rowcount} expired auctions as inactive") except Exception as e: logger.warning(f"Cleanup error: {e}") def main(): """Main entry point.""" print("="*60) print(f"๐Ÿพ POUNCE Auction Scraper") print(f" Started: {datetime.now().isoformat()}") print("="*60) # Run scrapers result = asyncio.run(run_scrapers()) # Run cleanup asyncio.run(cleanup_old_auctions()) print("="*60) print(f"โœ… Done!") print("="*60) # Exit with error code if no results if result.get("error") or result.get("total_found", 0) == 0: sys.exit(1) sys.exit(0) if __name__ == "__main__": main()