Yves Gugger
bb7ce97330
Some checks failed
CI / Frontend Lint & Type Check (push) Has been cancelled
CI / Frontend Build (push) Has been cancelled
CI / Backend Lint (push) Has been cancelled
CI / Backend Tests (push) Has been cancelled
CI / Docker Build (push) Has been cancelled
CI / Security Scan (push) Has been cancelled
Deploy / Build & Push Images (push) Has been cancelled
Deploy / Deploy to Server (push) Has been cancelled
Deploy / Notify (push) Has been cancelled
305 lines
12 KiB
Python
305 lines
12 KiB
Python
"""
|
|
Business KPIs exported as Prometheus metrics (4B Ops).
|
|
|
|
These KPIs are derived from real telemetry events in the database.
|
|
We cache computations to avoid putting load on the DB on every scrape.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from typing import Any, Optional
|
|
|
|
from sqlalchemy import and_, func, select
|
|
|
|
from app.config import get_settings
|
|
from app.database import AsyncSessionLocal
|
|
from app.models.telemetry import TelemetryEvent
|
|
|
|
|
|
settings = get_settings()
|
|
|
|
try:
|
|
from prometheus_client import Gauge
|
|
except Exception: # pragma: no cover
|
|
Gauge = None # type: ignore
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TelemetryWindowKpis:
|
|
window_days: int
|
|
start: datetime
|
|
end: datetime
|
|
|
|
# Deal
|
|
listing_views: int
|
|
inquiries_created: int
|
|
seller_replied_inquiries: int
|
|
inquiry_reply_rate: float
|
|
listings_with_inquiries: int
|
|
listings_sold: int
|
|
inquiry_to_sold_listing_rate: float
|
|
|
|
# Yield
|
|
connected_domains: int
|
|
clicks: int
|
|
conversions: int
|
|
conversion_rate: float
|
|
payouts_paid: int
|
|
payouts_paid_amount_total: float
|
|
|
|
|
|
_cache_until_by_days: dict[int, datetime] = {}
|
|
_cache_value_by_days: dict[int, TelemetryWindowKpis] = {}
|
|
|
|
|
|
def _safe_json(metadata_json: Optional[str]) -> dict[str, Any]:
|
|
if not metadata_json:
|
|
return {}
|
|
try:
|
|
value = json.loads(metadata_json)
|
|
return value if isinstance(value, dict) else {}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
async def _compute_window_kpis(days: int) -> TelemetryWindowKpis:
|
|
end = datetime.utcnow()
|
|
start = end - timedelta(days=days)
|
|
|
|
async with AsyncSessionLocal() as db:
|
|
# Fast path: grouped counts for pure counter events
|
|
count_events = [
|
|
"listing_view",
|
|
"inquiry_created",
|
|
"yield_connected",
|
|
"yield_click",
|
|
"yield_conversion",
|
|
"payout_paid",
|
|
]
|
|
grouped = (
|
|
await db.execute(
|
|
select(TelemetryEvent.event_name, func.count(TelemetryEvent.id))
|
|
.where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name.in_(count_events),
|
|
)
|
|
)
|
|
.group_by(TelemetryEvent.event_name)
|
|
)
|
|
).all()
|
|
counts = {name: int(cnt) for name, cnt in grouped}
|
|
|
|
listing_views = counts.get("listing_view", 0)
|
|
inquiries_created = counts.get("inquiry_created", 0)
|
|
connected_domains = counts.get("yield_connected", 0)
|
|
clicks = counts.get("yield_click", 0)
|
|
conversions = counts.get("yield_conversion", 0)
|
|
payouts_paid = counts.get("payout_paid", 0)
|
|
|
|
# Distinct listing counts (deal)
|
|
listings_with_inquiries = (
|
|
await db.execute(
|
|
select(func.count(func.distinct(TelemetryEvent.listing_id))).where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name == "inquiry_created",
|
|
TelemetryEvent.listing_id.isnot(None),
|
|
)
|
|
)
|
|
)
|
|
).scalar() or 0
|
|
|
|
listings_sold = (
|
|
await db.execute(
|
|
select(func.count(func.distinct(TelemetryEvent.listing_id))).where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name == "listing_marked_sold",
|
|
TelemetryEvent.listing_id.isnot(None),
|
|
)
|
|
)
|
|
)
|
|
).scalar() or 0
|
|
|
|
# For rates we need intersections/uniques; keep it exact via minimal event fetch
|
|
inquiry_listing_ids = (
|
|
await db.execute(
|
|
select(func.distinct(TelemetryEvent.listing_id)).where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name == "inquiry_created",
|
|
TelemetryEvent.listing_id.isnot(None),
|
|
)
|
|
)
|
|
)
|
|
).scalars().all()
|
|
sold_listing_ids = (
|
|
await db.execute(
|
|
select(func.distinct(TelemetryEvent.listing_id)).where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name == "listing_marked_sold",
|
|
TelemetryEvent.listing_id.isnot(None),
|
|
)
|
|
)
|
|
)
|
|
).scalars().all()
|
|
|
|
inquiry_set = {int(x) for x in inquiry_listing_ids if x is not None}
|
|
sold_set = {int(x) for x in sold_listing_ids if x is not None}
|
|
sold_from_inquiry = inquiry_set.intersection(sold_set)
|
|
inquiry_to_sold_listing_rate = (len(sold_from_inquiry) / len(inquiry_set)) if inquiry_set else 0.0
|
|
|
|
# Seller reply rate: unique inquiries with at least one seller message
|
|
msg_rows = (
|
|
await db.execute(
|
|
select(TelemetryEvent.inquiry_id, TelemetryEvent.metadata_json).where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name == "message_sent",
|
|
TelemetryEvent.inquiry_id.isnot(None),
|
|
)
|
|
)
|
|
)
|
|
).all()
|
|
seller_replied_inquiries_set: set[int] = set()
|
|
for inquiry_id, metadata_json in msg_rows:
|
|
if inquiry_id is None:
|
|
continue
|
|
meta = _safe_json(metadata_json)
|
|
if meta.get("role") == "seller":
|
|
seller_replied_inquiries_set.add(int(inquiry_id))
|
|
|
|
seller_replied_inquiries = len(seller_replied_inquiries_set)
|
|
inquiry_reply_rate = (seller_replied_inquiries / inquiries_created) if inquiries_created else 0.0
|
|
|
|
# Payout amounts (sum of metadata amounts)
|
|
payout_rows = (
|
|
await db.execute(
|
|
select(TelemetryEvent.metadata_json).where(
|
|
and_(
|
|
TelemetryEvent.created_at >= start,
|
|
TelemetryEvent.created_at <= end,
|
|
TelemetryEvent.event_name == "payout_paid",
|
|
TelemetryEvent.metadata_json.isnot(None),
|
|
)
|
|
)
|
|
)
|
|
).scalars().all()
|
|
payouts_paid_amount_total = 0.0
|
|
for metadata_json in payout_rows:
|
|
meta = _safe_json(metadata_json)
|
|
amount = meta.get("amount")
|
|
if isinstance(amount, (int, float)):
|
|
payouts_paid_amount_total += float(amount)
|
|
|
|
conversion_rate = (conversions / clicks) if clicks else 0.0
|
|
|
|
return TelemetryWindowKpis(
|
|
window_days=days,
|
|
start=start,
|
|
end=end,
|
|
listing_views=int(listing_views),
|
|
inquiries_created=int(inquiries_created),
|
|
seller_replied_inquiries=int(seller_replied_inquiries),
|
|
inquiry_reply_rate=float(inquiry_reply_rate),
|
|
listings_with_inquiries=int(listings_with_inquiries),
|
|
listings_sold=int(listings_sold),
|
|
inquiry_to_sold_listing_rate=float(inquiry_to_sold_listing_rate),
|
|
connected_domains=int(connected_domains),
|
|
clicks=int(clicks),
|
|
conversions=int(conversions),
|
|
conversion_rate=float(conversion_rate),
|
|
payouts_paid=int(payouts_paid),
|
|
payouts_paid_amount_total=float(payouts_paid_amount_total),
|
|
)
|
|
|
|
|
|
async def get_cached_window_kpis(days: int) -> Optional[TelemetryWindowKpis]:
|
|
"""Return cached KPIs for a window (recompute if TTL expired)."""
|
|
if not settings.enable_business_metrics:
|
|
return None
|
|
|
|
now = datetime.utcnow()
|
|
until = _cache_until_by_days.get(days)
|
|
cached = _cache_value_by_days.get(days)
|
|
if until is not None and cached is not None and now < until:
|
|
return cached
|
|
|
|
value = await _compute_window_kpis(int(days))
|
|
ttl_seconds = max(5, int(settings.business_metrics_cache_seconds))
|
|
_cache_until_by_days[int(days)] = now + timedelta(seconds=ttl_seconds)
|
|
_cache_value_by_days[int(days)] = value
|
|
return value
|
|
|
|
|
|
# -----------------------------
|
|
# Prometheus Gauges
|
|
# -----------------------------
|
|
|
|
if Gauge is not None:
|
|
_g = {
|
|
"deal_listing_views": Gauge("pounce_deal_listing_views", "Deal: listing views in window", ["window_days"]),
|
|
"deal_inquiries_created": Gauge("pounce_deal_inquiries_created", "Deal: inquiries created in window", ["window_days"]),
|
|
"deal_seller_replied_inquiries": Gauge(
|
|
"pounce_deal_seller_replied_inquiries", "Deal: inquiries with seller reply in window", ["window_days"]
|
|
),
|
|
"deal_inquiry_reply_rate": Gauge("pounce_deal_inquiry_reply_rate", "Deal: inquiry reply rate in window", ["window_days"]),
|
|
"deal_listings_with_inquiries": Gauge(
|
|
"pounce_deal_listings_with_inquiries", "Deal: distinct listings with inquiries in window", ["window_days"]
|
|
),
|
|
"deal_listings_sold": Gauge("pounce_deal_listings_sold", "Deal: distinct listings marked sold in window", ["window_days"]),
|
|
"deal_inquiry_to_sold_listing_rate": Gauge(
|
|
"pounce_deal_inquiry_to_sold_listing_rate", "Deal: (listings with inquiry) -> sold rate in window", ["window_days"]
|
|
),
|
|
"yield_connected_domains": Gauge("pounce_yield_connected_domains", "Yield: connected domains in window", ["window_days"]),
|
|
"yield_clicks": Gauge("pounce_yield_clicks", "Yield: clicks in window", ["window_days"]),
|
|
"yield_conversions": Gauge("pounce_yield_conversions", "Yield: conversions in window", ["window_days"]),
|
|
"yield_conversion_rate": Gauge("pounce_yield_conversion_rate", "Yield: conversion rate in window", ["window_days"]),
|
|
"yield_payouts_paid": Gauge("pounce_yield_payouts_paid", "Yield: payouts paid in window", ["window_days"]),
|
|
"yield_payouts_paid_amount_total": Gauge(
|
|
"pounce_yield_payouts_paid_amount_total", "Yield: total amount paid out in window", ["window_days"]
|
|
),
|
|
}
|
|
else: # pragma: no cover
|
|
_g = {}
|
|
|
|
|
|
async def update_prometheus_business_metrics() -> None:
|
|
"""Compute KPIs and set Prometheus gauges (no-op when disabled)."""
|
|
if Gauge is None or not _g:
|
|
return
|
|
if not settings.enable_business_metrics:
|
|
return
|
|
|
|
windows = {1, int(settings.business_metrics_days)}
|
|
for days in sorted(windows):
|
|
kpis = await get_cached_window_kpis(days)
|
|
if kpis is None:
|
|
continue
|
|
w = str(int(kpis.window_days))
|
|
_g["deal_listing_views"].labels(window_days=w).set(kpis.listing_views)
|
|
_g["deal_inquiries_created"].labels(window_days=w).set(kpis.inquiries_created)
|
|
_g["deal_seller_replied_inquiries"].labels(window_days=w).set(kpis.seller_replied_inquiries)
|
|
_g["deal_inquiry_reply_rate"].labels(window_days=w).set(kpis.inquiry_reply_rate)
|
|
_g["deal_listings_with_inquiries"].labels(window_days=w).set(kpis.listings_with_inquiries)
|
|
_g["deal_listings_sold"].labels(window_days=w).set(kpis.listings_sold)
|
|
_g["deal_inquiry_to_sold_listing_rate"].labels(window_days=w).set(kpis.inquiry_to_sold_listing_rate)
|
|
_g["yield_connected_domains"].labels(window_days=w).set(kpis.connected_domains)
|
|
_g["yield_clicks"].labels(window_days=w).set(kpis.clicks)
|
|
_g["yield_conversions"].labels(window_days=w).set(kpis.conversions)
|
|
_g["yield_conversion_rate"].labels(window_days=w).set(kpis.conversion_rate)
|
|
_g["yield_payouts_paid"].labels(window_days=w).set(kpis.payouts_paid)
|
|
_g["yield_payouts_paid_amount_total"].labels(window_days=w).set(kpis.payouts_paid_amount_total)
|
|
|