COTexplorer/app/api/routes/analytics.py

from fastapi import APIRouter, HTTPException, Query
from typing import Optional

from app.db import get_db
from app.api.models import PercentileResponse, ScreenerRow

router = APIRouter(prefix="/api/analytics", tags=["analytics"])


@router.get("/screener", response_model=list[ScreenerRow])
def screener(
    exchange: Optional[str] = Query(None),
    lookback_weeks: int = Query(156, ge=4, le=1560),
    top_n: int = Query(50, ge=1, le=500),
    direction: Optional[str] = Query(None, pattern="^(long|short)$"),
):
    """
    Return markets ranked by their current non-commercial net position
    relative to the historical distribution (percentile rank).
    """
    with get_db() as conn:
        # Get all commodities, optionally filtered by exchange
        exchange_filter = "AND c.exchange_abbr = ?" if exchange else ""
        exchange_params = [exchange] if exchange else []

        # For each commodity: get latest date, latest noncomm_net,
        # and compute percentile rank over last N weeks
        rows = conn.execute(
            f"""
            WITH latest AS (
                SELECT c.cftc_code, c.name AS commodity, c.exchange_abbr AS exchange,
                       MAX(r.report_date) AS latest_date
                FROM commodities c
                JOIN reports r ON r.commodity_id = c.id
                {exchange_filter}
                GROUP BY c.cftc_code
            ),
            latest_pos AS (
                SELECT l.cftc_code, l.commodity, l.exchange, l.latest_date,
                       p.open_interest,
                       (p.noncomm_long - p.noncomm_short) AS noncomm_net,
                       p.chg_noncomm_long, p.chg_noncomm_short
                FROM latest l
                JOIN commodities c ON c.cftc_code = l.cftc_code
                JOIN reports r ON r.commodity_id = c.id AND r.report_date = l.latest_date
                JOIN positions p ON p.report_id = r.id AND p.row_type = 'All'
            ),
            lookback AS (
                SELECT c.cftc_code,
                       (p.noncomm_long - p.noncomm_short) AS net,
                       ROW_NUMBER() OVER (PARTITION BY c.cftc_code ORDER BY r.report_date DESC) AS rn
                FROM commodities c
                JOIN reports r ON r.commodity_id = c.id
                JOIN positions p ON p.report_id = r.id AND p.row_type = 'All'
            ),
            pct AS (
                SELECT lp.cftc_code,
                       lp.commodity,
                       lp.exchange,
                       lp.latest_date,
                       lp.open_interest,
                       lp.noncomm_net,
                       lp.chg_noncomm_long,
                       lp.chg_noncomm_short,
                       CAST(
                           (SELECT COUNT(*) FROM lookback lb2
                            WHERE lb2.cftc_code = lp.cftc_code
                              AND lb2.rn <= ? AND lb2.net < lp.noncomm_net)
                           AS REAL
                       ) / NULLIF(
                           (SELECT COUNT(*) FROM lookback lb3
                            WHERE lb3.cftc_code = lp.cftc_code AND lb3.rn <= ?),
                           0
                       ) * 100.0 AS pct_rank
                FROM latest_pos lp
            )
            SELECT cftc_code, commodity, exchange, latest_date,
                   noncomm_net, open_interest, pct_rank,
                   chg_noncomm_long, chg_noncomm_short
            FROM pct
            ORDER BY pct_rank DESC
            LIMIT ?
            """,
            exchange_params + [lookback_weeks, lookback_weeks, top_n],
        ).fetchall()

    result = [ScreenerRow(**dict(r)) for r in rows]
    if direction == 'long':
        result = [r for r in result if r.pct_rank is not None and r.pct_rank >= 50]
    elif direction == 'short':
        result = [r for r in result if r.pct_rank is not None and r.pct_rank < 50]
    return result


@router.get("/{cftc_code}/net-position-percentile", response_model=PercentileResponse)
def net_position_percentile(
    cftc_code: str,
    lookback_weeks: int = Query(156, ge=4, le=1560),
):
    """
    Where does the current non-commercial net position sit in the
    historical distribution over the last N weeks?
    """
    with get_db() as conn:
        row = conn.execute(
            "SELECT id, name FROM commodities WHERE cftc_code = ?", (cftc_code,)
        ).fetchone()
        if not row:
            raise HTTPException(status_code=404, detail=f"Commodity {cftc_code} not found")
        commodity_name = row['name']

        # Get last N weekly net positions
        history = conn.execute(
            """
            SELECT (p.noncomm_long - p.noncomm_short) AS net
            FROM positions p
            JOIN reports r ON r.id = p.report_id
            JOIN commodities c ON c.id = r.commodity_id
            WHERE c.cftc_code = ? AND p.row_type = 'All'
            ORDER BY r.report_date DESC
            LIMIT ?
            """,
            (cftc_code, lookback_weeks),
        ).fetchall()

    if not history:
        raise HTTPException(status_code=404, detail="No position data found")

    nets = [r[0] for r in history if r[0] is not None]
    if not nets:
        return PercentileResponse(
            cftc_code=cftc_code, commodity=commodity_name,
            current_net=None, percentile=None, z_score=None,
            lookback_weeks=lookback_weeks, period_min=None, period_max=None,
        )

    current = nets[0]
    n = len(nets)
    below = sum(1 for v in nets[1:] if v < current)
    percentile = round(below / max(n - 1, 1) * 100, 1)

    mean = sum(nets) / n
    variance = sum((v - mean) ** 2 for v in nets) / n
    std = variance ** 0.5
    z_score = round((current - mean) / std, 2) if std > 0 else 0.0

    return PercentileResponse(
        cftc_code=cftc_code,
        commodity=commodity_name,
        current_net=current,
        percentile=percentile,
        z_score=z_score,
        lookback_weeks=n,
        period_min=min(nets),
        period_max=max(nets),
    )


@router.get("/{cftc_code}/concentration")
def get_concentration(
    cftc_code: str,
    from_date: Optional[str] = Query(None),
    to_date: Optional[str] = Query(None),
    row_type: str = Query("All", pattern="^(All|Old|Other)$"),
):
    with get_db() as conn:
        row = conn.execute(
            "SELECT id FROM commodities WHERE cftc_code = ?", (cftc_code,)
        ).fetchone()
        if not row:
            raise HTTPException(status_code=404, detail=f"Commodity {cftc_code} not found")

        sql = """
            SELECT r.report_date,
                   cn.conc_gross_long_4, cn.conc_gross_short_4,
                   cn.conc_gross_long_8, cn.conc_gross_short_8,
                   cn.conc_net_long_4, cn.conc_net_short_4,
                   cn.conc_net_long_8, cn.conc_net_short_8
            FROM concentration cn
            JOIN reports r ON r.id = cn.report_id
            JOIN commodities c ON c.id = r.commodity_id
            WHERE c.cftc_code = ? AND cn.row_type = ?
        """
        params: list = [cftc_code, row_type]
        if from_date:
            sql += " AND r.report_date >= ?"
            params.append(from_date)
        if to_date:
            sql += " AND r.report_date <= ?"
            params.append(to_date)
        sql += " ORDER BY r.report_date ASC"

        rows = conn.execute(sql, params).fetchall()

    return {"cftc_code": cftc_code, "row_type": row_type, "data": [dict(r) for r in rows]}