#!/usr/bin/env python3
"""build_tierlist — the Foundry Hall of Fame (task #11).

Grade EVERY hand-cut loop in PLN's Samples corpus with the Foundry katana
(tools/foundry/engine/grade.py), rank kits into S/A/B/C/D tiers, and — the part
that earns its keep — **validate the rubric against reality**: a kit's grade
should track how often PLN actually reaches for it in his `.tidal` scores. If the
core-palette kits (used most) don't tier high, the rubric is wrong, not the
library (PHASE2_FINDINGS §5b).

Two sources, joined on the kit name (= the `s "kit"` token):
  • grades  — median over each kit's depth-1 *.wav (the loops, not nested packs),
              graded in parallel; the katana is the single source of truth (DRY).
  • usage   — per-sound track counts from catalog_view.json (same signal tide_eda
              calls `palette`), i.e. how many tracks play `s "kit"`.

    python3 build_tierlist.py            # → tierlist.json  (feeds tierlist.html)
    python3 build_tierlist.py --jobs 8   # parallelism (default: cpu-2)

Runs under system python3. Imports the Foundry grader off-tree via sys.path.
"""
from __future__ import annotations

import argparse
import json
import os
import sys
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

HERE = Path(__file__).resolve().parent
ROOT = HERE.parent.parent                              # …/Sound/Tidal
FOUNDRY = ROOT / "tools" / "foundry"
sys.path.insert(0, str(FOUNDRY))

from engine import grade as G                          # noqa: E402  (the katana)

SAMPLES = Path(os.environ.get("FOUNDRY_SAMPLES", Path.home() / "Work/Sound/Samples"))
CV = HERE / "catalog_view.json"
OUT = HERE / "tierlist.json"
AS_OF = os.environ.get("TIERLIST_AS_OF", "")           # stamp via env (no Date.now in scripts)

TIER_ORDER = ["S", "A", "B", "C", "D"]


# ── grading (top-level so ProcessPoolExecutor can pickle it) ─────────────────
def _grade_one(path_str: str) -> dict | None:
    try:
        g = G.grade(path_str)
        return {"name": Path(path_str).name, "grade": g.grade, "tier": g.tier,
                "kind": g.kind, "flags": g.flags, "sub": g.sub,
                "dur_s": g.metrics.get("dur_s"), "rms_dbfs": g.metrics.get("rms_dbfs")}
    except Exception as e:
        return {"name": Path(path_str).name, "error": str(e)}


def kit_wavs(kit_dir: Path) -> list[Path]:
    """The hand-cut loops = depth-1 wavs (skip nested sample-packs)."""
    return sorted(p for p in kit_dir.iterdir()
                  if p.is_file() and p.suffix.lower() == ".wav")


def usage_counts() -> Counter:
    """How many tracks play each `s "<sound>"` token (catalog_view, DRY w/ tide_eda)."""
    snd = Counter()
    if not CV.exists():
        return snd
    cv = json.loads(CV.read_text())
    for t in cv.get("tracks", []):
        for s in set(t.get("score_sounds", [])):       # once per track
            snd[s] += 1
    return snd


def median(xs: list[float]) -> float:
    s = sorted(xs)
    n = len(s)
    if not n:
        return 0.0
    m = n // 2
    return s[m] if n % 2 else (s[m - 1] + s[m]) / 2


def spearman(pairs: list[tuple[float, float]]) -> float | None:
    """Rank correlation, ties-averaged. None if <3 points or degenerate."""
    if len(pairs) < 3:
        return None
    xs, ys = zip(*pairs)

    def ranks(v):
        order = sorted(range(len(v)), key=lambda i: v[i])
        r = [0.0] * len(v)
        i = 0
        while i < len(v):
            j = i
            while j + 1 < len(v) and v[order[j + 1]] == v[order[i]]:
                j += 1
            avg = (i + j) / 2 + 1
            for k in range(i, j + 1):
                r[order[k]] = avg
            i = j + 1
        return r

    rx, ry = ranks(xs), ranks(ys)
    n = len(pairs)
    mx, my = sum(rx) / n, sum(ry) / n
    cov = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
    vx = sum((a - mx) ** 2 for a in rx) ** 0.5
    vy = sum((b - my) ** 2 for b in ry) ** 0.5
    return round(cov / (vx * vy), 4) if vx and vy else None


def tier_of(grade: float) -> str:
    for name, lo in G.TIERS:
        if grade >= lo:
            return name
    return "D"


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description="grade the Samples corpus into a tierlist")
    ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2))
    ap.add_argument("--limit", type=int, default=0, help="grade only N kits (smoke test)")
    a = ap.parse_args(argv)

    kits = sorted(d for d in SAMPLES.iterdir() if d.is_dir())
    if a.limit:
        kits = kits[: a.limit]
    usage = usage_counts()

    # flatten (kit, wav) and grade in parallel — the slow part
    jobs: list[tuple[str, str]] = []
    for kdir in kits:
        for w in kit_wavs(kdir):
            jobs.append((kdir.name, str(w)))
    print(f"grading {len(jobs)} wavs across {len(kits)} kits on {a.jobs} workers…",
          file=sys.stderr)

    results: dict[str, list[dict]] = {k.name: [] for k in kits}
    with ProcessPoolExecutor(max_workers=a.jobs) as ex:
        graded = list(ex.map(_grade_one, [j[1] for j in jobs], chunksize=8))
    for (kit, _), g in zip(jobs, graded):
        results[kit].append(g)

    # aggregate per kit
    kit_rows = []
    for kdir in kits:
        files = [f for f in results[kdir.name] if "grade" in f]
        if not files:
            continue
        grades = [f["grade"] for f in files]
        med = round(median(grades), 4)
        tier_dist = Counter(f["tier"] for f in files)
        flag_dist = Counter(fl for f in files for fl in f.get("flags", []))
        kit_rows.append({
            "kit": kdir.name,
            "n_wavs": len(files),
            "grade": med,
            "tier": tier_of(med),
            "best": round(max(grades), 4),
            "worst": round(min(grades), 4),
            "usage": usage.get(kdir.name, 0),
            "tier_dist": {t: tier_dist.get(t, 0) for t in TIER_ORDER},
            "top_flags": dict(flag_dist.most_common(5)),
            "files": sorted(files, key=lambda f: -f["grade"]),
        })

    kit_rows.sort(key=lambda r: -r["grade"])

    # rubric validation: do used kits tier higher?
    used = [(r["grade"], r["usage"]) for r in kit_rows if r["usage"] > 0]
    rho = spearman(used)
    core = [r for r in kit_rows if r["usage"] >= 5]        # set-staple kits
    core_b_plus = sum(1 for r in core if r["grade"] >= 0.70)

    corpus_tier = Counter(r["tier"] for r in kit_rows)
    report = {
        "schema": "foundry tierlist v1",
        "as_of": AS_OF,
        "samples_root": str(SAMPLES),
        "grader": "tools/foundry/engine/grade.py",
        "n_kits": len(kit_rows),
        "n_wavs": sum(r["n_wavs"] for r in kit_rows),
        "tier_distribution": {t: corpus_tier.get(t, 0) for t in TIER_ORDER},
        "validation": {
            "note": "rubric is sound iff used kits tier high (PHASE2_FINDINGS §5b)",
            "grade_usage_spearman": rho,
            "n_kits_with_usage": len(used),
            "core_kits_usage_ge_5": len(core),
            "core_kits_b_tier_or_above": core_b_plus,
            "core_kits_pass": (core_b_plus / len(core)) if core else None,
        },
        "kits": kit_rows,
    }
    OUT.write_text(json.dumps(report, indent=2))
    print(f"✓ {OUT.name}: {len(kit_rows)} kits, {report['n_wavs']} wavs | "
          f"tiers {dict(report['tier_distribution'])} | "
          f"grade↔usage ρ={rho} | core B+ {core_b_plus}/{len(core)}",
          file=sys.stderr)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
