feat(audio): full-track streaming analysis — accept big uploads instead of 413

Producers upload whole tracks, not 60s clips. Before, a large file was either
413-rejected at the edge or buffered wholesale in RAM (upload read + content
hash + engine decode each copied the whole file). Now ingest, hashing and the
signal/loudness engines all stream in bounded windows, so peak memory is
O(window) not O(file) and a big upload can't OOM the shared box.

- engines/streaming.py: probe() + windows() — bounded window iterator over the
  whole file at native SR (soundfile streams WAV/FLAC/OGG/MP3, exact frames).
- onsets/waveform/spectrum: full-track. waveform/spectrum aggregate EXACTLY
  (fixed output grid filled by a streaming reduction); onsets concatenates
  per-window onset envelopes (tiny) for near-exact global tempo/hits.
- loudness: full-track; peak/true-peak/RMS/crest exact, integrated LUFS
  energy-weighted across windows (pyloudnorm gating can't merge chunks exactly)
  and flagged approximate:true / lufs_method:chunked.
- clip-tier (emotion/features/samples/analyze/grade/naming) stay head-analysis
  (a 5-min track's 'sample role' is meaningless) but every response now carries
  an honest source{duration_s,sr,channels,size_bytes} block.
- app ingest streams the upload to disk (never one big read); size ceiling read
  per-request. content_id streams the decode-to-hash, byte-identical to the old
  whole-file hash so the deployed cache stays warm. Upload cap 40->100MB to
  match the gateway.
- tests: test_fulltrack.py proves >60s coverage, content_id stability, exact
  streaming reductions, contiguous windowing. Full suite green (parity w/ base).

armada/api/cache.py

@@ -21,18 +21,27 @@ import db
 # ── addressing ──────────────────────────────────────────────────────────────
 def content_id(audio_path: str | Path) -> str:
     """sha256 of decoded PCM (+ samplerate). Falls back to raw file bytes if the
-    format can't be decoded here (still dedupes identical files)."""
+    format can't be decoded here (still dedupes identical files).
+
+    STREAMED (#7): hashes the decoded PCM block-by-block instead of reading the
+    whole file into RAM — the concatenated float32 bytes are identical to the old
+    whole-array hash, so existing cache ids stay valid. Bounded memory either way."""
     try:
         import numpy as np
         import soundfile as sf
-        y, sr = sf.read(str(audio_path), dtype="float32", always_2d=False)
         h = hashlib.sha256()
-        h.update(np.ascontiguousarray(y).tobytes())
+        with sf.SoundFile(str(audio_path)) as f:
+            sr = f.samplerate
+            for block in f.blocks(blocksize=1 << 18, dtype="float32", always_2d=False):
+                h.update(np.ascontiguousarray(block).tobytes())
         h.update(str(sr).encode())
         return "cid_" + h.hexdigest()[:32]
     except Exception:
-        raw = Path(audio_path).read_bytes()
-        return "raw_" + hashlib.sha256(raw).hexdigest()[:32]
+        h = hashlib.sha256()
+        with open(audio_path, "rb") as fp:
+            for chunk in iter(lambda: fp.read(1 << 20), b""):
+                h.update(chunk)
+        return "raw_" + h.hexdigest()[:32]
 
 
 def params_hash(params: dict | None) -> str:

armada/api/config.py

@@ -42,7 +42,11 @@ DB_PATH = Path(os.environ.get("FOURIER_DB", HERE / "_cache" / "fourier.db"))
 DEV_TOKEN = os.environ.get("FOURIER_DEV_TOKEN", "")
 
 # ── limits ─────────────────────────────────────────────────────────────────────
-MAX_UPLOAD_MB = int(os.environ.get("FOURIER_MAX_UPLOAD_MB", "40"))
+# Match the gateway's client_max_body_size (100 MB) so the app never rejects a
+# body nginx let through. Safe to accept large uploads now: ingest + hashing +
+# the full-track engines all stream in bounded windows (#7), so peak memory is
+# O(window), not O(file). The gateway's friendly 413 fires first above this.
+MAX_UPLOAD_MB = int(os.environ.get("FOURIER_MAX_UPLOAD_MB", "100"))
 
 # ── artifact store (#27) — local disk on erable; freebox deferred ───────────────
 # Big binaries (fetched sources, stems, loops) live content-addressed under here,

armada/api/engines/loudness.py

@@ -5,11 +5,20 @@ The mastering numbers PLN + hexa actually gate on (reference_postprod_master:
 integrated loudness; true-peak is a cheap 4× oversample (catches inter-sample
 peaks a raw sample-peak misses). Returns gain-to-target so a caller knows exactly
 how much to push for each delivery target.
+
+FULL-TRACK (#7): streams the WHOLE track in bounded windows. Peak / true-peak /
+RMS / crest are EXACT (running max / accumulated sum-of-squares). Integrated LUFS
+is energy-weighted across per-window meters — pyloudnorm's gating can't be merged
+exactly across chunks, so this is a close APPROXIMATION, flagged `approximate:true`
+with `lufs_method:"chunked"`. For a steady master the error is well under ~0.5 LU.
 """
 import numpy as np
 
+from . import streaming
+
 _EPS = 1e-12
 TARGETS = {"streaming": -14.0, "club": -9.0}   # LUFS, reference_postprod_master
+_METER_MIN_S = 0.4                              # pyloudnorm needs ≥ one 400 ms gated block
 
 
 def _db(x):
@@ -26,37 +35,65 @@ def _true_peak_dbfs(mono, sr):
         return _db(np.max(np.abs(mono)))
 
 
-def loudness(audio_path, max_seconds=180.0) -> dict:
-    """Integrated LUFS, sample/true peak (dBFS), crest, + gain-to-target."""
-    import soundfile as sf
-    data, sr = sf.read(str(audio_path), dtype="float64")
-    if data.shape[0] < sr // 2:          # pyloudnorm needs ≥ ~0.4 s for a gated block
+def loudness(audio_path, win_s=streaming.DEFAULT_WIN_S) -> dict:
+    """Integrated LUFS (chunked-approx), sample/true peak (dBFS, exact), crest,
+    + gain-to-target, over the whole track."""
+    meta = streaming.probe(audio_path)
+    sr, channels = meta["sr"], meta["channels"]
+    if meta["frames"] and meta["frames"] < sr // 2:
         raise ValueError("clip too short for integrated loudness (need ≥ 0.5 s)")
-    if max_seconds:
-        data = data[: int(sr * max_seconds)]
-    mono = data if data.ndim == 1 else data.mean(axis=1)
-
-    rms_db = _db(np.sqrt(np.mean(mono ** 2) + _EPS))
-    peak_db = _db(np.max(np.abs(mono)))
-    tp_db = _true_peak_dbfs(mono, sr)
 
-    lufs = None
     try:
         import pyloudnorm as pyln
-        lufs = float(pyln.Meter(sr).integrated_loudness(data))
+        meter = pyln.Meter(sr)
     except Exception:
-        pass
+        meter = None
+
+    sumsq = 0.0
+    n = 0
+    peak = _EPS
+    tp = _EPS
+    energy_sum = 0.0        # Σ dur_i · 10^(LUFS_i/10) — linear loudness, duration-weighted
+    dur_sum = 0.0
+    for block, sr, _ in streaming.windows(audio_path, win_s, mono=False, dtype="float64"):
+        if not len(block):
+            continue
+        mono = block if block.ndim == 1 else block.mean(axis=1)
+        sumsq += float(np.sum(mono ** 2))
+        n += mono.size
+        peak = max(peak, float(np.max(np.abs(mono))))
+        tp = max(tp, 10 ** (_true_peak_dbfs(mono, sr) / 20.0))
+        if meter is not None and len(block) >= int(_METER_MIN_S * sr):
+            try:
+                lw = float(meter.integrated_loudness(block))
+            except Exception:
+                lw = float("-inf")
+            if np.isfinite(lw):                        # silence-gated windows → skip
+                w = len(block) / sr
+                energy_sum += w * (10 ** (lw / 10.0))
+                dur_sum += w
+
+    if n < sr // 2:
+        raise ValueError("clip too short for integrated loudness (need ≥ 0.5 s)")
 
-    gain = ({k: round(t - lufs, 2) for k, t in TARGETS.items()} if lufs is not None
-            and np.isfinite(lufs) else None)
+    rms_db = _db(np.sqrt(sumsq / max(n, 1) + _EPS))
+    peak_db = _db(peak)
+    tp_db = _db(tp)
+    lufs = 10.0 * np.log10(energy_sum / dur_sum) if dur_sum > 0 else None
+    gain = ({k: round(t - lufs, 2) for k, t in TARGETS.items()}
+            if lufs is not None and np.isfinite(lufs) else None)
     return {
         "lufs_integrated": round(lufs, 2) if lufs is not None and np.isfinite(lufs) else None,
+        "lufs_method": "chunked",
+        "approximate": True,          # energy-weighted across windows, not a single gated pass
         "peak_dbfs": round(peak_db, 2),
         "true_peak_dbfs": round(tp_db, 2),
         "rms_dbfs": round(rms_db, 2),
         "crest_db": round(peak_db - rms_db, 2),
-        "gain_to_target_db": gain,         # add this many dB to hit each LUFS target
-        "channels": 1 if data.ndim == 1 else data.shape[1],
+        "gain_to_target_db": gain,    # add this many dB to hit each LUFS target
+        "channels": int(channels),
         "sr": int(sr),
+        "duration_s": round(meta["duration_s"], 3),
         "engine": "light",
+        "full_track": True,
     }

armada/api/engines/streaming.py

+"""Bounded-memory streaming primitives for full-track analysis. CPU, NO torch.
+
+Producers upload whole tracks, not 60 s clips. Rather than 413-reject a big file
+(or buffer the whole thing in RAM), the signal/loudness engines decode it in
+fixed time windows and aggregate as they go — so peak memory is O(one window),
+not O(file), and a large upload can't OOM the shared erable box.
+
+soundfile (libsndfile 1.2) streams WAV/FLAC/OGG/MP3 with exact frame counts, so
+we read at NATIVE sample rate and let each engine pass `sr` to librosa — no
+per-block resampling (which would add seam artifacts at window boundaries).
+"""
+import numpy as np
+
+# 30 s windows: at 96 kHz stereo float64 that is ~44 MB resident — comfortably
+# under the container's 3 GB cap, with margin for the engine's own working set.
+DEFAULT_WIN_S = 30.0
+
+
+def probe(path) -> dict:
+    """Header-only metadata (no decode): frames, sr, channels, duration_s.
+    `frames <= 0` means the container can't frame-count this file (rare); callers
+    fall back to a bounded whole-file decode."""
+    import soundfile as sf
+    i = sf.info(str(path))
+    return {
+        "frames": int(i.frames),
+        "sr": int(i.samplerate),
+        "channels": int(i.channels),
+        "duration_s": (i.frames / i.samplerate) if i.samplerate else 0.0,
+        "format": i.format,
+    }
+
+
+def windows(path, win_s=DEFAULT_WIN_S, *, mono=True, dtype="float32"):
+    """Yield (block, sr, start_frame) over the WHOLE file in `win_s` windows.
+
+    `mono=True` downmixes channels (mean); `mono=False` yields 2-D (n, channels).
+    Non-overlapping and contiguous: start_frame of window k is the running sum of
+    prior window lengths, so callers can map a block's samples to absolute time.
+    Bounded memory: only one window is resident at a time."""
+    import soundfile as sf
+    with sf.SoundFile(str(path)) as f:
+        sr = f.samplerate
+        wf = max(1, int(win_s * sr))
+        start = 0
+        while True:
+            blk = f.read(frames=wf, dtype=dtype, always_2d=not mono)
+            if len(blk) == 0:
+                return
+            block = (blk if blk.ndim == 1 else blk.mean(axis=1)) if mono else blk
+            yield block, sr, start
+            start += len(block)

armada/api/tests/test_fulltrack.py

+"""Full-track streaming (#7) — the guarantees that make big uploads safe.
+
+Proves the two things unit tests on 2 s clips can't: (1) the signal/loudness
+engines now analyse the WHOLE track, past the old 60 s / 180 s head cap, and
+(2) streaming didn't change the numbers — content_id stays byte-identical to the
+old whole-file hash (so the deployed cache stays warm), and exact reductions
+(waveform min/max) equal a single-pass computation.
+
+    cd armada/api && python -m pytest tests/test_fulltrack.py -q
+"""
+import hashlib
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+import cache
+from engines import loudness, signal, streaming
+
+SR = 22050
+
+
+def _click_track(bpm=120, secs=70.0, sr=SR):
+    """A long pulse train (>60 s) so a head-only analysis would visibly truncate."""
+    n = int(sr * secs)
+    y = np.zeros(n, dtype="float32")
+    step = int(sr * 60 / bpm)
+    for i in range(0, n, step):
+        y[i:i + 200] += np.hanning(min(200, n - i)).astype("float32")
+    return y
+
+
+def _write(y, sr=SR, suffix=".wav"):
+    p = Path(tempfile.mktemp(suffix=suffix))
+    sf.write(str(p), y, sr)
+    return str(p)
+
+
+# ── (1) full-track coverage: past the old 60 s / 180 s head cap ─────────────────
+def test_onsets_cover_whole_track_not_just_head():
+    p = _write(_click_track(secs=70.0))
+    r = signal.onsets(p)
+    assert r["full_track"] is True
+    assert r["duration_s"] >= 69.0                 # whole track, not capped at 60
+    assert max(r["onsets"]) > 60.0                 # hits found beyond the old head window
+    assert r["n_onsets"] >= 120                     # ~120 bpm × 70 s ≈ 140 pulses
+
+
+def test_waveform_spans_whole_track():
+    p = _write(_click_track(secs=70.0))
+    r = signal.waveform(p, bins=100)
+    assert r["full_track"] is True and r["bins"] == 100
+    assert r["duration_s"] >= 69.0
+    # pulses recur throughout → energy present in the last bins, not only the first
+    assert r["rms"][-1] > 0.0 and r["rms"][90] > 0.0
+
+
+def test_loudness_full_track_exact_peak_and_flagged_approx():
+    y = _click_track(secs=70.0) * 0.5
+    r = loudness.loudness(_write(y))
+    assert r["full_track"] is True and r["approximate"] is True
+    assert r["lufs_method"] == "chunked" and r["lufs_integrated"] is not None
+    # peak is an EXACT running max over the stream
+    assert abs(r["peak_dbfs"] - 20 * np.log10(float(np.max(np.abs(y))))) < 0.1
+    assert r["duration_s"] >= 69.0
+
+
+# ── (2) streaming didn't change the numbers ─────────────────────────────────────
+def test_content_id_streaming_equals_whole_file_hash():
+    """The streamed content_id must equal the OLD whole-array hash, or every
+    deployed cache entry silently invalidates on deploy."""
+    y = (0.2 * np.sin(2 * np.pi * 220 * np.linspace(0, 5, 5 * SR, endpoint=False))).astype("float32")
+    p = _write(y)
+    # reproduce the pre-#7 computation: sha256(whole_pcm.tobytes() + str(sr))
+    data, sr = sf.read(p, dtype="float32", always_2d=False)
+    h = hashlib.sha256()
+    h.update(np.ascontiguousarray(data).tobytes())
+    h.update(str(sr).encode())
+    expected = "cid_" + h.hexdigest()[:32]
+    assert cache.content_id(p) == expected
+
+
+def test_waveform_reduction_is_exact_vs_single_pass():
+    """Streamed per-bin min/max must equal a single-pass numpy computation."""
+    y = (0.4 * np.sin(2 * np.pi * 110 * np.linspace(0, 8, 8 * SR, endpoint=False))).astype("float32")
+    p = _write(y)
+    bins = 64
+    r = signal.waveform(p, bins=bins)
+    edges = np.linspace(0, len(y), bins + 1).astype(int)
+    for i in range(bins):
+        seg = y[edges[i]:edges[i + 1]]
+        assert abs(r["peaks"][i][0] - round(float(seg.min()), 4)) < 1e-3
+        assert abs(r["peaks"][i][1] - round(float(seg.max()), 4)) < 1e-3
+
+
+def test_streaming_windows_cover_all_samples_once():
+    """The window iterator must yield every sample exactly once, contiguously."""
+    y = np.arange(int(2.5 * SR), dtype="float32")
+    p = _write(y)
+    seen = 0
+    last_end = 0
+    for block, sr, start in streaming.windows(p, win_s=1.0, mono=True):
+        assert start == last_end                    # contiguous, no gap/overlap
+        seen += len(block)
+        last_end = start + len(block)
+    assert seen == len(y)