#!/usr/bin/env bash
# sc-watchdog — restart SuperDirt when the AUDIO SERVER dies, not when sclang does.
#
#     tools/sc-watchdog.sh              # supervise (what the unit runs)
#     tools/sc-watchdog.sh --dry-run    # observe + log + notify, NEVER restart
#
# Why this exists (2026-08-01, OPAL J-7). The rig went silent overnight and nothing
# recovered it. The chain, from the journal:
#
#     10:37:36  PM: suspend entry (s2idle)
#     10:48:38  systemd-coredump: scsynth terminated abnormally
#     10:48:38  sclang: "Server 'localhost' exited with exit code 0."
#     11:18:47  Bridge: "scsynth process not found"   x63, journal only
#
# scsynth is a CHILD of sclang, and systemd's MainPID is sclang. So the audio server
# can die while the unit reports `active (running)` — a green unit is not sound. The
# d834667 fix (QT_QPA_PLATFORM=offscreen) successfully stopped suspend from killing
# *sclang*; suspend then killed *scsynth* instead. The mitigation was one layer too
# shallow, and `Restart=on-failure` cannot help because the unit never fails.
#
# This watchdog closes that gap from OUTSIDE the boot path. It deliberately does not
# touch start_and_midi.scd or the unit's ExecStart: a bug in those is gig-fatal, a bug
# in here is merely annoying. Folding it into an sclang-side ServerQuit handler is the
# correct long-term shape — POST-GIG (#115).
#
# ---------------------------------------------------------------------------
# THE FIRST VERSION FLAPPED. What it got wrong, so it is not repeated:
#
#  1. The unit had `PartOf=parvagues-sc.service`. PartOf propagates RESTARTS, so
#     issuing the restart killed the watchdog; Restart=always revived it with a
#     ZEROED rate-limit counter. The limiter could never trigger — the state that
#     enforced it was destroyed by the action it was limiting. A supervisor must
#     outlive its supervisee. Fixed in the unit (no PartOf) AND here, by persisting
#     the restart log to a file so it survives being killed anyway.
#  2. It slept a fixed 25s after restarting and then judged the outcome. But
#     TimeoutStopSec is 90s, and a SuperDirt boot recompiles the class library and
#     re-registers sample banks. Judging too early meant "still absent" -> restart
#     again, killing the boot in flight. Now it WAITS for scsynth to appear, up to
#     BOOT_WAIT_SECS, and only then decides.
#  3. It was never run in a mode that could be tested without silencing the rig.
#     Hence --dry-run. Exercise that first, always.
#
# It also honours the reason parvagues-sc.service carries its "audio gear should never
# flap" comment: MAX_RESTARTS in a rolling window, then give up LOUDLY and stay down,
# because a server that cannot boot needs eyes, not another try.

set -uo pipefail

# Injectable so the logic can be exercised against a FAKE unit + FAKE server, with
# zero risk to the rig. tools/tests/test-sc-watchdog.sh does exactly that — the first
# version of this script was only ever testable by killing the real audio server,
# which is why its flap was discovered on PLN's desk instead of in a test.
UNIT="${SCWD_UNIT:-parvagues-sc.service}"
PROC="${SCWD_PROC:-scsynth}"
POLL_SECS=2            # how often we look
MISSES_TO_ACT=3        # ~6s absent before we believe it
BOOT_WAIT_SECS=100     # > TimeoutStopSec (90s); a restart may legitimately take this
MAX_RESTARTS=3         # per WINDOW_SECS, then stop trying and stay loud
WINDOW_SECS=600

# Rate-limit state OUTSIDE the process, so a watchdog restart cannot launder it.
# XDG_RUNTIME_DIR is cleared at logout, which is the right lifetime.
STATE="${SCWD_STATE:-${XDG_RUNTIME_DIR:-/tmp}/parvagues-sc-watchdog.restarts}"

DRY_RUN=0
[ "${1:-}" = "--dry-run" ] && DRY_RUN=1

# stderr: unbuffered, so a crash cannot swallow the last thing we said.
log() { printf '%s %s\n' "$(date +%H:%M:%S)" "$*" >&2; }

# The mouth this rig was missing. Never fatal: a headless session has no notify
# daemon, and that must not stop the recovery.
#
# notify <urgency> <timeout-ms> <title> <body>
#
# STICKINESS IS THE SIGNAL, and it is chosen per message:
#   normal   + timeout  -> "this happened and is already handled", fades on its own
#   critical + 0        -> "this needs your hands", stays until dismissed
# Plasma (and most daemons) IGNORE the timeout for urgency=critical — that is the
# documented behaviour, not a bug. The first version shouted `critical` for every
# routine event, so PLN got a wall of notifications that would not go away while the
# rig was busy recovering itself. If everything is sticky, nothing is urgent.
notify() {
  local urgency="$1" timeout="$2" title="$3" body="$4"
  # SCWD_NO_NOTIFY: for the test harness. The first test run fired REAL sticky desktop
  # notifications about a FAKE unit ("3 restarts in 10 min. Giving up — this needs your
  # eyes. journalctl --user -u scwd-test-1269280.service"), which reached PLN minutes
  # after the test had finished and named a unit that no longer existed. A test must
  # never be able to page a human about something that is not real.
  [ "${SCWD_NO_NOTIFY:-0}" = "1" ] && return 0
  command -v notify-send >/dev/null 2>&1 || return 0
  notify-send -a "ParVagues" -u "$urgency" -t "$timeout" "$title" "$body" 2>/dev/null || true
}

# Restart timestamps within the rolling window, persisted across our own death.
recent_restarts() {
  local now t keep=()
  now=$(date +%s)
  if [ -f "$STATE" ]; then
    while read -r t; do
      [ -n "$t" ] && (( now - t < WINDOW_SECS )) && keep+=("$t")
    done < "$STATE"
  fi
  printf '%s\n' ${keep[@]+"${keep[@]}"} > "$STATE"
  echo "${#keep[@]}"
}

record_restart() { date +%s >> "$STATE"; }

# Wait for scsynth to come back, however long the unit legitimately takes.
# Returns 0 as soon as it appears.
await_scsynth() {
  local waited=0
  while (( waited < BOOT_WAIT_SECS )); do
    sleep "$POLL_SECS"; waited=$(( waited + POLL_SECS ))
    if pgrep -x "$PROC" >/dev/null 2>&1; then
      log "$PROC up after ${waited}s"; return 0
    fi
  done
  return 1
}

log "sc-watchdog: watching $PROC (poll ${POLL_SECS}s, act after ${MISSES_TO_ACT} misses)$( ((DRY_RUN)) && echo '  [DRY RUN — will not restart]')"

misses=0
gave_up=0

while true; do
  sleep "$POLL_SECS"

  # Only our business while the unit is meant to be up. If PLN stopped SuperDirt on
  # purpose, silence is the correct state and we must not resurrect it.
  if ! systemctl --user is-active --quiet "$UNIT"; then
    misses=0; gave_up=0; continue
  fi

  if pgrep -x "$PROC" >/dev/null 2>&1; then
    (( misses > 0 )) && log "$PROC back after ${misses} miss(es)"
    misses=0; gave_up=0; continue
  fi

  misses=$(( misses + 1 ))
  (( misses < MISSES_TO_ACT )) && continue
  (( gave_up )) && continue

  if (( DRY_RUN )); then
    log "[dry-run] $PROC GONE (${misses} polls) — WOULD restart ${UNIT}"
    notify normal 6000 "Audio server died (dry run)" "scsynth is gone. Not restarting: --dry-run."
    gave_up=1     # say it once, then stay quiet
    continue
  fi

  n=$(recent_restarts)
  if (( n >= MAX_RESTARTS )); then
    gave_up=1
    log "GIVING UP: ${n} restarts in ${WINDOW_SECS}s and $PROC will not stay up."
    notify critical 0 "SuperDirt will not start" \
      "${n} restarts in $((WINDOW_SECS/60)) min. Giving up — this needs your eyes.
journalctl --user -u ${UNIT} -n 50"
    continue
  fi

  log "$PROC GONE (${misses} polls) while ${UNIT} is active — restarting (${n} prior in window)"
  notify normal 6000 "Audio server died" "scsynth is gone. Restarting SuperDirt…"
  record_restart

  # Restart, then WAIT for the result rather than guessing how long it takes.
  #
  # UNVERIFIED (#122): PLN reported needing a ctrl+enter in Pulsar to resume playing
  # after a recovery. Likely cause: while SuperDirt was down the kernel answered Tidal's
  # OSC datagrams with ICMP port-unreachable, which a CONNECTED UDP socket surfaces as
  # ECONNREFUSED on the next send — so the stream may stop permanently until
  # re-evaluated. Until that is tested, the toast SAYS SO rather than claiming a full
  # recovery: a rig that is alive and silent is indistinguishable on stage from the
  # original fault.
  systemctl --user restart "$UNIT" || log "systemctl restart returned non-zero"
  if await_scsynth; then
    log "recovered"
    notify normal 9000 "Audio server recovered" \
      "SuperDirt is back. If the music does not resume on its own, ctrl+enter in Pulsar."
  else
    log "$PROC did not appear within ${BOOT_WAIT_SECS}s"
    notify critical 0 "SuperDirt did not come back" \
      "No scsynth ${BOOT_WAIT_SECS}s after restart. journalctl --user -u ${UNIT} -n 50"
  fi
  misses=0
done
