rig_health: take a sticky alert down when its fault leaves or the observer stops

tools/rig_health.py

@@ -293,14 +293,32 @@ def capture_evidence(check: str, detail: str, out_dir: Path | None = None) -> Pa
         return None
 
 
-def desktop_notify(title: str, body: str, urgent: bool) -> None:
+def desktop_notify(title: str, body: str, urgent: bool) -> int | None:
+    """Returns the notification id (urgent ones only) so the alert can be taken
+    down when the fault goes away: a critical -t 0 popup has no other way out."""
     if shutil.which("notify-send") is None:
+        return None
+    cmd = ["notify-send", "-a", "ParVagues", "-u", "critical" if urgent else "normal",
+           "-t", "0" if urgent else "6000", title, body]
+    try:
+        if not urgent:
+            subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            return None
+        r = subprocess.run(cmd[:1] + ["-p"] + cmd[1:], capture_output=True, text=True, timeout=5)
+        return int(r.stdout.strip()) if r.stdout.strip().isdigit() else None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def desktop_dismiss(nid: int) -> None:
+    if shutil.which("gdbus") is None:
         return
     try:
-        subprocess.Popen(["notify-send", "-a", "ParVagues",
-                          "-u", "critical" if urgent else "normal", "-t", "0" if urgent else "6000",
-                          title, body], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except OSError:
+        subprocess.run(["gdbus", "call", "--session", "--dest", "org.freedesktop.Notifications",
+                        "--object-path", "/org/freedesktop/Notifications", "--method",
+                        "org.freedesktop.Notifications.CloseNotification", str(nid)],
+                       capture_output=True, timeout=5)
+    except (OSError, subprocess.SubprocessError):
         pass
 
 
@@ -309,10 +327,12 @@ class Health:
 
     def __init__(self, path: Path | None = None, clock_watch: ClockWatch | None = None,
                  notify=desktop_notify, evidence=capture_evidence,
-                 now=time.time, async_evidence: bool = True):
+                 now=time.time, async_evidence: bool = True, dismiss=desktop_dismiss):
         self.path = Path(path) if path else health_file()
         self.clock = clock_watch
         self._notify, self._evidence, self._now = notify, evidence, now
+        self._dismiss = dismiss
+        self._sticky: dict[str, int] = {}         # check -> id of its open alert
         self._async = async_evidence
         self.state: dict[str, dict] = {}          # name -> {state, detail, since}
         self._bad: dict[str, int] = {}
@@ -370,6 +390,8 @@ class Health:
     def _transition(self, name: str, was: str, now_state: str, detail: str, t: float) -> dict:
         ev = {"t": round(t, 3), "k": "health", "check": name, "from": was,
               "to": now_state, "detail": detail}
+        if was == FAIL:
+            self._take_down(name)       # the fault is gone, whatever it became
         if now_state == FAIL:
             self._on_fail(name, detail, t)
         elif was == FAIL and now_state == OK:
@@ -391,7 +413,15 @@ class Health:
         if t - self._last_notify.get(name, 0.0) < NOTIFY_COOLDOWN_S and urgent:
             return
         self._last_notify[name] = t
-        self._notify(title, body, urgent)
+        nid = self._notify(title, body, urgent)
+        if urgent and isinstance(nid, int):
+            self._take_down(name)
+            self._sticky[name] = nid
+
+    def _take_down(self, name: str) -> None:
+        nid = self._sticky.pop(name, None)
+        if nid is not None:
+            self._dismiss(nid)
 
     def overall(self) -> str:
         states = [c["state"] for c in self.state.values()]
@@ -414,7 +444,10 @@ class Health:
             pass
 
     def close(self) -> None:
-        """Leave an honest tombstone: a dead observer must not read as health."""
+        """Leave an honest tombstone: a dead observer must not read as health.
+        Its open alerts go too: nobody is left watching to say they are over."""
+        for name in list(self._sticky):
+            self._take_down(name)
         try:
             self.path.write_text(json.dumps({"v": 1, "t": round(self._now(), 3),
                                              "pid": os.getpid(), "overall": "stopped",

tools/tests/test_rig_health.py

@@ -400,3 +400,26 @@ def test_a_recording_failure_offers_to_start_the_recorder_and_nothing_else():
     assert s["badge"] == "down" and s["fixes"] == ["start_recorder"] and "recording" in s["row"]
     s2 = rh.tray_summary(_doc("down", surface=(rh.FAIL, "x"), recording=(rh.FAIL, "y")))
     assert s2["fixes"] == ["start_driver", "start_recorder"]
+
+
+def test_sticky_alert_is_taken_down_when_the_fault_leaves(tmp_path):
+    gone = []
+    r = Rig(tmp_path)
+    r.h._notify = lambda *a: 42 if a[2] else None
+    r.h._dismiss = gone.append
+    r.tick(rh.OK)
+    r.tick(rh.FAIL, n=rh.FAIL_AFTER)
+    assert gone == []
+    r.tick(rh.NA, n=rh.OK_AFTER)            # the rig closed: FAIL -> na, no "recovered"
+    assert gone == [42]
+
+
+def test_close_takes_down_open_alerts(tmp_path):
+    gone = []
+    r = Rig(tmp_path)
+    r.h._notify = lambda *a: 7 if a[2] else None
+    r.h._dismiss = gone.append
+    r.tick(rh.OK)
+    r.tick(rh.FAIL, n=rh.FAIL_AFTER)
+    r.h.close()
+    assert gone == [7]