# HTTP checks that tell the truth
curl -sf -m 3 URL > /dev/null ; echo $? # 0 ok, 22 HTTP >= 400, 28 timeout, 7 refused
curl -sf -m 3 URL | grep -q '^ok$' # …and the body says what you expect
timeout 10 CMD # a deadline for anything (124 = timed out)
# what systemd knows
systemctl is-active UNIT # main process not exited — NOT "working"
systemctl show -p Result -p NRestarts -p ActiveEnterTimestamp UNIT
journalctl -u UNIT --since -10min
# alert on failure
# alert@.service: [Service] Type=oneshot ExecStart=/path/to/notify "unit %i failed"
# in the unit: [Unit] OnFailure=alert@%n.service (fires before each Restart= too)
# end hangs
# [Service] Type=notify WatchdogSec=5 Restart=on-failure
# the program: send READY=1 once, then WATCHDOG=1 more often than every $WATCHDOG_USEC µs
# — from the code path that does the work, not a separate timer thread
systemd-notify --ready ; systemd-notify WATCHDOG=1 # from a shell service; with User= (non-root)
# it is ignored unless NotifyAccess=all
# crash loops
systemctl show -p NRestarts UNIT # alert when it grows between checks
# simulate failures
kill -STOP PID ; kill -CONT PID # a hang, and back
kill -9 PID # a crash