Four reasons a service will not stay up — cheat sheet

# which story is it?
systemctl status UNIT              # Loaded: (enabled?), Active:, the exit code, Drop-In:
sudo journalctl -u UNIT -b --no-pager   # this unit, this boot (without sudo/adm: nothing)
sudo journalctl -u UNIT -f              # follow it and wait for the next death
systemctl show -p User -p Restart -p MainPID --value UNIT
# 2xx exit codes = systemd could not run it; status=1 + program output = it ran and quit

# as the service user
sudo -u notes cat /etc/notes/notes.ini      # a system user with nologin still works here
runuser -u notes -- cat /etc/notes/notes.ini
sudo -u notes -s /bin/bash                  # …if you really need a shell
# config a service must read:  chown root:SERVICE + chmod 0640

# who owns the port
ss -lntp | grep :8080              # listening/numeric/tcp/process   (-u UDP, -x unix)
lsof -i :8080
systemctl status PID               # which UNIT a process belongs to
systemctl disable --now other-unit # free it now and at the next boot
# Conflicts=other.service          # …if the two must never run together

# AppArmor
aa-status                          # profiles loaded, and their modes
sudo cat /sys/kernel/security/apparmor/profiles | grep prog     # (enforce) / (complain)
sudo journalctl -k -b | grep -i apparmor   # the denials: profile=, name=, denied_mask=
dmesg | grep -i apparmor
apparmor_parser -r /etc/apparmor.d/PROFILE   # RELOAD after editing (-a add, -R remove)
aa-complain PROG ; aa-logprof ; aa-enforce PROG   # collect denials → rules → back to enforce
aa-genprof PROG                    # build a profile by exercising the program
# rules:  /srv/notes/ r,           ← the directory itself (trailing slash!)
#         /srv/notes/** r,         ← its contents (* stops at /, ** does not)
#         r w a k l m  ix px ux    ← read write append lock link mmap / exec flavours
#         include <abstractions/python>   ← prefer an abstraction to twenty guesses

# restart policy
# [Service] Restart=on-failure|always|on-abnormal|no ; RestartSec=2s
# [Unit]    StartLimitIntervalSec=10s ; StartLimitBurst=5
systemctl reset-failed UNIT         # after "start request repeated too quickly"
kill -9 $(systemctl show -p MainPID --value UNIT) ; sleep 4 ; systemctl is-active UNIT