The space that belongs to a file with no name — cheat sheet
storage-lvm, logging-journald
# is it really full, and of what?
df -h /path # blocks
df -i /path # inodes — ENOSPC with free blocks
du -sh /path # what the visible files add up to
du -sh --apparent-size /path # …ignoring sparseness/preallocation
du -xh --max-depth=1 /path | sort -h # where it went, one level at a time (-x: one filesystem)
sudo tune2fs -l /dev/vdb | grep -i reserved # the 5% root reservation (ext4)
sudo tune2fs -m 1 /dev/vdb # …make it 1%
# space with no name
lsof -a +L1 /path # open files with no remaining link, on that fs ← the command
# (-a: AND the options; lsof ORs them by default)
lsof +D /path # everything open under a directory
fuser -vm /path # every process with anything open on that filesystem
sudo sh -c 'ls -l /proc/[0-9]*/fd/* 2>/dev/null | grep deleted' # without lsof
# (not `sudo ls /proc/*/fd/*`: that glob expands as YOU, first)
stat -L /proc/PID/fd/N # the real size through the descriptor
# giving it back
kill -HUP PID # ask the daemon to reopen its log ← cheapest, loses nothing
systemctl restart UNIT # honest and obvious
rc-service NAME restart # OpenRC
truncate -s 0 /proc/PID/fd/N # last resort; leaves a sparse file and may corrupt
# hidden under a mount point
mount --bind / /mnt/root ; du -sh /mnt/root/var/log/app ; umount /mnt/root
# logrotate
logrotate -d /etc/logrotate.d/x # dry run: what it would do
logrotate -f /etc/logrotate.d/x # force it now
cat /var/lib/logrotate/status # when each path last rotated
systemctl list-timers logrotate.timer # what runs it (systemd)
ls /etc/periodic/daily/ # what runs it (Alpine)
# a rule that works
# /etc/logrotate.d/ledger
# /var/log/app/*.log {
# size 20M ← a limit: size|maxsize|hourly|daily|weekly|monthly
# rotate 4 ← how many to keep (NOT a limit)
# compress ← delaycompress to defer by one round
# missingok notifempty
# copytruncate ← keeps the inode: for daemons that cannot be signalled
# # or:
# # postrotate
# # /bin/kill -HUP $(cat /run/ledger.pid) 2>/dev/null || true
# # endscript
# # sharedscripts ← run the script once, not once per matched file
# }
# after a forced rotation, the test that matters
tail -1 /var/log/app/ledger.log ; sleep 2 ; tail -1 /var/log/app/ledger.log