norboten · cheat sheet

The space that belongs to a file with no name — cheat sheet

storage-lvm, logging-journald
# is it really full, and of what?
df -h /path                    # blocks
df -i /path                    # inodes — ENOSPC with free blocks
du -sh /path                   # what the visible files add up to
du -sh --apparent-size /path   # …ignoring sparseness/preallocation
du -xh --max-depth=1 /path | sort -h   # where it went, one level at a time (-x: one filesystem)
sudo tune2fs -l /dev/vdb | grep -i reserved     # the 5% root reservation (ext4)
sudo tune2fs -m 1 /dev/vdb                      # …make it 1%

# space with no name
lsof -a +L1 /path              # open files with no remaining link, on that fs  ← the command
                               # (-a: AND the options; lsof ORs them by default)
lsof +D /path                  # everything open under a directory
fuser -vm /path                # every process with anything open on that filesystem
sudo sh -c 'ls -l /proc/[0-9]*/fd/* 2>/dev/null | grep deleted'   # without lsof
                               # (not `sudo ls /proc/*/fd/*`: that glob expands as YOU, first)
stat -L /proc/PID/fd/N         # the real size through the descriptor

# giving it back
kill -HUP PID                  # ask the daemon to reopen its log  ← cheapest, loses nothing
systemctl restart UNIT         # honest and obvious
rc-service NAME restart        # OpenRC
truncate -s 0 /proc/PID/fd/N   # last resort; leaves a sparse file and may corrupt

# hidden under a mount point
mount --bind / /mnt/root ; du -sh /mnt/root/var/log/app ; umount /mnt/root

# logrotate
logrotate -d /etc/logrotate.d/x    # dry run: what it would do
logrotate -f /etc/logrotate.d/x    # force it now
cat /var/lib/logrotate/status      # when each path last rotated
systemctl list-timers logrotate.timer   # what runs it (systemd)
ls /etc/periodic/daily/                 # what runs it (Alpine)

# a rule that works
# /etc/logrotate.d/ledger
#   /var/log/app/*.log {
#       size 20M            ← a limit: size|maxsize|hourly|daily|weekly|monthly
#       rotate 4            ← how many to keep (NOT a limit)
#       compress            ← delaycompress to defer by one round
#       missingok notifempty
#       copytruncate        ← keeps the inode: for daemons that cannot be signalled
#       # or:
#       # postrotate
#       #     /bin/kill -HUP $(cat /run/ledger.pid) 2>/dev/null || true
#       # endscript
#       # sharedscripts     ← run the script once, not once per matched file
#   }

# after a forced rotation, the test that matters
tail -1 /var/log/app/ledger.log ; sleep 2 ; tail -1 /var/log/app/ledger.log