norboten · cheat sheet

The volume that filled overnight — cheat sheet

storage-lvm, boot-systemd
# reading the stack
lsblk                          # disks, partitions, LVM, mount points, in one tree
pvs / vgs / lvs                # the three LVM layers, one line each
df -h /path                    # filesystem free space (the top layer)
du -sh /path                   # what the files actually add up to
blkid -s UUID -o value DEV     # the UUID to put in fstab
findmnt /path                  # what is mounted there, and how

# growing
vgextend vg0 /dev/vdc          # give the group another physical volume
lvextend -r -l +100%FREE vg0/applv    # grow the volume and the filesystem
lvextend -r -L +2G vg0/applv          # …by a size instead
xfs_growfs /var/lib/app        # grow XFS alone (what -r calls)
resize2fs /dev/vg0/applv       # the ext4 equivalent

# space that du cannot see
lsof -a +L1 /path              # deleted files still held open (-a: AND, not OR)
ls -l /proc/PID/fd | grep deleted
systemctl restart UNIT         # the honest way to make it let go

# fstab
findmnt --verify --tab-file /etc/fstab   # check before you reboot
systemctl daemon-reload                  # after every fstab edit
mount -a                                 # mount everything in fstab
systemctl cat var-lib-app.mount          # the unit systemd generated

# swap
mkswap /swapfile ; chmod 600 /swapfile ; swapon /swapfile
swapon --show
echo '/swapfile none swap defaults 0 0' >> /etc/fstab

# emergency mode
journalctl -xb -p err          # why the boot stopped  ← first
findmnt --verify               # fstab's problems, as warnings as well as errors
systemctl --failed             # what is broken now (a device timeout leaves this EMPTY)
systemctl default              # continue the boot instead of rebooting