Eric Radman : a Journal

Custom FreeBSD Network-Boot Ramdisk

Automating a FreeBSD install by fetching a ramdisk image over TFTP can be dramatically improved by building a smaller image.

loader.efi     0.7 MB
kernel.gz     11.0 MB
tmpfs.ko.gz    0.6 MB
zfs.ko.gz      1.6 MB
mfsroot.gz    27.4 MB

By building a custom PXE-boot image we can build a completely custom installer, or use it for system repair.

Build Settings

For provisioning, most essential modules are built-in. Provide a list of extra features in make.conf

# /etc/make.conf
MODULES_OVERRIDE=tmpfs zfs

Provide a very long list of excludes in src.conf

# /etc/src.conf
WITHOUT_ACCT=YES
WITHOUT_APM=YES
WITHOUT_AT=YES
WITHOUT_AUDIT=YES
WITHOUT_AUTOFS=YES
WITHOUT_BHYVE=YES
WITHOUT_BLOCKLIST=YES
WITHOUT_BLUETOOTH=YES
WITHOUT_BOOTPARAMD=YES
WITHOUT_BOOTPD=YES
WITHOUT_BSNMP=YES
WITHOUT_CALENDAR=YES
WITHOUT_CCD=YES
WITHOUT_CROSS_COMPILER=YES
WITHOUT_CUSE=YES
WITHOUT_CXGBETOOL=YES
WITHOUT_DEBUG_FILES=YES
WITHOUT_DTRACE=YES
WITHOUT_EE=YES
WITHOUT_EXAMPLES=YES
WITHOUT_FILE=YES
WITHOUT_FINGER=YES
WITHOUT_FLOPPY=YES
WITHOUT_FORTH=YES
WITHOUT_FREEBSD_UPDATE=YES
WITHOUT_GAMES=YES
WITHOUT_GNU_DIFF=YES
WITHOUT_GPIO=YES
WITHOUT_HAST=YES
WITHOUT_HTML=YES
WITHOUT_HYPERV=YES
WITHOUT_ICONV=YES
WITHOUT_INCLUDES=YES
WITHOUT_INETD=YES
WITHOUT_IPFILTER=YES
WITHOUT_IPFW=YES
WITHOUT_ISCSI=YES
WITHOUT_JAIL=YES
WITHOUT_KDUMP=YES
WITHOUT_KERBEROS=YES
WITHOUT_KERNEL_SYMBOLS=YES
WITHOUT_LIB32=YES
WITHOUT_LOADER_PXEBOOT=YES
WITHOUT_LOCALES=YES
WITHOUT_LOCATE=YES
WITHOUT_LPR=YES
WITHOUT_LS_COLORS=YES
WITHOUT_MAIL=YES
WITHOUT_MAKE=YES
WITHOUT_MAN=YES
WITHOUT_MLX5TOOL=YES
WITHOUT_NETGRAPH=YES
WITHOUT_NETLINK=YES
WITHOUT_NIS=YES
WITHOUT_NLS=YES
WITHOUT_NS_CACHING=YES
WITHOUT_NTP=YES
WITHOUT_NUAGEINIT=YES
WITHOUT_OFED=YES
WITHOUT_OPENMP=YES
WITHOUT_PF=YES
WITHOUT_PPP=YES
WITHOUT_QUOTAS=YES
WITHOUT_RADIUS_SUPPORT=YES
WITHOUT_RBOOTD=YES
WITHOUT_RESCUE=YES
WITHOUT_ROUTED=YES
WITHOUT_SCTP_SUPPORT=YES
WITHOUT_SERVICESDB=YES
WITHOUT_SHAREDOCS=YES
WITHOUT_SOUND=YES
WITHOUT_STATS=YES
WITHOUT_SYSCONS=YES
WITHOUT_TALK=YES
WITHOUT_TCP_WRAPPERS=YES
WITHOUT_TCSH=YES
WITHOUT_TESTS=YES
WITHOUT_TEXTPROC=YES
WITHOUT_TFTP=YES
WITHOUT_TOOLCHAIN=YES
WITHOUT_UTMPX=YES
WITHOUT_WIRELESS=YES
WITHOUT_ZONEINFO=YES

To trim it down even further, remove OpenSSH server and client

WITHOUT_OPENSSH=YES

And if bsdinstall bsdinstall(8) is not needed

WITHOUT_BSDINSTALL=YES

Build Release

cd /usr/src
make -j4 buildworld buildkernel > /tmp/build.log

Create binary install sets, skipping pkgbase, sources and ports

cd /usr/src/release
make clean
make release -DNODOC -DNOSRC -DNOPORTS -DNOPKGBASE -DPKG_ABI=FreeBSD:15:amd64 > /tmp/release.log
make install DESTDIR=/usr/obj/release -DNOPKGBASE

The final size is now one fourth the standard distribution size (~200MB)

# ls -alh /usr/obj/usr/src/amd64.amd64/release/*.txz
-rw-r--r--  1 root wheel   31M Nov 18 14:24 /usr/obj/usr/src/amd64.amd64/release/base.txz
-rw-r--r--  1 root wheel   10M Nov 18 14:24 /usr/obj/usr/src/amd64.amd64/release/kernel.txz

Custom initrd

Create a root file system and mount

truncate -s 160M /root/mfsroot
mdconfig -f /root/mfsroot
newfs -U /dev/md0

mkdir -p /mnt/mfsroot
mount /dev/md0 /mnt/mfsroot
mkdir /root/tftpboot

Extract FreeBSD sets to construct the TFTP boot tree and root file system:

rootfs Exclude static libraries
rootfs Exclude second and thrid level boot loader files
tftpboot Load kernel and kernel at boot time
tftpboot Compress kernel and loadable modules
tar -xpf /usr/obj/usr/src/amd64.amd64/release/base.txz \
    --exclude '*.a' \
    --exclude 'boot/*' \
    -C /mnt/mfsroot

tar -xpf /usr/obj/usr/src/amd64.amd64/release/base.txz \
    --include 'boot/defaults/*' \
    --include 'boot/kernel/*' \
    --include 'boot/lua/*' \
    --include 'boot/loader.efi' \
    --exclude 'gfx-*.lua' \
    -C /root/tftpboot

tar -xpf /usr/obj/usr/src/amd64.amd64/release/kernel.txz \
    -C /root/tftpboot

gzip -9 /root/tftpboot/boot/kernel/*

Configure the loader to fetch and mount root on md0

cat > /root/tftpboot/boot/loader.conf <<EOF
mfs_load="YES"
mfs_type="mfs_root"
mfs_name="/mfsroot"
beastie_disable="YES"
autoboot_delay="-1"
zfs_load="YES"
EOF

Mount /tmp on a separate memory file system to allow for large downloads such as installation sets

cat > /mnt/mfsroot/etc/fstab <<EOF
tmpfs   /tmp    tmpfs   rw,mode=01777   0   0
EOF

Install utilities from packages

pkg -r /mnt/mfsroot install -y tmux dhcpcd

Enable automatic network autoconfiguration

cat > /mnt/mfsroot/etc/rc.conf <<EOF
dhcpcd_enable="YES"
EOF

Use hardware MAC for IPv6 autoconfiguration and set hostname based on DHCP reply

cat >> /mnt/mfsroot/usr/local/etc/dhcpcd.conf <<EOF
slaac hdaddr
env force_hostname=YES
EOF

Set login message

echo > /mnt/mfsroot/etc/motd.template

Unmount root image and compress

umount /mnt/mfsroot
mdconfig -u 0 -d
mv /root/mfsroot /root/tftpboot/
gzip -9 /root/tftpboot/mfsroot

Create distribution for TFTP server

tar -C /root/tftpboot -cvf freebsd-rd-15.1-amd64.tar .
rm -r /root/tftpboot

DHCP/PXE

Build from source or download prebuild image: freebsd-rd-15.1-amd64.tar

Install

mkdir /tftpboot/freebsd-rd-15.1
tar -xvf freebsd-rd-15.1-amd64.tar -C /tftpboot/freebsd-rd-15.1

Configure DHCP

# /etc/dhcpd.conf
subnet 192.168.2.0 netmask 255.255.255.0 {
    # ...
    next-server 192.168.0.2;

    host FS1 {
      hardware ethernet 00:16:3e:17:92:73;
      fixed-address 192.168.2.29;
      option root-path "tftp://192.168.0.2/freebsd-rd-15.1";
      filename "freebsd-rd-15.1-amd64/boot/loader.efi";
      option host-name "fs1";
   }
}

Implementation Notes

  1. The compressed size of the rootfs cannot be reduced after installing sets and installing packages. Unlike the Linux initrd which expands contents using cpio(1), FreeBSD mounts a UFS file system, and contents on disk are not zeroed by rm(1).
  2. iwm firmware is installed even WITHOUT_WIRELESS was set.
  3. Since 15.0, "make release" with NOPKGBASE=1 fails requring some manual workarounds.
  4. Sometimes root-path must be set in the iPXE shell
    chain tftp://${next-server}/freebsd-rd-15.1/boot/loader.efi dhcp.root-path=${root-path}