Development discussion for sr.ht. When contributing patches to sr.ht, please edit the [PATCH] line to include the specific sr.ht project you're contributing to, e.g.

[PATCH lists.sr.ht v2] Add thing to stuff
6 3

[PATCH builds.sr.ht] images/freebsd: add a custom install script

Details
Message ID
<hmIDEuu5vNixhWyZZm4gFjJ3bpU6dqhz9i1RtzJCBJrE5oW5qNG-etocRr26Orrh53s65jx6bPyor51x4OMbDiaDysOPwXf37g0UNuIlgP4=@emersion.fr>
Sender timestamp
1549881337
DKIM signature
pass
Download raw message
Patch: +81 -64
From: emersion <contact@emersion.fr>

---

References:

- https://lists.freebsd.org/pipermail/freebsd-hackers/2019-February/054030.html
- https://hackmd.io/s/SJRD7QRNE#

 images/freebsd/12.0/genimg |   2 +-
 images/freebsd/genimg      | 108 +++++++++++++++++++++++++++----------
 images/freebsd/install     |  35 ------------
 3 files changed, 81 insertions(+), 64 deletions(-)
 delete mode 100644 images/freebsd/install

diff --git a/images/freebsd/12.0/genimg b/images/freebsd/12.0/genimg
index b85d949..13f136c 100755
--- a/images/freebsd/12.0/genimg
+++ b/images/freebsd/12.0/genimg
@@ -1,3 +1,3 @@
 #!/bin/sh
-export release=12.0
+export release=12.0-RELEASE
 exec ../genimg "$@"
diff --git a/images/freebsd/genimg b/images/freebsd/genimg
index 930b09f..3365cce 100755
--- a/images/freebsd/genimg
+++ b/images/freebsd/genimg
@@ -3,42 +3,94 @@
 echo "$release" >/dev/null # fail on -u if release unset
 arch="${1:-amd64}"
 
-export DISTRIBUTIONS="kernel.txz base.txz ports.txz"
-export BSDINSTALL_DISTSITE="https://download.freebsd.org/ftp/releases/$arch/$release/"
-export BSDINSTALL_DISTDIR="/usr/freebsd-dist/$arch/$release"
-export BSDINSTALL_LOG="/tmp/bsdinstall.log"
+dist_base="https://download.freebsd.org/ftp/releases/$arch/$release"
+dist_files="kernel.txz base.txz ports.txz"
+dist_dir="/usr/freebsd-dist/$arch/$release"
+
+mkdir -p "$dist_dir"
+for f in $dist_files
+do
+	curl -C - -o "$dist_dir/$f" "$dist_base/$f"
+done
 
 cleanup() {
-	cat "$BSDINSTALL_LOG" || true
-	# The order here is important if you don't want to hose your mounts
-	umount -f /mnt/dev/pts 2>/dev/null || true
-	umount -f /mnt/dev/shm 2>/dev/null || true
-	umount -f /mnt/dev 2>/dev/null || true
-	umount -f /mnt/proc 2>/dev/null || true
-	umount -f /mnt/run 2>/dev/null || true
-	umount -f /mnt/sys 2>/dev/null || true
-	umount -f /mnt/boot 2>/dev/null || true
-	umount -f /mnt 2>/dev/null || true 
-	mdconfig -d -u 0
+	sync || true
+	umount /mnt/dev || true
+	umount /mnt || true
+	mdconfig -du md0 || true
 }
-
-# qemu-nbd is not supported, so we just create a raw image and convert it
-rm -f "$BSDINSTALL_LOG"
-qemu-img create -f raw root.img.raw 6G
-mdconfig -a -t vnode -f root.img.raw -u 0
 trap cleanup EXIT
 
-export nonInteractive=YES
-mkdir -p "$BSDINSTALL_DISTDIR"
-bsdinstall distfetch
-bsdinstall checksum
-bsdinstall script ./install
+rm -f disk.img
+truncate -s 4G disk.img
+mdconfig -a -t vnode -f disk.img -u md0
+gpart create -s gpt /dev/md0
+gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0
+gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md0
+gpart add -t freebsd-ufs -l rootfs -b 1M -s 3G md0
+newfs -U /dev/md0p2
+
+mount /dev/md0p2 /mnt
+mkdir -p /mnt/dev
+mount -t devfs devfs /mnt/dev
+
+run_root() {
+	chroot /mnt /bin/sh -c "$*"
+}
+
+for f in $dist_files
+do
+	tar -C /mnt -xJf "$dist_dir/$f"
+done
+
+echo "/dev/ada0p2 / ufs rw,noatime 1 1" >/mnt/etc/fstab
+touch /mnt/firstboot
+echo 'autoboot_delay="-1"' >>/mnt/boot/loader.conf
+
+cat >>/mnt/etc/rc.conf <<EOF
+ntpd_enable=YES
+sshd_enable=YES
+growfs_enable=YES
+ifconfig_em0="inet 10.0.2.15 netmask 255.255.255.0"
+defaultrouter="10.0.2.2"
+EOF
+echo "nameserver 1.1.1.1" >/mnt/etc/resolv.conf
+tzsetup -s -C /mnt UTC
+
+cat >>/mnt/etc/ssh/sshd_config <<EOF
+PermitRootLogin yes
+PasswordAuthentication yes
+PermitEmptyPasswords yes
+EOF
+
+mkdir -p /mnt/usr/local/etc/pkg/repos/
+cat >/mnt/usr/local/etc/pkg/repos/FreeBSD.conf <<EOF
+FreeBSD: {
+	url: pkg+http://pkg.FreeBSD.org/\$\{ABI\}/latest
+	enabled: yes
+}
+EOF
+
+/usr/sbin/freebsd-update -b /mnt \
+	--currently-running 12.0-RELEASE \
+	--not-running-from-cron \
+	fetch install \
+	>/dev/null
+
+env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f
+# TODO: remove bash
+env ASSUME_ALWAYS_YES=YES pkg -c /mnt install git bash sudo curl
+
+echo "build:1000:::::::/usr/local/bin/bash:" | run_root adduser -w none -f -
+run_root pw groupadd sudo
+run_root pw groupmod sudo -M build
+echo "%sudo ALL=(ALL) NOPASSWD: ALL" >>/mnt/usr/local/etc/sudoers
 
-trap - EXIT
 cleanup
+trap : EXIT
 
-qemu-img convert -f raw -O qcow2 root.img.raw root.img.qcow2
-rm root.img.raw
+qemu-img convert -f raw -O qcow2 disk.img root.img.qcow2
+rm disk.img
 
 # Filesystem will be enlarged by growfs(7) on next startup
 qemu-img resize root.img.qcow2 16G
diff --git a/images/freebsd/install b/images/freebsd/install
deleted file mode 100644
index f9565df..0000000
--- a/images/freebsd/install
@@ -1,35 +0,0 @@
-PARTITIONS="md0 GPT { 512K freebsd-boot, 2G freebsd-swap, auto freebsd-ufs / }"
-DISTRIBUTIONS="base.txz kernel.txz ports.txz"
-
-#!/bin/sh -eux
-
-sysrc ntpd_enable=YES
-sysrc sshd_enable=YES
-sysrc growfs_enable=YES
-sysrc 'ifconfig_em0="inet 10.0.2.15 netmask 255.255.255.0"'
-sysrc 'defaultrouter="10.0.2.2"'
-echo "nameserver 1.1.1.1" >/etc/resolv.conf
-echo 'autoboot_delay="-1"' >>/boot/loader.conf
-
-# Device names at runtime are different from the ones at install time
-sed -e s:/dev/md0:/dev/ada0: -i .orig /etc/fstab
-
-# Run firstboot scripts on next startup
-touch /firstboot
-
-cat <<EOF >>/etc/ssh/sshd_config
-PermitRootLogin yes
-PasswordAuthentication yes
-PermitEmptyPasswords yes
-EOF
-
-ln -sf /usr/share/zoneinfo/UTC /etc/localtime
-
-# TODO: remove bash
-pkg install -y git bash sudo curl
-
-echo "build:1000:::::::/usr/local/bin/bash:" | adduser -w none -f -
-pw groupadd sudo
-pw groupmod sudo -M build
-
-echo "%sudo ALL=(ALL) NOPASSWD: ALL" >>/usr/local/etc/sudoers
-- 
2.20.1
Details
Message ID
<1549888091.3610553.1655453736.547353B3@webmail.messagingengine.com>
In-Reply-To
<hmIDEuu5vNixhWyZZm4gFjJ3bpU6dqhz9i1RtzJCBJrE5oW5qNG-etocRr26Orrh53s65jx6bPyor51x4OMbDiaDysOPwXf37g0UNuIlgP4=@emersion.fr> (view parent)
Sender timestamp
1549888091
DKIM signature
pass
Download raw message
On Mon, 11 Feb 2019, at 11:35, Simon Ser wrote:
> From: emersion <contact@emersion.fr>

LGTM Simon! A few comments follow.

> +dist_files="kernel.txz base.txz ports.txz"
> +dist_dir="/usr/freebsd-dist/$arch/$release"
> +
> +mkdir -p "$dist_dir"
> +for f in $dist_files
> +do
> +	curl -C - -o "$dist_dir/$f" "$dist_base/$f"
> +done

curl is not part of base but fetch(1) is, your call if you want to change it.

Do we need to verify checksums? If so, we would need to import &
compare against https://ftp.freebsd.org/pub/FreeBSD/releases/amd64/amd64/12.0-RELEASE/MANIFEST

We are using latest pkg repo but older downloaded ports.txz.  These should
probably be the same? Either is fine, but I think sr.ht users are more likely
to be current ports users. Personally I'd not bundle ports at all here, what
do you think?

> +echo "build:1000:::::::/usr/local/bin/bash:" | run_root adduser -w none -f -

Could use (after groupadd are done)

pw -R /mnt useradd -n build -u 1000  -s /usr/local/bin/bash -m -w none -G sudo

> +run_root pw groupadd sudo
> +run_root pw groupmod sudo -M build
> +echo "%sudo ALL=(ALL) NOPASSWD: ALL" >>/mnt/usr/local/etc/sudoers

could we avoid the run_root function entirely & need to mount devfs inside
by using pw(8)'s `-R rootdir` flag? Most of the core FreeBSD admin tools
are chroot/jail aware in some form.

A+
Dave
Details
Message ID
<kmKToTEX4CalWpzVShmfUN0MuQkZeQ3iLiEZmCv1dqkVnpEitqOJ9pM15Ed6yEC6i4kU9HLxxNcn2ixu_shko7-OgGLoaJmvDDcXjRH3_j8=@emersion.fr>
In-Reply-To
<1549888091.3610553.1655453736.547353B3@webmail.messagingengine.com> (view parent)
Sender timestamp
1549898617
DKIM signature
pass
Download raw message
On Monday, February 11, 2019 1:28 PM, Dave Cottlehuber <dch@skunkwerks.at> wrote:
> On Mon, 11 Feb 2019, at 11:35, Simon Ser wrote:
>
> > From: emersion contact@emersion.fr
>
> LGTM Simon! A few comments follow.

Thanks!

> > +dist_files="kernel.txz base.txz ports.txz"
> > +dist_dir="/usr/freebsd-dist/$arch/$release"
> > +
> > +mkdir -p "$dist_dir"
> > +for f in $dist_files
> > +do
> >
> > -   curl -C - -o "$dist_dir/$f" "$dist_base/$f"
> >     +done
> >
>
> curl is not part of base but fetch(1) is, your call if you want to change it.

Switched to fetch(1).

> Do we need to verify checksums? If so, we would need to import &
> compare against https://ftp.freebsd.org/pub/FreeBSD/releases/amd64/amd64/12.0-RELEASE/MANIFEST

I personally think using HTTPS is enough.

> We are using latest pkg repo but older downloaded ports.txz. These should
> probably be the same? Either is fine, but I think sr.ht users are more likely
> to be current ports users. Personally I'd not bundle ports at all here, what
> do you think?

I've tried the svn route, checking out ports takes a while. Maybe it's
fine, or maybe we don't want to fetch ports after all -- users could
selectively fetch what they need.

> > +echo "build:1000:::::::/usr/local/bin/bash:" | run_root adduser -w none -f -
>
> Could use (after groupadd are done)
>
> pw -R /mnt useradd -n build -u 1000 -s /usr/local/bin/bash -m -w none -G sudo
>
> > +run_root pw groupadd sudo
> > +run_root pw groupmod sudo -M build
> > +echo "%sudo ALL=(ALL) NOPASSWD: ALL" >>/mnt/usr/local/etc/sudoers
>
> could we avoid the run_root function entirely & need to mount devfs inside
> by using pw(8)'s `-R rootdir` flag? Most of the core FreeBSD admin tools
> are chroot/jail aware in some form.

These two suggestions are indeed better!
Details
Message ID
<20190211152438.GC1614@homura.localdomain>
In-Reply-To
<kmKToTEX4CalWpzVShmfUN0MuQkZeQ3iLiEZmCv1dqkVnpEitqOJ9pM15Ed6yEC6i4kU9HLxxNcn2ixu_shko7-OgGLoaJmvDDcXjRH3_j8=@emersion.fr> (view parent)
Sender timestamp
1549898678
DKIM signature
pass
Download raw message
On 2019-02-11  3:23 PM, Simon Ser wrote:
> I've tried the svn route, checking out ports takes a while. Maybe it's
> fine, or maybe we don't want to fetch ports after all -- users could
> selectively fetch what they need.

How big is a checked-out ports tree? Is it available as a tarball rather
than via SVN? I'd like to have first-class ports support if possible. On
NetBSD I have first-class pkgsrc support via the packages array.
Details
Message ID
<jAkOwYSSiypq0dBfadNl4j1cOAhFGILisgRoSUxFP642E-Dq6OcYEX8Zuhdqwr2c38LDEcXytXp0ISLn-R3oFb7mtJsJYiSkzKe0Vn4jpnM=@emersion.fr>
In-Reply-To
<20190211152438.GC1614@homura.localdomain> (view parent)
Sender timestamp
1549899258
DKIM signature
pass
Download raw message
On Monday, February 11, 2019 4:24 PM, Drew DeVault <sir@cmpwn.com> wrote:
> On 2019-02-11 3:23 PM, Simon Ser wrote:
>
> > I've tried the svn route, checking out ports takes a while. Maybe it's
> > fine, or maybe we don't want to fetch ports after all -- users could
> > selectively fetch what they need.
>
> How big is a checked-out ports tree? Is it available as a tarball rather
> than via SVN? I'd like to have first-class ports support if possible. On
> NetBSD I have first-class pkgsrc support via the packages array.

It's 1.7G.

Actually it seems there is a tool called portsnap(8), I'll try to see if
it's faster.
Details
Message ID
<1549899899.3692503.1655587472.0B86E396@webmail.messagingengine.com>
In-Reply-To
<jAkOwYSSiypq0dBfadNl4j1cOAhFGILisgRoSUxFP642E-Dq6OcYEX8Zuhdqwr2c38LDEcXytXp0ISLn-R3oFb7mtJsJYiSkzKe0Vn4jpnM=@emersion.fr> (view parent)
Sender timestamp
1549899899
DKIM signature
pass
Download raw message
On Mon, 11 Feb 2019, at 16:34, Simon Ser wrote:
> On Monday, February 11, 2019 4:24 PM, Drew DeVault <sir@cmpwn.com> wrote:
> > On 2019-02-11 3:23 PM, Simon Ser wrote:
> >
> > > I've tried the svn route, checking out ports takes a while. Maybe it's
> > > fine, or maybe we don't want to fetch ports after all -- users could
> > > selectively fetch what they need.
> >
> > How big is a checked-out ports tree? Is it available as a tarball rather
> > than via SVN? I'd like to have first-class ports support if possible. On
> > NetBSD I have first-class pkgsrc support via the packages array.
> 
> It's 1.7G.
> 
> Actually it seems there is a tool called portsnap(8), I'll try to see if
> it's faster.

http://ftp.freebsd.org/pub/FreeBSD/snapshots/amd64/13.0-CURRENT/ports.txz  or http://ftp.freebsd.org/pub/FreeBSD/snapshots/amd64/amd64/12.0-STABLE/ports.txz as you prefer are ideal.

~ 40MiB download -> ~ 800MiB unpacked on disk.

A+
Dave
Details
Message ID
<D1_fspxyQS_hfQ2MP3iuBHSV-Y1NFbsol4CX70zrunrB_X0fV0RAsQkxRNglz2lRoOerkADLc94xDqlSI5gbVw2ODRaTb7hWE_Fedcgs3ts=@emersion.fr>
In-Reply-To
<1549899899.3692503.1655587472.0B86E396@webmail.messagingengine.com> (view parent)
Sender timestamp
1549900419
DKIM signature
pass
Download raw message
On Monday, February 11, 2019 4:44 PM, Dave Cottlehuber <dch@skunkwerks.at> wrote:
> http://ftp.freebsd.org/pub/FreeBSD/snapshots/amd64/13.0-CURRENT/ports.txz or http://ftp.freebsd.org/pub/FreeBSD/snapshots/amd64/amd64/12.0-STABLE/ports.txz as you prefer are ideal.
>
> ~ 40MiB download -> ~ 800MiB unpacked on disk.

It seems portsnap gets a more up-to-date snapshot (Feb 11 instead of
Feb 7).

Alternatively we could use this one:

    http://ftp.freebsd.org/pub/FreeBSD/ports/ports/ports.tar.gz