Update content for html

17 files changed, 13585 insertions, 737 deletions

diff --git a/gemfeed/2016-04-09-jails-and-zfs-on-freebsd-with-puppet.html b/gemfeed/2016-04-09-jails-and-zfs-on-freebsd-with-puppet.html
index e030a2be..d5f81d05 100644
--- a/gemfeed/2016-04-09-jails-and-zfs-on-freebsd-with-puppet.html
+++ b/gemfeed/2016-04-09-jails-and-zfs-on-freebsd-with-puppet.html
@@ -413,6 +413,7 @@ Notice: Finished catalog run in 206.09 seconds
 <br />
 <span>Other *BSD related posts are:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
diff --git a/gemfeed/2022-07-30-lets-encrypt-with-openbsd-and-rex.html b/gemfeed/2022-07-30-lets-encrypt-with-openbsd-and-rex.html
index 7ad59bf4..2e8782f8 100644
--- a/gemfeed/2022-07-30-lets-encrypt-with-openbsd-and-rex.html
+++ b/gemfeed/2022-07-30-lets-encrypt-with-openbsd-and-rex.html
@@ -692,6 +692,7 @@ rex commons
 <br />
 <span>Other *BSD related posts are:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
diff --git a/gemfeed/2024-01-13-one-reason-why-i-love-openbsd.html b/gemfeed/2024-01-13-one-reason-why-i-love-openbsd.html
index f60200f2..78a1d5d5 100644
--- a/gemfeed/2024-01-13-one-reason-why-i-love-openbsd.html
+++ b/gemfeed/2024-01-13-one-reason-why-i-love-openbsd.html
@@ -70,6 +70,7 @@ $ doas reboot <i><font color="silver"># Just in case, reboot one more time</font
 <br />
 <span>Other *BSD related posts are:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
diff --git a/gemfeed/2024-04-01-KISS-high-availability-with-OpenBSD.html b/gemfeed/2024-04-01-KISS-high-availability-with-OpenBSD.html
index e2a81eeb..5840974c 100644
--- a/gemfeed/2024-04-01-KISS-high-availability-with-OpenBSD.html
+++ b/gemfeed/2024-04-01-KISS-high-availability-with-OpenBSD.html
@@ -331,6 +331,7 @@ http://www.gnu.org/software/src-highlite -->
 <br />
 <span>Other *BSD and KISS related posts are:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
diff --git a/gemfeed/2024-11-17-f3s-kubernetes-with-freebsd-part-1.html b/gemfeed/2024-11-17-f3s-kubernetes-with-freebsd-part-1.html
index 5468cd1d..8382e37b 100644
--- a/gemfeed/2024-11-17-f3s-kubernetes-with-freebsd-part-1.html
+++ b/gemfeed/2024-11-17-f3s-kubernetes-with-freebsd-part-1.html
@@ -21,6 +21,7 @@
 <br />
 <span>These are all the posts so far:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage (You are currently reading this)</a><br />
@@ -47,7 +48,7 @@
 <li>⇢ <a href='#monitoring-keeping-an-eye-on-everything'>Monitoring: Keeping an eye on everything</a></li>
 <li>⇢ ⇢ <a href='#prometheus-and-grafana'>Prometheus and Grafana</a></li>
 <li>⇢ ⇢ <a href='#gogios-my-custom-alerting-system'>Gogios: My custom alerting system</a></li>
-<li>⇢ <a href='#what-s-after-this-all'>What&#39;s after this all?</a></li>
+<li>⇢ <a href='#conclusion'>Conclusion</a></li>
 </ul><br />
 <h2 style='display: inline' id='why-this-setup'>Why this setup?</h2><br />
 <br />
@@ -157,7 +158,7 @@
 <br />
 <span>Ironically, I implemented Gogios to avoid using more complex alerting systems like Prometheus, but here we go—it integrates well now.</span><br />
 <br />
-<h2 style='display: inline' id='what-s-after-this-all'>What&#39;s after this all?</h2><br />
+<h2 style='display: inline' id='conclusion'>Conclusion</h2><br />
 <br />
 <span>This setup may be just the beginning. Some ideas I&#39;m thinking about for the future:</span><br />
 <br />
@@ -175,6 +176,7 @@
 <br />
 <span>Other *BSD-related posts:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage (You are currently reading this)</a><br />
diff --git a/gemfeed/2024-12-03-f3s-kubernetes-with-freebsd-part-2.html b/gemfeed/2024-12-03-f3s-kubernetes-with-freebsd-part-2.html
index 1ecf2f2c..361ab8ec 100644
--- a/gemfeed/2024-12-03-f3s-kubernetes-with-freebsd-part-2.html
+++ b/gemfeed/2024-12-03-f3s-kubernetes-with-freebsd-part-2.html
@@ -21,6 +21,7 @@
 <br />
 <span>These are all the posts so far:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation (You are currently reading this)</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
@@ -354,6 +355,7 @@ dev.cpu.<font color="#000000">0</font>.freq: <font color="#000000">2922</font>
 <br />
 <span>Other *BSD-related posts:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation (You are currently reading this)</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
diff --git a/gemfeed/2025-02-01-f3s-kubernetes-with-freebsd-part-3.html b/gemfeed/2025-02-01-f3s-kubernetes-with-freebsd-part-3.html
index b9116a4d..45841713 100644
--- a/gemfeed/2025-02-01-f3s-kubernetes-with-freebsd-part-3.html
+++ b/gemfeed/2025-02-01-f3s-kubernetes-with-freebsd-part-3.html
@@ -20,6 +20,7 @@
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts (You are currently reading this)</a><br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <br />
 <a href='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png'><img alt='f3s logo' title='f3s logo' src='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png' /></a><br />
 <br />
@@ -41,6 +42,7 @@
 <li>⇢ <a href='#power-outage-simulation'>Power outage simulation</a></li>
 <li>⇢ ⇢ <a href='#pulling-the-plug'>Pulling the plug</a></li>
 <li>⇢ ⇢ <a href='#restoring-power'>Restoring power</a></li>
+<li>⇢ <a href='#conclusion'>Conclusion</a></li>
 </ul><br />
 <h2 style='display: inline' id='introduction'>Introduction</h2><br />
 <br />
@@ -402,10 +404,17 @@ Jan 26 17:36:32 f2 apcupsd[2159]: apcupsd exiting, signal 15
 Jan 26 17:36:32 f2 apcupsd[2159]: apcupsd shutdown succeeded
 </pre>
 <br />
-<span>All good :-) See you in the next post of this series!</span><br />
+<span>All good :-)</span><br />
+<br />
+<h2 style='display: inline' id='conclusion'>Conclusion</h2><br />
+<br />
+<span>I have the same UPS (but with a bit more capacity) for my main work setup, which powers my 28" screen, music equipment, etc. It has already been helpful a couple of times during power outages here, so I am sure that the smaller UPS for the F3s setup will be of great use.</span><br />
+<br />
+<span>See you in the next post of this series!</span><br />
 <br />
 <span>Other BSD related posts are:</span><br />
 <br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a><br />
 <a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts (You are currently reading this)</a><br />
 <a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
 <a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
diff --git a/gemfeed/2025-02-08-random-weird-things-ii.html b/gemfeed/2025-02-08-random-weird-things-ii.html
index 770e287a..03f2d8de 100644
--- a/gemfeed/2025-02-08-random-weird-things-ii.html
+++ b/gemfeed/2025-02-08-random-weird-things-ii.html
@@ -73,7 +73,7 @@
 <br />
 <h3 style='display: inline' id='13-go-functions-can-have-methods'>13. Go functions can have methods</h3><br />
 <br />
-<span>Functions on struct types? Well, know. Functions on types like <span class='inlinecode'>int</span> and <span class='inlinecode'>string</span>? It&#39;s also known of, but a bit lesser. Functions on function types? That sounds a bit funky, but it&#39;s possible, too! For demonstration, have a look at this snippet:</span><br />
+<span>Functions on struct types? Well known. Functions on types like <span class='inlinecode'>int</span> and <span class='inlinecode'>string</span>? It&#39;s also known of, but a bit lesser. Functions on function types? That sounds a bit funky, but it&#39;s possible, too! For demonstration, have a look at this snippet:</span><br />
 <br />
 <!-- Generator: GNU source-highlight 3.1.9
 by Lorenzo Bettini
@@ -119,7 +119,7 @@ http://www.gnu.org/software/src-highlite -->
 <br />
 <h3 style='display: inline' id='14--and-ss-are-treated-the-same'>14. ß and ss are treated the same</h3><br />
 <br />
-<span>Know German? In German, the letter "sarp s" is written as ß. ß is treated the same as ss on macOS.</span><br />
+<span>Know German? In German, the letter "sharp s" is written as ß. ß is treated the same as ss on macOS.</span><br />
 <br />
 <span>On a case-insensitive file system like macOS, not only are uppercase and lowercase letters treated the same, but non-Latin characters like the German "ß" are also considered equivalent to their Latin counterparts (in this case, "ss").</span><br />
 <br />
diff --git a/gemfeed/2025-04-05-f3s-kubernetes-with-freebsd-part-4.html b/gemfeed/2025-04-05-f3s-kubernetes-with-freebsd-part-4.html
new file mode 100644
index 00000000..b7d665c9
--- /dev/null
+++ b/gemfeed/2025-04-05-f3s-kubernetes-with-freebsd-part-4.html
@@ -0,0 +1,609 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<title>f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</title>
+<link rel="shortcut icon" type="image/gif" href="/favicon.ico" />
+<link rel="stylesheet" href="../style.css" />
+<link rel="stylesheet" href="style-override.css" />
+</head>
+<body>
+<p class="header">
+<a href="https://foo.zone">Home</a> | <a href="https://codeberg.org/snonux/foo.zone/src/branch/content-md/gemfeed/2025-04-05-f3s-kubernetes-with-freebsd-part-4.md">Markdown</a> | <a href="gemini://foo.zone/gemfeed/2025-04-05-f3s-kubernetes-with-freebsd-part-4.gmi">Gemini</a>
+</p>
+<h1 style='display: inline' id='f3s-kubernetes-with-freebsd---part-4-rocky-linux-bhyve-vms'>f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</h1><br />
+<br />
+<span class='quote'>Published at 2025-04-04T23:21:01+03:00</span><br />
+<br />
+<span>This is the fourth blog post about the f3s series for self-hosting demands in a home lab. f3s? The "f" stands for FreeBSD, and the "3s" stands for k3s, the Kubernetes distribution used on FreeBSD-based physical machines.</span><br />
+<br />
+<a class='textlink' href='./2024-11-17-f3s-kubernetes-with-freebsd-part-1.html'>2024-11-17 f3s: Kubernetes with FreeBSD - Part 1: Setting the stage</a><br />
+<a class='textlink' href='./2024-12-03-f3s-kubernetes-with-freebsd-part-2.html'>2024-12-03 f3s: Kubernetes with FreeBSD - Part 2: Hardware and base installation</a><br />
+<a class='textlink' href='./2025-02-01-f3s-kubernetes-with-freebsd-part-3.html'>2025-02-01 f3s: Kubernetes with FreeBSD - Part 3: Protecting from power cuts</a><br />
+<a class='textlink' href='./2025-04-05-f3s-kubernetes-with-freebsd-part-4.html'>2025-04-05 f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs (You are currently reading this)</a><br />
+<br />
+<a href='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png'><img alt='f3s logo' title='f3s logo' src='./f3s-kubernetes-with-freebsd-part-1/f3slogo.png' /></a><br />
+<br />
+<h2 style='display: inline' id='table-of-contents'>Table of Contents</h2><br />
+<br />
+<ul>
+<li><a href='#f3s-kubernetes-with-freebsd---part-4-rocky-linux-bhyve-vms'>f3s: Kubernetes with FreeBSD - Part 4: Rocky Linux Bhyve VMs</a></li>
+<li>⇢ <a href='#introduction'>Introduction</a></li>
+<li>⇢ <a href='#check-for-popcnt-cpu-support'>Check for <span class='inlinecode'>POPCNT</span> CPU support</a></li>
+<li>⇢ <a href='#basic-bhyve-setup'>Basic Bhyve setup</a></li>
+<li>⇢ <a href='#rocky-linux-vms'>Rocky Linux VMs</a></li>
+<li>⇢ ⇢ <a href='#iso-download'>ISO download</a></li>
+<li>⇢ ⇢ <a href='#vm-configuration'>VM configuration</a></li>
+<li>⇢ ⇢ <a href='#vm-installation'>VM installation</a></li>
+<li>⇢ ⇢ <a href='#increase-of-the-disk-image'>Increase of the disk image</a></li>
+<li>⇢ ⇢ <a href='#connect-to-vnc'>Connect to VNC</a></li>
+<li>⇢ <a href='#after-install'>After install</a></li>
+<li>⇢ ⇢ <a href='#vm-auto-start-after-host-reboot'>VM auto-start after host reboot</a></li>
+<li>⇢ ⇢ <a href='#static-ip-configuration'>Static IP configuration</a></li>
+<li>⇢ ⇢ <a href='#permitting-root-login'>Permitting root login</a></li>
+<li>⇢ ⇢ <a href='#install-latest-updates'>Install latest updates</a></li>
+<li>⇢ <a href='#stress-testing-cpu'>Stress testing CPU</a></li>
+<li>⇢ ⇢ <a href='#silly-freebsd-host-benchmark'>Silly FreeBSD host benchmark</a></li>
+<li>⇢ ⇢ <a href='#silly-rocky-linux-vm--bhyve-benchmark'>Silly Rocky Linux VM @ Bhyve benchmark</a></li>
+<li>⇢ ⇢ <a href='#silly-freebsd-vm--bhyve-benchmark'>Silly FreeBSD VM @ Bhyve benchmark</a></li>
+<li>⇢ <a href='#benchmarking-with-ubench'>Benchmarking with <span class='inlinecode'>ubench</span></a></li>
+<li>⇢ ⇢ <a href='#freebsd-host-ubench-benchmark'>FreeBSD host <span class='inlinecode'>ubench</span> benchmark</a></li>
+<li>⇢ ⇢ <a href='#freebsd-vm--bhyve-ubench-benchmark'>FreeBSD VM @ Bhyve <span class='inlinecode'>ubench</span> benchmark</a></li>
+<li>⇢ ⇢ <a href='#rocky-linux-vm--bhyve-ubench-benchmark'>Rocky Linux VM @ Bhyve <span class='inlinecode'>ubench</span> benchmark</a></li>
+<li>⇢ <a href='#conclusion'>Conclusion</a></li>
+</ul><br />
+<h2 style='display: inline' id='introduction'>Introduction</h2><br />
+<br />
+<span>In this blog post, we are going to install the Bhyve hypervisor.</span><br />
+<br />
+<span>The FreeBSD Bhyve hypervisor is a lightweight, modern hypervisor that enables virtualization on FreeBSD systems. Bhyve&#39;s strengths include its minimal overhead, which allows it to achieve near-native performance for virtual machines. It&#39;s efficient and lightweight, leveraging the capabilities of the FreeBSD operating system for performance and network management.</span><br />
+<br />
+<span>Bhyve supports running various guest operating systems, including FreeBSD, Linux, and Windows, on hardware platforms that support hardware virtualization extensions (such as Intel VT-x or AMD-V). In our case, we are going to virtualize Rocky Linux, which will later in this series be used to run k3s.</span><br />
+<br />
+<h2 style='display: inline' id='check-for-popcnt-cpu-support'>Check for <span class='inlinecode'>POPCNT</span> CPU support</h2><br />
+<br />
+<span>POPCNT is a CPU instruction that counts the number of set bits (ones) in a binary number. CPU virtualization and Bhyve support for the POPCNT instruction are important because guest operating systems utilize this instruction to perform various tasks more efficiently. If the host CPU supports POPCNT, Bhyve can pass this capability to virtual machines for better performance. Without POPCNT support, some applications might not run or perform sub-optimally in virtualized environments.</span><br />
+<br />
+<span>To check for <span class='inlinecode'>POPCNT</span> support, run:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % dmesg | grep <font color="#808080">'Features2=.*POPCNT'</font>
+  Features2=<font color="#000000">0x7ffafbbf</font>&lt;SSE3,PCLMULQDQ,DTES64,MON,DS_CPL,VMX,EST,TM2,SSSE3,SDBG,
+	FMA,CX16,xTPR,PDCM,PCID,SSE4.<font color="#000000">1</font>,SSE4.<font color="#000000">2</font>,x2APIC,MOVBE,POPCNT,TSCDLT,AESNI,XSAVE,
+	OSXSAVE,AVX,F16C,RDRAND&gt;
+</pre>
+<br />
+<span>So it&#39;s there! All good.</span><br />
+<br />
+<h2 style='display: inline' id='basic-bhyve-setup'>Basic Bhyve setup</h2><br />
+<br />
+<span>For managing the Bhyve VMs, we are using <span class='inlinecode'>vm-bhyve</span>, a tool not part of the FreeBSD operating system but available as a ready-to-use package. It eases VM management and reduces a lot of overhead. We also install the required package to make Bhyve work with the UEFI firmware.</span><br />
+<br />
+<a class='textlink' href='https://github.com/churchers/vm-bhyve'>https://github.com/churchers/vm-bhyve</a><br />
+<br />
+<span>The following commands are executed on all three hosts <span class='inlinecode'>f0</span>, <span class='inlinecode'>f1</span>, and <span class='inlinecode'>f2</span>, where <span class='inlinecode'>re0</span> is the name of the Ethernet interface (which may need to be adjusted if your hardware is different):</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % doas pkg install vm-bhyve bhyve-firmware
+paul@f0:~ % doas sysrc vm_enable=YES
+vm_enable:  -&gt; YES
+paul@f0:~ % doas sysrc vm_dir=zfs:zroot/bhyve
+vm_dir:  -&gt; zfs:zroot/bhyve
+paul@f0:~ % doas zfs create zroot/bhyve
+paul@f0:~ % doas vm init
+paul@f0:~ % doas vm switch create public
+paul@f0:~ % doas vm switch add public re0
+</pre>
+<br />
+<span>Bhyve stores all it&#39;s data in the <span class='inlinecode'>/bhyve</span> of the <span class='inlinecode'>zroot</span> ZFS pool:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % zfs list | grep bhyve
+zroot/bhyve                                   <font color="#000000">1</font>.74M   453G  <font color="#000000">1</font>.74M  /zroot/bhyve
+</pre>
+<br />
+<span>For convenience, we also create this symlink:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % doas ln -s /zroot/bhyve/ /bhyve
+
+</pre>
+<br />
+<span>Now, Bhyve is ready to rumble, but no VMs are there yet:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % doas vm list
+NAME  DATASTORE  LOADER  CPU  MEMORY  VNC  AUTO  STATE
+</pre>
+<br />
+<h2 style='display: inline' id='rocky-linux-vms'>Rocky Linux VMs</h2><br />
+<br />
+<span>As guest VMs I decided to use Rocky Linux.</span><br />
+<br />
+<span>Using Rocky Linux 9 as a VM-based OS is beneficial primarily because of its long-term support and stable release cycle. This ensures a reliable environment that receives security updates and bug fixes for an extended period, reducing the need for frequent upgrades. Rocky Linux is community-driven and aims to be fully compatible with enterprise Linux, making it a solid choice for consistency and performance in various deployment scenarios.</span><br />
+<br />
+<a class='textlink' href='https://rockylinux.org/'>https://rockylinux.org/</a><br />
+<br />
+<h3 style='display: inline' id='iso-download'>ISO download</h3><br />
+<br />
+<span>We&#39;re going to install the Rocky Linux from the latest minimal iso:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % doas vm iso \
+ https://download.rockylinux.org/pub/rocky/<font color="#000000">9</font>/isos/x86_64/Rocky-<font color="#000000">9.5</font>-x86_64-minimal.iso
+/zroot/bhyve/.iso/Rocky-<font color="#000000">9.5</font>-x86_64-minimal.iso        <font color="#000000">1808</font> MB <font color="#000000">4780</font> kBps 06m28s
+paul@f0:/bhyve % doas vm create rocky
+</pre>
+<h3 style='display: inline' id='vm-configuration'>VM configuration</h3><br />
+<br />
+<span>The default Bhyve VM configuration looks like this now:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:/bhyve/rocky % cat rocky.conf
+loader=<font color="#808080">"bhyveload"</font>
+cpu=<font color="#000000">1</font>
+memory=256M
+network0_type=<font color="#808080">"virtio-net"</font>
+network0_switch=<font color="#808080">"public"</font>
+disk0_type=<font color="#808080">"virtio-blk"</font>
+disk0_name=<font color="#808080">"disk0.img"</font>
+uuid=<font color="#808080">"1c4655ac-c828-11ef-a920-e8ff1ed71ca0"</font>
+network0_mac=<font color="#808080">"58:9c:fc:0d:13:3f"</font>
+</pre>
+<br />
+<span>The <span class='inlinecode'>uuid</span> and the <span class='inlinecode'>network0_mac</span> differ for each of the three VMs.</span><br />
+<br />
+<span>But to make Rocky Linux boot it (plus some other adjustments, e.g. as we intend to run the majority of the workload in the k3s cluster running on those Linux VMs, we give them beefy specs like 4 CPU cores and 14GB RAM). So we run <span class='inlinecode'>doas vm configure rocky</span> and modified it to:</span><br />
+<br />
+<pre>
+guest="linux"
+loader="uefi"
+uefi_vars="yes"
+cpu=4
+memory=14G
+network0_type="virtio-net"
+network0_switch="public"
+disk0_type="virtio-blk"
+disk0_name="disk0.img"
+graphics="yes"
+graphics_vga=io
+uuid="1c45400b-c828-11ef-8871-e8ff1ed71cac"
+network0_mac="58:9c:fc:0d:13:3f"
+</pre>
+<br />
+<h3 style='display: inline' id='vm-installation'>VM installation</h3><br />
+<br />
+<span>To start the installer from the downloaded ISO, we run:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % doas vm install rocky Rocky-<font color="#000000">9.5</font>-x86_64-minimal.iso
+Starting rocky
+  * found guest <b><u><font color="#000000">in</font></u></b> /zroot/bhyve/rocky
+  * booting...
+
+paul@f0:/bhyve/rocky % doas vm list
+NAME   DATASTORE  LOADER  CPU  MEMORY  VNC           AUTO  STATE
+rocky  default    uefi    <font color="#000000">4</font>    14G     <font color="#000000">0.0</font>.<font color="#000000">0.0</font>:<font color="#000000">5900</font>  No    Locked (f0.lan.buetow.org)
+
+paul@f0:/bhyve/rocky % doas sockstat -<font color="#000000">4</font> | grep <font color="#000000">5900</font>
+root     bhyve       <font color="#000000">6079</font> <font color="#000000">8</font>   tcp4   *:<font color="#000000">5900</font>                *:*
+</pre>
+<br />
+<span>Port 5900 now also opens for VNC connections, so I connected it with a VNC client and ran through the installation dialogues. This could be done unattended or more automated, but there are only three VMs to install, and the automation doesn&#39;t seem worth it as we do it only once a year or less often.</span><br />
+<br />
+<h3 style='display: inline' id='increase-of-the-disk-image'>Increase of the disk image</h3><br />
+<br />
+<span>By default, the VM disk image is only 20G, which is a bit small for our purposes, so I stopped the VMs again, ran <span class='inlinecode'>truncate</span> on the image file to enlarge them to 100G, and re-started the installation:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:/bhyve/rocky % doas vm stop rocky
+paul@f0:/bhyve/rocky % doas truncate -s 100G disk0.img
+paul@f0:/bhyve/rocky % doas vm install rocky Rocky-<font color="#000000">9.5</font>-x86_64-minimal.iso
+</pre>
+<br />
+<h3 style='display: inline' id='connect-to-vnc'>Connect to VNC</h3><br />
+<br />
+<span>For the installation, I opened the VNC client on my Fedora laptop (GNOME comes with a simple VNC client) and manually ran through the base installation for each of the VMs. Again, I am sure this could have been automated a bit more, but there were just three VMs, and it wasn&#39;t worth the effort. The three VNC addresses of the VMs were <span class='inlinecode'>vnc://f0:5900</span>, <span class='inlinecode'>vnc://f1:5900</span>, and <span class='inlinecode'>vnc://f0:5900</span>.</span><br />
+<br />
+<a href='./f3s-kubernetes-with-freebsd-part-4/1.png'><img src='./f3s-kubernetes-with-freebsd-part-4/1.png' /></a><br />
+<br />
+<a href='./f3s-kubernetes-with-freebsd-part-4/2.png'><img src='./f3s-kubernetes-with-freebsd-part-4/2.png' /></a><br />
+<br />
+<span>I primarily selected the default settings (auto partitioning on the 100GB drive and a root user password). After the installation, the VMs were rebooted.</span><br />
+<br />
+<a href='./f3s-kubernetes-with-freebsd-part-4/3.png'><img src='./f3s-kubernetes-with-freebsd-part-4/3.png' /></a><br />
+<br />
+<a href='./f3s-kubernetes-with-freebsd-part-4/4.png'><img src='./f3s-kubernetes-with-freebsd-part-4/4.png' /></a><br />
+<br />
+<h2 style='display: inline' id='after-install'>After install</h2><br />
+<br />
+<span>We perform the following steps for all 3 VMs. In the following, the examples are all executed on <span class='inlinecode'>f0</span> (the VM <span class='inlinecode'>r0</span> running on <span class='inlinecode'>f0</span>):</span><br />
+<br />
+<h3 style='display: inline' id='vm-auto-start-after-host-reboot'>VM auto-start after host reboot</h3><br />
+<br />
+<span>To automatically start the VM on the servers, we add the following to the <span class='inlinecode'>rc.conf</span> on the FreeBSD hosts:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:/bhyve/rocky % cat &lt;&lt;END | doas tee -a /etc/rc.conf
+vm_list=<font color="#808080">"rocky"</font>
+vm_delay=<font color="#808080">"5"</font>
+</pre>
+<br />
+<span>The <span class='inlinecode'>vm_delay</span> isn&#39;t really required. It is used to wait 5 seconds before starting each VM, but there is currently only one VM per host. Maybe later, when there are more, this will be useful. After adding, there&#39;s now a <span class='inlinecode'>Yes</span> indicator in the <span class='inlinecode'>AUTO</span> column.</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:~ % doas vm list
+NAME   DATASTORE  LOADER  CPU  MEMORY  VNC           AUTO     STATE
+rocky  default    uefi    <font color="#000000">4</font>    14G     <font color="#000000">0.0</font>.<font color="#000000">0.0</font>:<font color="#000000">5900</font>  Yes [<font color="#000000">1</font>]  Running (<font color="#000000">2063</font>)
+</pre>
+<br />
+<h3 style='display: inline' id='static-ip-configuration'>Static IP configuration</h3><br />
+<br />
+<span>After that, we change the network configuration of the VMs to be static (from DHCP) here. As per the previous post of this series, the 3 FreeBSD hosts were already in my <span class='inlinecode'>/etc/hosts</span> file:</span><br />
+<br />
+<pre>
+192.168.1.130 f0 f0.lan f0.lan.buetow.org
+192.168.1.131 f1 f1.lan f1.lan.buetow.org
+192.168.1.132 f2 f2.lan f2.lan.buetow.org
+</pre>
+<br />
+<span>For the Rocky VMs, we add those to the FreeBSD host systems as well:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>paul@f0:/bhyve/rocky % cat &lt;&lt;END | doas tee -a /etc/hosts
+<font color="#000000">192.168</font>.<font color="#000000">1.120</font> r0 r0.lan r0.lan.buetow.org
+<font color="#000000">192.168</font>.<font color="#000000">1.121</font> r1 r1.lan r1.lan.buetow.org
+<font color="#000000">192.168</font>.<font color="#000000">1.122</font> r2 r2.lan r2.lan.buetow.org
+END
+</pre>
+<br />
+<span>And we configure the IPs accordingly on the VMs themselves by opening a root shell via RDP to the VMs and entering the following commands on each of the VMs:</span><br />
+<br />
+<!-- Generator: GNU source-highlight 3.1.9
+by Lorenzo Bettini
+http://www.lorenzobettini.it
+http://www.gnu.org/software/src-highlite -->
+<pre>[root@r0 ~] % dnmcli connection modify enp0s5 ipv4.address <font color="#000000">192.168</font>.<font color="#000000">1.120</font>/<font color="#000000">24</font>
+[root@r0 ~] % dnmcli connection modify enp0s5 ipv4.gateway <font color="#000000">192.168</font>.<font color="#000000">1.1</font>
+[root@r0 ~] % dnmcli connection modify enp0s5 ipv4.DNS <font color="#000000">192.168</font>.<font color="#000000">1.1</font>
+[root@r0 ~] % dnmcli connection modify enp0s5 ipv4.method manual
+[root@r0 ~] % dnmcli connection down enp0s5
+[root@r0 ~] % dnmcli connection up enp0s5
+[root@r0 ~] % hostnamectl set-hostname r0.lan.buetow.org
+[root@r0 ~] % cat &lt;&lt;END &gt;&gt;/etc/hosts
+<font color="#000000">192.168</font>.<font color="#000000">1.120</font> r0 r0.lan r0.lan.buetow.org
+<font color="#000000">192.168</font>.<font color="#000000">1.121</font> r1 r1.lan r1.lan.buetow.org
+<font color="#000000">192.168</font>.<font color="#000000">1.122</font> r2 r2.lan r2.lan.buetow.org
+END