drvbhyve: Use boot-order for grub-bhyve boot device

Rather than just picking the first CD (or failing that, HDD) we come
across, if the user has picked a boot device ordering with <boot
order=''>, respect that (and just try to boot the lowest-index device).

Adds two sets of tests to bhyve2xmlargv; 'grub-bootorder' shows that we
pick a user-specified device over the first device in the domain;
'grub-bootorder2' shows that we pick the first (lowest index) device.

docs/drvbhyve.html.in

@@ -234,10 +234,11 @@ management.</p>
 <p>It's possible to boot non-FreeBSD guests by specifying an explicit
 bootloader, e.g. <code>grub-bhyve(1)</code>. Arguments to the bootloader may be
 specified as well. If the bootloader is <code>grub-bhyve</code> and arguments
-are omitted, libvirt will try and boot the first disk in the domain (either
+are omitted, libvirt will try and infer boot ordering from user-supplied
-<code>cdrom</code>- or <code>disk</code>-type devices). If the disk type is
+&lt;boot order='N'&gt; configuration in the domain. Failing that, it will boot
-<code>disk</code>, it will attempt to boot from the first partition in the disk
+the first disk in the domain (either <code>cdrom</code>- or
-image.</p>
+<code>disk</code>-type devices). If the disk type is <code>disk</code>, it will
+attempt to boot from the first partition in the disk image.</p>
 
 <pre>
   ...

src/bhyve/bhyve_command.c

@@ -381,38 +381,62 @@ virBhyveUsableDisk(virConnectPtr conn, virDomainDiskDefPtr disk)
     return true;
 }
 
+static void
+virBhyveFormatGrubDevice(virBufferPtr devicemap, virDomainDiskDefPtr def)
+{
+
+    if (def->device == VIR_DOMAIN_DISK_DEVICE_CDROM)
+        virBufferAsprintf(devicemap, "(cd) %s\n",
+                          virDomainDiskGetSource(def));
+    else
+        virBufferAsprintf(devicemap, "(hd0) %s\n",
+                          virDomainDiskGetSource(def));
+}
+
 static virCommandPtr
 virBhyveProcessBuildGrubbhyveCmd(virDomainDefPtr def,
                                  virConnectPtr conn,
                                  const char *devmap_file,
                                  char **devicesmap_out)
 {
-    virDomainDiskDefPtr disk, cd;
+    virDomainDiskDefPtr hdd, cd, userdef, diskdef;
     virBuffer devicemap;
     virCommandPtr cmd;
+    int best_idx;
     size_t i;
 
     if (def->os.bootloaderArgs != NULL)
         return virBhyveProcessBuildCustomLoaderCmd(def);
 
+    best_idx = INT_MAX;
     devicemap = (virBuffer)VIR_BUFFER_INITIALIZER;
 
-    /* Search disk list for CD or HDD device. */
+    /* Search disk list for CD or HDD device. We'll respect <boot order=''> if
-    cd = disk = NULL;
+     * present and otherwise pick the first CD or failing that HDD we come
+     * across. */
+    cd = hdd = userdef = NULL;
     for (i = 0; i < def->ndisks; i++) {
         if (!virBhyveUsableDisk(conn, def->disks[i]))
             continue;
 
-        if (cd == NULL &&
+        diskdef = def->disks[i];
-            def->disks[i]->device == VIR_DOMAIN_DISK_DEVICE_CDROM) {
+
-            cd = def->disks[i];
+        if (diskdef->info.bootIndex && diskdef->info.bootIndex < best_idx) {
-            VIR_INFO("Picking %s as boot CD", virDomainDiskGetSource(cd));
+            userdef = diskdef;
+            best_idx = userdef->info.bootIndex;
+            continue;
         }
 
-        if (disk == NULL &&
+        if (cd == NULL &&
+            def->disks[i]->device == VIR_DOMAIN_DISK_DEVICE_CDROM) {
+            cd = diskdef;
+            VIR_INFO("Picking %s as CD", virDomainDiskGetSource(cd));
+        }
+
+        if (hdd == NULL &&
             def->disks[i]->device == VIR_DOMAIN_DISK_DEVICE_DISK) {
-            disk = def->disks[i];
+            hdd = diskdef;
-            VIR_INFO("Picking %s as HDD", virDomainDiskGetSource(disk));
+            VIR_INFO("Picking %s as HDD", virDomainDiskGetSource(hdd));
         }
     }
 
@@ -422,22 +446,28 @@ virBhyveProcessBuildGrubbhyveCmd(virDomainDefPtr def,
 
     if (devicesmap_out != NULL) {
         /* Grub device.map (just for boot) */
-        if (disk != NULL)
+        if (userdef != NULL) {
-            virBufferAsprintf(&devicemap, "(hd0) %s\n",
+            virBhyveFormatGrubDevice(&devicemap, userdef);
-                              virDomainDiskGetSource(disk));
+        } else {
+            if (hdd != NULL)
+                virBhyveFormatGrubDevice(&devicemap, hdd);
 
             if (cd != NULL)
-            virBufferAsprintf(&devicemap, "(cd) %s\n",
+                virBhyveFormatGrubDevice(&devicemap, cd);
-                              virDomainDiskGetSource(cd));
+        }
 
         *devicesmap_out = virBufferContentAndReset(&devicemap);
     }
 
-    if (cd != NULL) {
     virCommandAddArg(cmd, "--root");
+    if (userdef != NULL) {
+        if (userdef->device == VIR_DOMAIN_DISK_DEVICE_CDROM)
+            virCommandAddArg(cmd, "cd");
+        else
+            virCommandAddArg(cmd, "hd0,msdos1");
+    } else if (cd != NULL) {
         virCommandAddArg(cmd, "cd");
     } else {
-        virCommandAddArg(cmd, "--root");
         virCommandAddArg(cmd, "hd0,msdos1");
     }
 

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder.args

@@ -0,0 +1,6 @@
+/usr/sbin/bhyve -c 1 -m 214 -H -P -s 0:0,hostbridge \
+-s 3:0,virtio-net,faketapdev,mac=52:54:00:00:00:00 \
+-s 2:0,ahci-hd,/tmp/freebsd1.img \
+-s 2:0,ahci-hd,/tmp/freebsd2.img \
+-s 2:0,ahci-hd,/tmp/freebsd3.img \
+bhyve

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder.devmap

@@ -0,0 +1 @@
+(hd0) /tmp/freebsd3.img

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder.ldargs

@@ -0,0 +1,2 @@
+/usr/local/sbin/grub-bhyve --root hd0,msdos1 --device-map '<device.map>' \
+--memory 214 bhyve

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder.xml

@@ -0,0 +1,36 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <bootloader>/usr/local/sbin/grub-bhyve</bootloader>
+  <os>
+    <type>hvm</type>
+  </os>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd1.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </disk>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd2.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </disk>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd3.img'/>
+      <target dev='hda' bus='sata'/>
+      <boot order='1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </disk>
+    <interface type='bridge'>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder2.args

@@ -0,0 +1,6 @@
+/usr/sbin/bhyve -c 1 -m 214 -H -P -s 0:0,hostbridge \
+-s 3:0,virtio-net,faketapdev,mac=52:54:00:00:00:00 \
+-s 2:0,ahci-hd,/tmp/freebsd1.img \
+-s 2:0,ahci-hd,/tmp/freebsd2.img \
+-s 2:0,ahci-hd,/tmp/freebsd3.img \
+bhyve

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder2.devmap

@@ -0,0 +1 @@
+(hd0) /tmp/freebsd3.img

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder2.ldargs

@@ -0,0 +1,2 @@
+/usr/local/sbin/grub-bhyve --root hd0,msdos1 --device-map '<device.map>' \
+--memory 214 bhyve

tests/bhyvexml2argvdata/bhyvexml2argv-grub-bootorder2.xml

@@ -0,0 +1,38 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <bootloader>/usr/local/sbin/grub-bhyve</bootloader>
+  <os>
+    <type>hvm</type>
+  </os>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd1.img'/>
+      <target dev='hda' bus='sata'/>
+      <boot order='111'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </disk>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd2.img'/>
+      <target dev='hda' bus='sata'/>
+      <boot order='22'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </disk>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd3.img'/>
+      <target dev='hda' bus='sata'/>
+      <boot order='3'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </disk>
+    <interface type='bridge'>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

tests/bhyvexml2argvtest.c

@@ -163,6 +163,8 @@ mymain(void)
     DO_TEST("serial");
     DO_TEST("console");
     DO_TEST("grub-defaults");
+    DO_TEST("grub-bootorder");
+    DO_TEST("grub-bootorder2");
     DO_TEST("bhyveload-explicitargs");
     DO_TEST("custom-loader");
     DO_TEST("disk-cdrom-grub");