arch, vmm: Add CPUID check to the 'Config' step of live migration

We now send not only the 'VmConfig' at the 'Command::Config' step of
live migration, but also send the 'common CPUID'. In this way, we can
check the compatibility of CPUID features between the source and
destination VMs, and abort live migration early if needed.

Signed-off-by: Bo Chen <chen.bo@intel.com>

arch/src/lib.rs

@@ -85,7 +85,7 @@ pub mod x86_64;
 pub use x86_64::{
     arch_memory_regions, configure_system, configure_vcpu, generate_common_cpuid,
     get_host_cpu_phys_bits, initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE,
-    layout::CMDLINE_START, regs, EntryPoint,
+    layout::CMDLINE_START, regs, CpuidFeatureEntry, EntryPoint,
 };
 
 /// Safe wrapper for `sysconf(_SC_PAGESIZE)`.

arch/src/x86_64/mod.rs

@@ -185,6 +185,9 @@ pub enum Error {
 
     /// Error populating CPUID with CPU identification
     CpuidIdentification(vmm_sys_util::fam::Error),
+
+    /// Error checking CPUID compatibility
+    CpuidCheckCompatibility,
 }
 
 impl From<Error> for super::Error {
@@ -194,7 +197,7 @@ impl From<Error> for super::Error {
 }
 
 #[allow(dead_code, clippy::upper_case_acronyms)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub enum CpuidReg {
     EAX,
     EBX,
@@ -338,6 +341,165 @@ impl CpuidPatch {
     }
 }
 
+pub struct CpuidFeatureEntry {
+    pub function: u32,
+    pub index: u32,
+    pub feature_reg: CpuidReg,
+}
+
+impl CpuidFeatureEntry {
+    fn checked_feature_entry_list() -> Vec<CpuidFeatureEntry> {
+        vec![
+            // The following list includes all hardware features bits from
+            // the CPUID Wiki Page: https://en.wikipedia.org/wiki/CPUID
+            // Leaf 0x1, ECX/EDX, feature bits
+            CpuidFeatureEntry {
+                function: 1,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+            },
+            CpuidFeatureEntry {
+                function: 1,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+            },
+            // Leaf 0x7, EAX/EBX/ECX/EDX, extended features
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::EAX,
+            },
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::EBX,
+            },
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+            },
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+            },
+            // Leaf 0x7 subleaf 0x1, EAX, extended features
+            CpuidFeatureEntry {
+                function: 7,
+                index: 1,
+                feature_reg: CpuidReg::EAX,
+            },
+            // Leaf 0x8000_0001, ECX/EDX, CPUID features bits
+            CpuidFeatureEntry {
+                function: 0x8000_0001,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+            },
+            CpuidFeatureEntry {
+                function: 0x8000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+            },
+            // KVM CPUID bits: https://www.kernel.org/doc/html/latest/virt/kvm/cpuid.html
+            // Leaf 0x4000_0001, EAX/EBX/ECX/EDX, KVM CPUID features
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EAX,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EBX,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+            },
+        ]
+    }
+
+    fn get_features_from_cpuid(
+        cpuid: &CpuId,
+        feature_entry_list: &[CpuidFeatureEntry],
+    ) -> Vec<u32> {
+        let mut features = vec![0; feature_entry_list.len()];
+        for (i, feature_entry) in feature_entry_list.iter().enumerate() {
+            for cpuid_entry in cpuid.as_slice().iter() {
+                if cpuid_entry.function == feature_entry.function
+                    && cpuid_entry.index == feature_entry.index
+                {
+                    match feature_entry.feature_reg {
+                        CpuidReg::EAX => {
+                            features[i] = cpuid_entry.eax;
+                        }
+                        CpuidReg::EBX => {
+                            features[i] = cpuid_entry.ebx;
+                        }
+                        CpuidReg::ECX => {
+                            features[i] = cpuid_entry.ecx;
+                        }
+                        CpuidReg::EDX => {
+                            features[i] = cpuid_entry.edx;
+                        }
+                    }
+
+                    break;
+                }
+            }
+        }
+
+        features
+    }
+
+    // The function returns `Error` (a.k.a. "incompatible"), when the CPUID features from `src_vm_cpuid`
+    // is not a subset of those of the `dest_vm_cpuid`.
+    pub fn check_cpuid_compatibility(
+        src_vm_cpuid: &CpuId,
+        dest_vm_cpuid: &CpuId,
+    ) -> Result<(), Error> {
+        let feature_entry_list = &Self::checked_feature_entry_list();
+        let src_vm_features = Self::get_features_from_cpuid(src_vm_cpuid, feature_entry_list);
+        let dest_vm_features = Self::get_features_from_cpuid(dest_vm_cpuid, feature_entry_list);
+
+        // Loop on feature bit and check if the 'source vm' feature is a subset
+        // of those of the 'destination vm' feature
+        let mut compatible = true;
+        for (i, (src_vm_feature, dest_vm_feature)) in src_vm_features
+            .iter()
+            .zip(dest_vm_features.iter())
+            .enumerate()
+        {
+            let different_feature_bits = src_vm_feature ^ dest_vm_feature;
+            let src_vm_feature_bits_only = different_feature_bits & src_vm_feature;
+            if src_vm_feature_bits_only != 0 {
+                error!(
+                    "Detected incompatible CPUID entry: leaf={:#02x} (subleaf={:#02x}), register='{:?}', \
+                    source VM feature='{:#04x}', destination VM feature'{:#04x}'.",
+                    feature_entry_list[i].function, feature_entry_list[i].index, feature_entry_list[i].feature_reg,
+                    src_vm_feature, dest_vm_feature
+                    );
+
+                compatible = false;
+            }
+        }
+
+        if compatible {
+            info!("No CPU incompatibility detected.");
+            Ok(())
+        } else {
+            Err(Error::CpuidCheckCompatibility)
+        }
+    }
+}
+
 pub fn generate_common_cpuid(
     hypervisor: Arc<dyn hypervisor::Hypervisor>,
     topology: Option<(u8, u8, u8)>,

vmm/src/lib.rs

@@ -284,6 +284,13 @@ pub fn start_vmm_thread(
     Ok(thread)
 }
 
+#[derive(Clone, Deserialize, Serialize)]
+struct VmMigrationConfig {
+    vm_config: Arc<Mutex<VmConfig>>,
+    #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+    common_cpuid: hypervisor::CpuId,
+}
+
 pub struct Vmm {
     epoll: EpollContext,
     exit_evt: EventFd,
@@ -743,8 +750,47 @@ impl Vmm {
         socket
             .read_exact(&mut data)
             .map_err(MigratableError::MigrateSocket)?;
-        let config: VmConfig = serde_json::from_slice(&data).map_err(|e| {
-            MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e))
+
+        let vm_migration_config: VmMigrationConfig =
+            serde_json::from_slice(&data).map_err(|e| {
+                MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e))
+            })?;
+
+        #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+        // We check the `CPUID` compatibility of between the source vm and destination, which is
+        // mostly about feature compatibility and "topology/sgx" leaves are not relevant.
+        let dest_cpuid = {
+            let vm_config = &vm_migration_config.vm_config.lock().unwrap();
+            #[cfg(feature = "tdx")]
+            let tdx_enabled = vm_config.tdx.is_some();
+            let phys_bits = vm::physical_bits(
+                vm_config.cpus.max_phys_bits,
+                #[cfg(feature = "tdx")]
+                tdx_enabled,
+            );
+            arch::generate_common_cpuid(
+                self.hypervisor.clone(),
+                None,
+                None,
+                phys_bits,
+                vm_config.cpus.kvm_hyperv,
+                #[cfg(feature = "tdx")]
+                tdx_enabled,
+            )
+            .map_err(|e| {
+                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid': {:?}", e))
+            })?
+        };
+        #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+        arch::CpuidFeatureEntry::check_cpuid_compatibility(
+            &vm_migration_config.common_cpuid,
+            &dest_cpuid,
+        )
+        .map_err(|e| {
+            MigratableError::MigrateReceive(anyhow!(
+                "Error checking cpu feature compatibility': {:?}",
+                e
+            ))
         })?;
 
         let exit_evt = self.exit_evt.try_clone().map_err(|e| {
@@ -757,7 +803,7 @@ impl Vmm {
             MigratableError::MigrateReceive(anyhow!("Error cloning activate EventFd: {}", e))
         })?;
 
-        self.vm_config = Some(Arc::new(Mutex::new(config)));
+        self.vm_config = Some(vm_migration_config.vm_config);
         let vm = Vm::new_from_migration(
             self.vm_config.clone().unwrap(),
             exit_evt,
@@ -998,7 +1044,40 @@ impl Vmm {
             }
 
             // Send config
-            let config_data = serde_json::to_vec(&vm.get_config()).unwrap();
+            let vm_config = vm.get_config();
+            #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+            let common_cpuid = {
+                #[cfg(feature = "tdx")]
+                let tdx_enabled = vm_config.lock().unwrap().tdx.is_some();
+                let phys_bits = vm::physical_bits(
+                    vm_config.lock().unwrap().cpus.max_phys_bits,
+                    #[cfg(feature = "tdx")]
+                    tdx_enabled,
+                );
+                arch::generate_common_cpuid(
+                    self.hypervisor.clone(),
+                    None,
+                    None,
+                    phys_bits,
+                    vm_config.lock().unwrap().cpus.kvm_hyperv,
+                    #[cfg(feature = "tdx")]
+                    tdx_enabled,
+                )
+                .map_err(|e| {
+                    MigratableError::MigrateReceive(anyhow!(
+                        "Error generating common cpuid': {:?}",
+                        e
+                    ))
+                })?
+            };
+
+            let vm_migration_config = VmMigrationConfig {
+                vm_config,
+                #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+                common_cpuid,
+            };
+
+            let config_data = serde_json::to_vec(&vm_migration_config).unwrap();
             Request::config(config_data.len() as u64).write_to(&mut socket)?;
             socket
                 .write_all(&config_data)