AArch64: Preparation for vCPU save/restore

This commit ports code from firecracker and refactors the existing
AArch64 code as the preparation for implementing save/restore
AArch64 vCPU, including:

1. Modification of `arm64_core_reg` macro to retrive the index of
arm64 core register and implemention of a helper to determine if
a register is a system register.

2. Move some macros and helpers in `arch` crate to the `hypervisor`
crate.

3. Added related unit tests for above functions and macros.

Signed-off-by: Henry Wang <Henry.Wang@arm.com>

arch/src/aarch64/mod.rs

@@ -42,6 +42,9 @@ pub enum Error {
     /// Error configuring the general purpose registers
     REGSConfiguration(regs::Error),
 
+    /// Error configuring the MPIDR register
+    VcpuRegMPIDR(hypervisor::HypervisorCpuError),
+
     /// Error fetching prefered target
     VcpuArmPreferredTarget(hypervisor::HypervisorVmError),
 
@@ -94,7 +97,7 @@ pub fn configure_vcpu(
         .map_err(Error::REGSConfiguration)?;
     }
 
-    let mpidr = regs::read_mpidr(fd).map_err(Error::REGSConfiguration)?;
+    let mpidr = fd.read_mpidr().map_err(Error::VcpuRegMPIDR)?;
     Ok(mpidr)
 }
 

arch/src/aarch64/regs.rs

@@ -7,12 +7,9 @@
 
 use super::get_fdt_addr;
 use hypervisor::kvm::kvm_bindings::{
-    user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM64_SYSREG_CRM_MASK,
+    kvm_regs, user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM_CORE, KVM_REG_SIZE_U64,
-    KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT,
-    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK,
-    KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT,
-    KVM_REG_ARM_CORE, KVM_REG_SIZE_U64,
 };
+use hypervisor::{arm64_core_reg_id, offset__of};
 use std::sync::Arc;
 use std::{mem, result};
 use vm_memory::GuestMemoryMmap;
@@ -38,80 +35,6 @@ const PSR_D_BIT: u64 = 0x0000_0200;
 // Taken from arch/arm64/kvm/inject_fault.c.
 const PSTATE_FAULT_BITS_64: u64 = PSR_MODE_EL1h | PSR_A_BIT | PSR_F_BIT | PSR_I_BIT | PSR_D_BIT;
 
-// Following are macros that help with getting the ID of a aarch64 core register.
-// The core register are represented by the user_pt_regs structure. Look for it in
-// arch/arm64/include/uapi/asm/ptrace.h.
-
-// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
-// an instance of that structure.
-// It uses a null pointer to retrieve the offset to the field.
-// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`.
-// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following:
-// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize }
-// The dereference expression produces an lvalue, but that lvalue is not actually read from,
-// we're just doing pointer math on it, so in theory, it should safe.
-macro_rules! offset__of {
-    ($str:ty, $field:ident) => {
-        unsafe { &(*std::ptr::null::<user_pt_regs>()).$field as *const _ as usize }
-    };
-}
-
-macro_rules! arm64_core_reg {
-    ($reg: tt) => {
-        // As per `kvm_arm_copy_reg_indices`, the id of a core register can be obtained like this:
-        // `const u64 core_reg = KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | i`, where i is obtained with:
-        // `for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {`
-        // We are using here `user_pt_regs` since this structure contains the core register and it is at
-        // the start of `kvm_regs`.
-        // struct kvm_regs {
-        //	struct user_pt_regs regs;	/* sp = sp_el0 */
-        //
-        //	__u64	sp_el1;
-        //	__u64	elr_el1;
-        //
-        //	__u64	spsr[KVM_NR_SPSR];
-        //
-        //	struct user_fpsimd_state fp_regs;
-        //};
-        // struct user_pt_regs {
-        //	__u64		regs[31];
-        //	__u64		sp;
-        //	__u64		pc;
-        //	__u64		pstate;
-        //};
-        // In our implementation we need: pc, pstate and user_pt_regs->regs[0].
-        KVM_REG_ARM64 as u64
-            | KVM_REG_SIZE_U64 as u64
-            | u64::from(KVM_REG_ARM_CORE)
-            | ((offset__of!(user_pt_regs, $reg) / mem::size_of::<u32>()) as u64)
-    };
-}
-
-// This macro computes the ID of a specific ARM64 system register similar to how
-// the kernel C macro does.
-// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203
-macro_rules! arm64_sys_reg {
-    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
-        const $name: u64 = KVM_REG_ARM64 as u64
-            | KVM_REG_SIZE_U64 as u64
-            | KVM_REG_ARM64_SYSREG as u64
-            | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
-            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
-            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
-                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
-            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
-                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
-            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
-    };
-}
-
-// Constant imported from the Linux kernel:
-// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135
-arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
-
 /// Configure core registers for a given CPU.
 ///
 /// # Arguments
@@ -126,31 +49,33 @@ pub fn setup_regs(
     boot_ip: u64,
     mem: &GuestMemoryMmap,
 ) -> Result<()> {
+    let kreg_off = offset__of!(kvm_regs, regs);
+
     // Get the register index of the PSTATE (Processor State) register.
-    vcpu.set_one_reg(arm64_core_reg!(pstate), PSTATE_FAULT_BITS_64)
+    let pstate = offset__of!(user_pt_regs, pstate) + kreg_off;
-        .map_err(Error::SetCoreRegister)?;
+    vcpu.set_one_reg(
+        arm64_core_reg_id!(KVM_REG_SIZE_U64, pstate),
+        PSTATE_FAULT_BITS_64,
+    )
+    .map_err(Error::SetCoreRegister)?;
 
     // Other vCPUs are powered off initially awaiting PSCI wakeup.
     if cpu_id == 0 {
         // Setting the PC (Processor Counter) to the current program address (kernel address).
-        vcpu.set_one_reg(arm64_core_reg!(pc), boot_ip)
+        let pc = offset__of!(user_pt_regs, pc) + kreg_off;
+        vcpu.set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, pc), boot_ip as u64)
             .map_err(Error::SetCoreRegister)?;
 
         // Last mandatory thing to set -> the address pointing to the FDT (also called DTB).
         // "The device tree blob (dtb) must be placed on an 8-byte boundary and must
         // not exceed 2 megabytes in size." -> https://www.kernel.org/doc/Documentation/arm64/booting.txt.
         // We are choosing to place it the end of DRAM. See `get_fdt_addr`.
-        vcpu.set_one_reg(arm64_core_reg!(regs), get_fdt_addr(mem) as u64)
+        let regs0 = offset__of!(user_pt_regs, regs) + kreg_off;
-            .map_err(Error::SetCoreRegister)?;
+        vcpu.set_one_reg(
+            arm64_core_reg_id!(KVM_REG_SIZE_U64, regs0),
+            get_fdt_addr(mem) as u64,
+        )
+        .map_err(Error::SetCoreRegister)?;
     }
     Ok(())
 }
-
-/// Read the MPIDR - Multiprocessor Affinity Register.
-///
-/// # Arguments
-///
-/// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
-pub fn read_mpidr(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<u64> {
-    vcpu.get_one_reg(MPIDR_EL1).map_err(Error::GetSysRegister)
-}

hypervisor/src/cpu.rs

@@ -153,6 +153,11 @@ pub enum HypervisorCpuError {
     ///
     #[error("Failed to enable HyperV SynIC")]
     EnableHyperVSynIC(#[source] anyhow::Error),
+    ///
+    /// Getting AArch64 system register error
+    ///
+    #[error("Failed to get system register: {0}")]
+    GetSysRegister(#[source] anyhow::Error),
 }
 
 #[derive(Debug)]
@@ -306,6 +311,11 @@ pub trait Vcpu: Send + Sync {
     #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
     fn get_one_reg(&self, reg_id: u64) -> Result<u64>;
     ///
+    /// Read the MPIDR - Multiprocessor Affinity Register.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn read_mpidr(&self) -> Result<u64>;
+    ///
     /// Retrieve the vCPU state.
     /// This function is necessary to snapshot the VM
     ///
@@ -315,7 +325,6 @@ pub trait Vcpu: Send + Sync {
     /// This function is required when restoring the VM
     ///
     fn set_state(&self, state: &CpuState) -> Result<()>;
-
     ///
     /// Triggers the running of the current virtual CPU returning an exit reason.
     ///

hypervisor/src/kvm/aarch64/mod.rs

@@ -13,9 +13,114 @@
 ///
 use crate::kvm::{KvmError, KvmResult};
 pub use kvm_bindings::kvm_vcpu_init as VcpuInit;
+use kvm_bindings::{
+    KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM64_SYSREG_CRM_MASK,
+    KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT,
+    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK,
+    KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT,
+    KVM_REG_ARM_COPROC_MASK, KVM_REG_ARM_CORE, KVM_REG_SIZE_MASK, KVM_REG_SIZE_U32,
+    KVM_REG_SIZE_U64,
+};
 use serde_derive::{Deserialize, Serialize};
 pub use {kvm_ioctls::Cap, kvm_ioctls::Kvm};
 
+// Following are macros that help with getting the ID of a aarch64 core register.
+// The core register are represented by the user_pt_regs structure. Look for it in
+// arch/arm64/include/uapi/asm/ptrace.h.
+
+// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
+// an instance of that structure.
+// It uses a null pointer to retrieve the offset to the field.
+// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`.
+// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following:
+// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize }
+// The dereference expression produces an lvalue, but that lvalue is not actually read from,
+// we're just doing pointer math on it, so in theory, it should safe.
+#[macro_export]
+macro_rules! offset__of {
+    ($str:ty, $field:ident) => {
+        unsafe { &(*std::ptr::null::<$str>()).$field as *const _ as usize }
+    };
+}
+
+// Get the ID of a core register
+#[macro_export]
+macro_rules! arm64_core_reg_id {
+    ($size: tt, $offset: tt) => {
+        // The core registers of an arm64 machine are represented
+        // in kernel by the `kvm_regs` structure. This structure is a
+        // mix of 32, 64 and 128 bit fields:
+        // struct kvm_regs {
+        //     struct user_pt_regs      regs;
+        //
+        //     __u64                    sp_el1;
+        //     __u64                    elr_el1;
+        //
+        //     __u64                    spsr[KVM_NR_SPSR];
+        //
+        //     struct user_fpsimd_state fp_regs;
+        // };
+        // struct user_pt_regs {
+        //     __u64 regs[31];
+        //     __u64 sp;
+        //     __u64 pc;
+        //     __u64 pstate;
+        // };
+        // The id of a core register can be obtained like this:
+        // offset = id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE). Thus,
+        // id = KVM_REG_ARM64 | KVM_REG_SIZE_U64/KVM_REG_SIZE_U32/KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | offset
+        KVM_REG_ARM64 as u64
+            | u64::from(KVM_REG_ARM_CORE)
+            | $size
+            | (($offset / mem::size_of::<u32>()) as u64)
+    };
+}
+
+// This macro computes the ID of a specific ARM64 system register similar to how
+// the kernel C macro does.
+// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203
+#[macro_export]
+macro_rules! arm64_sys_reg {
+    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
+        pub const $name: u64 = KVM_REG_ARM64 as u64
+            | KVM_REG_SIZE_U64 as u64
+            | KVM_REG_ARM64_SYSREG as u64
+            | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
+            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
+            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
+            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
+            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
+    };
+}
+
+// Constant imported from the Linux kernel:
+// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135
+arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
+
+/// Specifies whether a particular register is a system register or not.
+/// The kernel splits the registers on aarch64 in core registers and system registers.
+/// So, below we get the system registers by checking that they are not core registers.
+///
+/// # Arguments
+///
+/// * `regid` - The index of the register we are checking.
+pub fn is_system_register(regid: u64) -> bool {
+    if (regid & KVM_REG_ARM_COPROC_MASK as u64) == KVM_REG_ARM_CORE as u64 {
+        return false;
+    }
+
+    let size = regid & KVM_REG_SIZE_MASK;
+    if size != KVM_REG_SIZE_U32 && size != KVM_REG_SIZE_U64 {
+        panic!("Unexpected register size for system register {}", size);
+    }
+    true
+}
+
 pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
     if !kvm.check_extension(Cap::SignalMsi) {
         return Err(KvmError::CapabilityMissing(Cap::SignalMsi));

hypervisor/src/kvm/mod.rs

@@ -9,7 +9,9 @@
 //
 
 #[cfg(target_arch = "aarch64")]
-pub use crate::aarch64::{check_required_kvm_extensions, VcpuInit, VcpuKvmState as CpuState};
+pub use crate::aarch64::{
+    check_required_kvm_extensions, VcpuInit, VcpuKvmState as CpuState, MPIDR_EL1,
+};
 use crate::cpu;
 use crate::device;
 use crate::hypervisor;
@@ -763,6 +765,15 @@ impl cpu::Vcpu for KvmVcpu {
             .get_one_reg(reg_id)
             .map_err(|e| cpu::HypervisorCpuError::GetOneReg(e.into()))
     }
+    ///
+    /// Read the MPIDR - Multiprocessor Affinity Register.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn read_mpidr(&self) -> cpu::Result<u64> {
+        self.fd
+            .get_one_reg(MPIDR_EL1)
+            .map_err(|e| cpu::HypervisorCpuError::GetSysRegister(e.into()))
+    }
     #[cfg(target_arch = "x86_64")]
     ///
     /// Get the current CPU state

vmm/src/cpu.rs

@@ -1595,7 +1595,13 @@ mod tests {
 mod tests {
     use arch::aarch64::layout;
     use arch::aarch64::regs::*;
-    use hypervisor::kvm::kvm_bindings;
+    use hypervisor::kvm::aarch64::is_system_register;
+    use hypervisor::kvm::kvm_bindings::{
+        kvm_vcpu_init, user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM_CORE,
+        KVM_REG_SIZE_U64,
+    };
+    use hypervisor::{arm64_core_reg_id, offset__of};
+    use std::mem;
     use vm_memory::{GuestAddress, GuestMemoryMmap};
 
     #[test]
@@ -1610,7 +1616,11 @@ mod tests {
         ));
         let mem = GuestMemoryMmap::from_ranges(&regions).expect("Cannot initialize memory");
 
-        let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
+        let res = setup_regs(&vcpu, 0, 0x0, &mem);
+        // Must fail when vcpu is not initialized yet.
+        assert!(res.is_err());
+
+        let mut kvi: kvm_vcpu_init = kvm_vcpu_init::default();
         vm.get_preferred_target(&mut kvi).unwrap();
         vcpu.vcpu_init(&kvi).unwrap();
 
@@ -1622,7 +1632,7 @@ mod tests {
         let hv = hypervisor::new().unwrap();
         let vm = hv.create_vm().unwrap();
         let vcpu = vm.create_vcpu(0).unwrap();
-        let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
+        let mut kvi: kvm_vcpu_init = kvm_vcpu_init::default();
         vm.get_preferred_target(&mut kvi).unwrap();
 
         // Must fail when vcpu is not initialized yet.
@@ -1631,4 +1641,13 @@ mod tests {
         vcpu.vcpu_init(&kvi).unwrap();
         assert_eq!(read_mpidr(&vcpu).unwrap(), 0x80000000);
     }
+
+    #[test]
+    fn test_is_system_register() {
+        let offset = offset__of!(user_pt_regs, pc);
+        let regid = arm64_core_reg_id!(KVM_REG_SIZE_U64, offset);
+        assert!(!is_system_register(regid));
+        let regid = KVM_REG_ARM64 as u64 | KVM_REG_SIZE_U64 as u64 | KVM_REG_ARM64_SYSREG as u64;
+        assert!(is_system_register(regid));
+    }
 }