From e3d45be6f7d46b672f781050d50466822fa41a6b Mon Sep 17 00:00:00 2001 From: Henry Wang Date: Fri, 28 Aug 2020 17:06:54 +0800 Subject: [PATCH] AArch64: Preparation for vCPU save/restore This commit ports code from firecracker and refactors the existing AArch64 code as the preparation for implementing save/restore AArch64 vCPU, including: 1. Modification of `arm64_core_reg` macro to retrive the index of arm64 core register and implemention of a helper to determine if a register is a system register. 2. Move some macros and helpers in `arch` crate to the `hypervisor` crate. 3. Added related unit tests for above functions and macros. Signed-off-by: Henry Wang --- arch/src/aarch64/mod.rs | 5 +- arch/src/aarch64/regs.rs | 111 +++++------------------------- hypervisor/src/cpu.rs | 11 ++- hypervisor/src/kvm/aarch64/mod.rs | 105 ++++++++++++++++++++++++++++ hypervisor/src/kvm/mod.rs | 13 +++- vmm/src/cpu.rs | 25 ++++++- 6 files changed, 171 insertions(+), 99 deletions(-) diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs index f6b4fa237..07e209f57 100644 --- a/arch/src/aarch64/mod.rs +++ b/arch/src/aarch64/mod.rs @@ -42,6 +42,9 @@ pub enum Error { /// Error configuring the general purpose registers REGSConfiguration(regs::Error), + /// Error configuring the MPIDR register + VcpuRegMPIDR(hypervisor::HypervisorCpuError), + /// Error fetching prefered target VcpuArmPreferredTarget(hypervisor::HypervisorVmError), @@ -94,7 +97,7 @@ pub fn configure_vcpu( .map_err(Error::REGSConfiguration)?; } - let mpidr = regs::read_mpidr(fd).map_err(Error::REGSConfiguration)?; + let mpidr = fd.read_mpidr().map_err(Error::VcpuRegMPIDR)?; Ok(mpidr) } diff --git a/arch/src/aarch64/regs.rs b/arch/src/aarch64/regs.rs index 4847aa1cb..4ba48ef07 100644 --- a/arch/src/aarch64/regs.rs +++ b/arch/src/aarch64/regs.rs @@ -7,12 +7,9 @@ use super::get_fdt_addr; use hypervisor::kvm::kvm_bindings::{ - user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM64_SYSREG_CRM_MASK, - KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT, - KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK, - KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT, - KVM_REG_ARM_CORE, KVM_REG_SIZE_U64, + kvm_regs, user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM_CORE, KVM_REG_SIZE_U64, }; +use hypervisor::{arm64_core_reg_id, offset__of}; use std::sync::Arc; use std::{mem, result}; use vm_memory::GuestMemoryMmap; @@ -38,80 +35,6 @@ const PSR_D_BIT: u64 = 0x0000_0200; // Taken from arch/arm64/kvm/inject_fault.c. const PSTATE_FAULT_BITS_64: u64 = PSR_MODE_EL1h | PSR_A_BIT | PSR_F_BIT | PSR_I_BIT | PSR_D_BIT; -// Following are macros that help with getting the ID of a aarch64 core register. -// The core register are represented by the user_pt_regs structure. Look for it in -// arch/arm64/include/uapi/asm/ptrace.h. - -// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having -// an instance of that structure. -// It uses a null pointer to retrieve the offset to the field. -// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`. -// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following: -// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize } -// The dereference expression produces an lvalue, but that lvalue is not actually read from, -// we're just doing pointer math on it, so in theory, it should safe. -macro_rules! offset__of { - ($str:ty, $field:ident) => { - unsafe { &(*std::ptr::null::()).$field as *const _ as usize } - }; -} - -macro_rules! arm64_core_reg { - ($reg: tt) => { - // As per `kvm_arm_copy_reg_indices`, the id of a core register can be obtained like this: - // `const u64 core_reg = KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | i`, where i is obtained with: - // `for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {` - // We are using here `user_pt_regs` since this structure contains the core register and it is at - // the start of `kvm_regs`. - // struct kvm_regs { - // struct user_pt_regs regs; /* sp = sp_el0 */ - // - // __u64 sp_el1; - // __u64 elr_el1; - // - // __u64 spsr[KVM_NR_SPSR]; - // - // struct user_fpsimd_state fp_regs; - //}; - // struct user_pt_regs { - // __u64 regs[31]; - // __u64 sp; - // __u64 pc; - // __u64 pstate; - //}; - // In our implementation we need: pc, pstate and user_pt_regs->regs[0]. - KVM_REG_ARM64 as u64 - | KVM_REG_SIZE_U64 as u64 - | u64::from(KVM_REG_ARM_CORE) - | ((offset__of!(user_pt_regs, $reg) / mem::size_of::()) as u64) - }; -} - -// This macro computes the ID of a specific ARM64 system register similar to how -// the kernel C macro does. -// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203 -macro_rules! arm64_sys_reg { - ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => { - const $name: u64 = KVM_REG_ARM64 as u64 - | KVM_REG_SIZE_U64 as u64 - | KVM_REG_ARM64_SYSREG as u64 - | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) - & KVM_REG_ARM64_SYSREG_OP0_MASK as u64) - | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT) - & KVM_REG_ARM64_SYSREG_OP1_MASK as u64) - | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT) - & KVM_REG_ARM64_SYSREG_CRN_MASK as u64) - | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT) - & KVM_REG_ARM64_SYSREG_CRM_MASK as u64) - | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT) - & KVM_REG_ARM64_SYSREG_OP2_MASK as u64); - }; -} - -// Constant imported from the Linux kernel: -// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135 -arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5); - /// Configure core registers for a given CPU. /// /// # Arguments @@ -126,31 +49,33 @@ pub fn setup_regs( boot_ip: u64, mem: &GuestMemoryMmap, ) -> Result<()> { + let kreg_off = offset__of!(kvm_regs, regs); + // Get the register index of the PSTATE (Processor State) register. - vcpu.set_one_reg(arm64_core_reg!(pstate), PSTATE_FAULT_BITS_64) - .map_err(Error::SetCoreRegister)?; + let pstate = offset__of!(user_pt_regs, pstate) + kreg_off; + vcpu.set_one_reg( + arm64_core_reg_id!(KVM_REG_SIZE_U64, pstate), + PSTATE_FAULT_BITS_64, + ) + .map_err(Error::SetCoreRegister)?; // Other vCPUs are powered off initially awaiting PSCI wakeup. if cpu_id == 0 { // Setting the PC (Processor Counter) to the current program address (kernel address). - vcpu.set_one_reg(arm64_core_reg!(pc), boot_ip) + let pc = offset__of!(user_pt_regs, pc) + kreg_off; + vcpu.set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, pc), boot_ip as u64) .map_err(Error::SetCoreRegister)?; // Last mandatory thing to set -> the address pointing to the FDT (also called DTB). // "The device tree blob (dtb) must be placed on an 8-byte boundary and must // not exceed 2 megabytes in size." -> https://www.kernel.org/doc/Documentation/arm64/booting.txt. // We are choosing to place it the end of DRAM. See `get_fdt_addr`. - vcpu.set_one_reg(arm64_core_reg!(regs), get_fdt_addr(mem) as u64) - .map_err(Error::SetCoreRegister)?; + let regs0 = offset__of!(user_pt_regs, regs) + kreg_off; + vcpu.set_one_reg( + arm64_core_reg_id!(KVM_REG_SIZE_U64, regs0), + get_fdt_addr(mem) as u64, + ) + .map_err(Error::SetCoreRegister)?; } Ok(()) } - -/// Read the MPIDR - Multiprocessor Affinity Register. -/// -/// # Arguments -/// -/// * `vcpu` - Structure for the VCPU that holds the VCPU's fd. -pub fn read_mpidr(vcpu: &Arc) -> Result { - vcpu.get_one_reg(MPIDR_EL1).map_err(Error::GetSysRegister) -} diff --git a/hypervisor/src/cpu.rs b/hypervisor/src/cpu.rs index f378d30c3..d03f5c54d 100644 --- a/hypervisor/src/cpu.rs +++ b/hypervisor/src/cpu.rs @@ -153,6 +153,11 @@ pub enum HypervisorCpuError { /// #[error("Failed to enable HyperV SynIC")] EnableHyperVSynIC(#[source] anyhow::Error), + /// + /// Getting AArch64 system register error + /// + #[error("Failed to get system register: {0}")] + GetSysRegister(#[source] anyhow::Error), } #[derive(Debug)] @@ -306,6 +311,11 @@ pub trait Vcpu: Send + Sync { #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] fn get_one_reg(&self, reg_id: u64) -> Result; /// + /// Read the MPIDR - Multiprocessor Affinity Register. + /// + #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] + fn read_mpidr(&self) -> Result; + /// /// Retrieve the vCPU state. /// This function is necessary to snapshot the VM /// @@ -315,7 +325,6 @@ pub trait Vcpu: Send + Sync { /// This function is required when restoring the VM /// fn set_state(&self, state: &CpuState) -> Result<()>; - /// /// Triggers the running of the current virtual CPU returning an exit reason. /// diff --git a/hypervisor/src/kvm/aarch64/mod.rs b/hypervisor/src/kvm/aarch64/mod.rs index 05022e217..00934539f 100644 --- a/hypervisor/src/kvm/aarch64/mod.rs +++ b/hypervisor/src/kvm/aarch64/mod.rs @@ -13,9 +13,114 @@ /// use crate::kvm::{KvmError, KvmResult}; pub use kvm_bindings::kvm_vcpu_init as VcpuInit; +use kvm_bindings::{ + KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM64_SYSREG_CRM_MASK, + KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT, + KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK, + KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT, + KVM_REG_ARM_COPROC_MASK, KVM_REG_ARM_CORE, KVM_REG_SIZE_MASK, KVM_REG_SIZE_U32, + KVM_REG_SIZE_U64, +}; use serde_derive::{Deserialize, Serialize}; pub use {kvm_ioctls::Cap, kvm_ioctls::Kvm}; +// Following are macros that help with getting the ID of a aarch64 core register. +// The core register are represented by the user_pt_regs structure. Look for it in +// arch/arm64/include/uapi/asm/ptrace.h. + +// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having +// an instance of that structure. +// It uses a null pointer to retrieve the offset to the field. +// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`. +// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following: +// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize } +// The dereference expression produces an lvalue, but that lvalue is not actually read from, +// we're just doing pointer math on it, so in theory, it should safe. +#[macro_export] +macro_rules! offset__of { + ($str:ty, $field:ident) => { + unsafe { &(*std::ptr::null::<$str>()).$field as *const _ as usize } + }; +} + +// Get the ID of a core register +#[macro_export] +macro_rules! arm64_core_reg_id { + ($size: tt, $offset: tt) => { + // The core registers of an arm64 machine are represented + // in kernel by the `kvm_regs` structure. This structure is a + // mix of 32, 64 and 128 bit fields: + // struct kvm_regs { + // struct user_pt_regs regs; + // + // __u64 sp_el1; + // __u64 elr_el1; + // + // __u64 spsr[KVM_NR_SPSR]; + // + // struct user_fpsimd_state fp_regs; + // }; + // struct user_pt_regs { + // __u64 regs[31]; + // __u64 sp; + // __u64 pc; + // __u64 pstate; + // }; + // The id of a core register can be obtained like this: + // offset = id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE). Thus, + // id = KVM_REG_ARM64 | KVM_REG_SIZE_U64/KVM_REG_SIZE_U32/KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | offset + KVM_REG_ARM64 as u64 + | u64::from(KVM_REG_ARM_CORE) + | $size + | (($offset / mem::size_of::()) as u64) + }; +} + +// This macro computes the ID of a specific ARM64 system register similar to how +// the kernel C macro does. +// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203 +#[macro_export] +macro_rules! arm64_sys_reg { + ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => { + pub const $name: u64 = KVM_REG_ARM64 as u64 + | KVM_REG_SIZE_U64 as u64 + | KVM_REG_ARM64_SYSREG as u64 + | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) + & KVM_REG_ARM64_SYSREG_OP0_MASK as u64) + | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT) + & KVM_REG_ARM64_SYSREG_OP1_MASK as u64) + | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT) + & KVM_REG_ARM64_SYSREG_CRN_MASK as u64) + | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT) + & KVM_REG_ARM64_SYSREG_CRM_MASK as u64) + | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT) + & KVM_REG_ARM64_SYSREG_OP2_MASK as u64); + }; +} + +// Constant imported from the Linux kernel: +// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135 +arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5); + +/// Specifies whether a particular register is a system register or not. +/// The kernel splits the registers on aarch64 in core registers and system registers. +/// So, below we get the system registers by checking that they are not core registers. +/// +/// # Arguments +/// +/// * `regid` - The index of the register we are checking. +pub fn is_system_register(regid: u64) -> bool { + if (regid & KVM_REG_ARM_COPROC_MASK as u64) == KVM_REG_ARM_CORE as u64 { + return false; + } + + let size = regid & KVM_REG_SIZE_MASK; + if size != KVM_REG_SIZE_U32 && size != KVM_REG_SIZE_U64 { + panic!("Unexpected register size for system register {}", size); + } + true +} + pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> { if !kvm.check_extension(Cap::SignalMsi) { return Err(KvmError::CapabilityMissing(Cap::SignalMsi)); diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index 7acb1c6a5..1b1e29ca1 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -9,7 +9,9 @@ // #[cfg(target_arch = "aarch64")] -pub use crate::aarch64::{check_required_kvm_extensions, VcpuInit, VcpuKvmState as CpuState}; +pub use crate::aarch64::{ + check_required_kvm_extensions, VcpuInit, VcpuKvmState as CpuState, MPIDR_EL1, +}; use crate::cpu; use crate::device; use crate::hypervisor; @@ -763,6 +765,15 @@ impl cpu::Vcpu for KvmVcpu { .get_one_reg(reg_id) .map_err(|e| cpu::HypervisorCpuError::GetOneReg(e.into())) } + /// + /// Read the MPIDR - Multiprocessor Affinity Register. + /// + #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] + fn read_mpidr(&self) -> cpu::Result { + self.fd + .get_one_reg(MPIDR_EL1) + .map_err(|e| cpu::HypervisorCpuError::GetSysRegister(e.into())) + } #[cfg(target_arch = "x86_64")] /// /// Get the current CPU state diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index ff70094c5..9212ed67e 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -1595,7 +1595,13 @@ mod tests { mod tests { use arch::aarch64::layout; use arch::aarch64::regs::*; - use hypervisor::kvm::kvm_bindings; + use hypervisor::kvm::aarch64::is_system_register; + use hypervisor::kvm::kvm_bindings::{ + kvm_vcpu_init, user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM_CORE, + KVM_REG_SIZE_U64, + }; + use hypervisor::{arm64_core_reg_id, offset__of}; + use std::mem; use vm_memory::{GuestAddress, GuestMemoryMmap}; #[test] @@ -1610,7 +1616,11 @@ mod tests { )); let mem = GuestMemoryMmap::from_ranges(®ions).expect("Cannot initialize memory"); - let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default(); + let res = setup_regs(&vcpu, 0, 0x0, &mem); + // Must fail when vcpu is not initialized yet. + assert!(res.is_err()); + + let mut kvi: kvm_vcpu_init = kvm_vcpu_init::default(); vm.get_preferred_target(&mut kvi).unwrap(); vcpu.vcpu_init(&kvi).unwrap(); @@ -1622,7 +1632,7 @@ mod tests { let hv = hypervisor::new().unwrap(); let vm = hv.create_vm().unwrap(); let vcpu = vm.create_vcpu(0).unwrap(); - let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default(); + let mut kvi: kvm_vcpu_init = kvm_vcpu_init::default(); vm.get_preferred_target(&mut kvi).unwrap(); // Must fail when vcpu is not initialized yet. @@ -1631,4 +1641,13 @@ mod tests { vcpu.vcpu_init(&kvi).unwrap(); assert_eq!(read_mpidr(&vcpu).unwrap(), 0x80000000); } + + #[test] + fn test_is_system_register() { + let offset = offset__of!(user_pt_regs, pc); + let regid = arm64_core_reg_id!(KVM_REG_SIZE_U64, offset); + assert!(!is_system_register(regid)); + let regid = KVM_REG_ARM64 as u64 | KVM_REG_SIZE_U64 as u64 | KVM_REG_ARM64_SYSREG as u64; + assert!(is_system_register(regid)); + } }