diff --git a/hypervisor/src/arch/aarch64/gic.rs b/hypervisor/src/arch/aarch64/gic.rs index 889dc8d16..2ea8550e1 100644 --- a/hypervisor/src/arch/aarch64/gic.rs +++ b/hypervisor/src/arch/aarch64/gic.rs @@ -1,6 +1,6 @@ // Copyright 2022 Arm Limited (or its affiliates). All rights reserved. -use crate::{CpuState, Device, HypervisorDeviceError, HypervisorVmError}; +use crate::{CpuState, Device, GicState, HypervisorDeviceError, HypervisorVmError}; use std::any::Any; use std::result; use std::sync::Arc; @@ -18,13 +18,7 @@ pub enum Error { pub type Result = result::Result; /// Hypervisor agnostic interface for a virtualized GIC -pub trait Vgic: Send { - /// Returns the hypervisor agnostic Device of the GIC device - fn device(&self) -> &Arc; - - /// Returns the hypervisor agnostic Device of the ITS device - fn its_device(&self) -> Option<&Arc>; - +pub trait Vgic: Send + Sync { /// Returns the fdt compatibility property of the device fn fdt_compatibility(&self) -> &str; @@ -54,11 +48,9 @@ pub trait Vgic: Send { /// Downcast the trait object to its concrete type. fn as_any_concrete_mut(&mut self) -> &mut dyn Any; - /* - /// Save the state of GICv3ITS. - fn state(&self, gicr_typers: &[u64]) -> Result; + /// Save the state of GICv3ITS. + fn state(&self, gicr_typers: &[u64]) -> Result; - /// Restore the state of GICv3ITS. - fn set_state(&mut self, gicr_typers: &[u64], state: &GicState) -> Result<()>; - */ + /// Restore the state of GICv3ITS. + fn set_state(&mut self, gicr_typers: &[u64], state: &GicState) -> Result<()>; } diff --git a/hypervisor/src/kvm/aarch64/gic/dist_regs.rs b/hypervisor/src/kvm/aarch64/gic/dist_regs.rs new file mode 100644 index 000000000..02f3134da --- /dev/null +++ b/hypervisor/src/kvm/aarch64/gic/dist_regs.rs @@ -0,0 +1,182 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::arch::aarch64::gic::{Error, Result}; +use crate::kvm::kvm_bindings::{ + kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, +}; +use crate::Device; +use std::sync::Arc; + +/* + Distributor registers as detailed at page 456 from + https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_architecture_specification.pdf. + Address offsets are relative to the Distributor base address defined +by the system memory map. Unless otherwise stated in the register description, +all GIC registers are 32-bits wide. + */ +const GICD_CTLR: u32 = 0x0; +const GICD_STATUSR: u32 = 0x0010; +const GICD_IGROUPR: u32 = 0x0080; +const GICD_ISENABLER: u32 = 0x0100; +const GICD_ICENABLER: u32 = 0x0180; +const GICD_ISPENDR: u32 = 0x0200; +const GICD_ICPENDR: u32 = 0x0280; +const GICD_ISACTIVER: u32 = 0x0300; +const GICD_ICACTIVER: u32 = 0x0380; +const GICD_IPRIORITYR: u32 = 0x0400; +const GICD_ICFGR: u32 = 0x0C00; +const GICD_IROUTER: u32 = 0x6000; + +/// This is how we represent the registers of the vgic's distributor. +/// Some of the distributor register )(i.e GICD_STATUSR) are simple +/// registers (i.e they are associated to a 32 bit value). +/// However, there are other registers that have variable lengths since +/// they dedicate some of the 32 bits to some specific interrupt. So, their length +/// depends on the number of interrupts (i.e the ones that are represented as GICD_REG) +/// in the documentation mentioned above. +struct DistReg { + /// Offset from distributor address. + base: u32, + /// Bits per interrupt. + /// Relevant for registers that DO share IRQs. + bpi: u8, + /// Length of the register. + /// Relevant for registers that DO NOT share IRQs. + length: u16, +} + +// All or at least the registers we are interested in are 32 bit, so +// we use a constant for size(u32). +const REG_SIZE: u8 = 4; + +// Creates a vgic distributor register. +macro_rules! VGIC_DIST_REG { + ($base:expr, $bpi:expr, $length:expr) => { + DistReg { + base: $base, + bpi: $bpi, + length: $length, + } + }; +} + +// List with relevant distributor registers that we will be restoring. +// Order is taken from qemu. +static VGIC_DIST_REGS: &[DistReg] = &[ + VGIC_DIST_REG!(GICD_STATUSR, 0, 4), + VGIC_DIST_REG!(GICD_ICENABLER, 1, 0), + VGIC_DIST_REG!(GICD_ISENABLER, 1, 0), + VGIC_DIST_REG!(GICD_IGROUPR, 1, 0), + VGIC_DIST_REG!(GICD_IROUTER, 64, 0), + VGIC_DIST_REG!(GICD_ICFGR, 2, 0), + VGIC_DIST_REG!(GICD_ICPENDR, 1, 0), + VGIC_DIST_REG!(GICD_ISPENDR, 1, 0), + VGIC_DIST_REG!(GICD_ICACTIVER, 1, 0), + VGIC_DIST_REG!(GICD_ISACTIVER, 1, 0), + VGIC_DIST_REG!(GICD_IPRIORITYR, 8, 0), +]; + +fn dist_attr_access(gic: &Arc, offset: u32, val: &u32, set: bool) -> Result<()> { + let mut gic_dist_attr = kvm_device_attr { + group: KVM_DEV_ARM_VGIC_GRP_DIST_REGS, + attr: offset as u64, + addr: val as *const u32 as u64, + flags: 0, + }; + if set { + gic.set_device_attr(&gic_dist_attr) + .map_err(Error::SetDeviceAttribute)?; + } else { + gic.get_device_attr(&mut gic_dist_attr) + .map_err(Error::GetDeviceAttribute)?; + } + Ok(()) +} + +/// Get the distributor control register. +pub fn read_ctlr(gic: &Arc) -> Result { + let val: u32 = 0; + dist_attr_access(gic, GICD_CTLR, &val, false)?; + Ok(val) +} + +/// Set the distributor control register. +pub fn write_ctlr(gic: &Arc, val: u32) -> Result<()> { + dist_attr_access(gic, GICD_CTLR, &val, true) +} + +fn get_interrupts_num(gic: &Arc) -> Result { + let num_irq = 0; + + let mut nr_irqs_attr = kvm_device_attr { + group: KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + attr: 0, + addr: &num_irq as *const u32 as u64, + flags: 0, + }; + gic.get_device_attr(&mut nr_irqs_attr) + .map_err(Error::GetDeviceAttribute)?; + Ok(num_irq) +} + +fn compute_reg_len(gic: &Arc, reg: &DistReg, base: u32) -> Result { + // FIXME: + // Redefine some GIC constants to avoid the dependency on `layout` crate. + // This is temporary solution, will be fixed in future refactoring. + const LAYOUT_IRQ_BASE: u32 = 32; + + let mut end = base; + let num_irq = get_interrupts_num(gic)?; + if reg.length > 0 { + // This is the single type register (i.e one that is not DIST_X) and for which + // the bpi is 0. + // Look in the kernel for REGISTER_DESC_WITH_LENGTH. + end = base + reg.length as u32; + } + if reg.bpi > 0 { + // This is the type of register that takes into account the number of interrupts + // that the model has. It is also the type of register where + // a register relates to multiple interrupts. + end = base + (reg.bpi as u32 * (num_irq - LAYOUT_IRQ_BASE) / 8); + if reg.bpi as u32 * (num_irq - LAYOUT_IRQ_BASE) % 8 > 0 { + end += REG_SIZE as u32; + } + } + Ok(end) +} + +/// Set distributor registers of the GIC. +pub fn set_dist_regs(gic: &Arc, state: &[u32]) -> Result<()> { + let mut idx = 0; + + for dreg in VGIC_DIST_REGS { + let mut base = dreg.base + REG_SIZE as u32 * dreg.bpi as u32; + let end = compute_reg_len(gic, dreg, base)?; + + while base < end { + let val = state[idx]; + dist_attr_access(gic, base, &val, true)?; + idx += 1; + base += REG_SIZE as u32; + } + } + Ok(()) +} +/// Get distributor registers of the GIC. +pub fn get_dist_regs(gic: &Arc) -> Result> { + let mut state = Vec::new(); + + for dreg in VGIC_DIST_REGS { + let mut base = dreg.base + REG_SIZE as u32 * dreg.bpi as u32; + let end = compute_reg_len(gic, dreg, base)?; + + while base < end { + let val: u32 = 0; + dist_attr_access(gic, base, &val, false)?; + state.push(val); + base += REG_SIZE as u32; + } + } + Ok(state) +} diff --git a/hypervisor/src/kvm/aarch64/gic/icc_regs.rs b/hypervisor/src/kvm/aarch64/gic/icc_regs.rs new file mode 100644 index 000000000..b043e1a29 --- /dev/null +++ b/hypervisor/src/kvm/aarch64/gic/icc_regs.rs @@ -0,0 +1,193 @@ +// Copyright 2022 Arm Limited (or its affiliates). All rights reserved. +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::arch::aarch64::gic::{Error, Result}; +use crate::kvm::kvm_bindings::{ + kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, KVM_REG_ARM64_SYSREG_CRM_MASK, + KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT, + KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK, + KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT, +}; +use crate::Device; +use std::sync::Arc; + +const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32; +const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64; + +const ICC_CTLR_EL1_PRIBITS_SHIFT: u32 = 8; +const ICC_CTLR_EL1_PRIBITS_MASK: u32 = 7 << ICC_CTLR_EL1_PRIBITS_SHIFT; + +macro_rules! arm64_vgic_sys_reg { + ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: expr) => { + const $name: u64 = ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) + & KVM_REG_ARM64_SYSREG_OP0_MASK as u64) + | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT) + & KVM_REG_ARM64_SYSREG_OP1_MASK as u64) + | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT) + & KVM_REG_ARM64_SYSREG_CRN_MASK as u64) + | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT) + & KVM_REG_ARM64_SYSREG_CRM_MASK as u64) + | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT) + & KVM_REG_ARM64_SYSREG_OP2_MASK as u64); + }; +} + +macro_rules! SYS_ICC_AP0Rn_EL1 { + ($name: tt, $n: tt) => { + arm64_vgic_sys_reg!($name, 3, 0, 12, 8, (4 | $n)); + }; +} + +macro_rules! SYS_ICC_AP1Rn_EL1 { + ($name: tt, $n: tt) => { + arm64_vgic_sys_reg!($name, 3, 0, 12, 9, $n); + }; +} + +arm64_vgic_sys_reg!(SYS_ICC_SRE_EL1, 3, 0, 12, 12, 5); +arm64_vgic_sys_reg!(SYS_ICC_CTLR_EL1, 3, 0, 12, 12, 4); +arm64_vgic_sys_reg!(SYS_ICC_IGRPEN0_EL1, 3, 0, 12, 12, 6); +arm64_vgic_sys_reg!(SYS_ICC_IGRPEN1_EL1, 3, 0, 12, 12, 7); +arm64_vgic_sys_reg!(SYS_ICC_PMR_EL1, 3, 0, 4, 6, 0); +arm64_vgic_sys_reg!(SYS_ICC_BPR0_EL1, 3, 0, 12, 8, 3); +arm64_vgic_sys_reg!(SYS_ICC_BPR1_EL1, 3, 0, 12, 12, 3); +SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R0_EL1, 0); +SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R1_EL1, 1); +SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R2_EL1, 2); +SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R3_EL1, 3); +SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R0_EL1, 0); +SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R1_EL1, 1); +SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R2_EL1, 2); +SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R3_EL1, 3); + +static VGIC_ICC_REGS: &[u64] = &[ + SYS_ICC_SRE_EL1, + SYS_ICC_CTLR_EL1, + SYS_ICC_IGRPEN0_EL1, + SYS_ICC_IGRPEN1_EL1, + SYS_ICC_PMR_EL1, + SYS_ICC_BPR0_EL1, + SYS_ICC_BPR1_EL1, + SYS_ICC_AP0R0_EL1, + SYS_ICC_AP0R1_EL1, + SYS_ICC_AP0R2_EL1, + SYS_ICC_AP0R3_EL1, + SYS_ICC_AP1R0_EL1, + SYS_ICC_AP1R1_EL1, + SYS_ICC_AP1R2_EL1, + SYS_ICC_AP1R3_EL1, +]; + +fn icc_attr_access( + gic: &Arc, + offset: u64, + typer: u64, + val: &u32, + set: bool, +) -> Result<()> { + let mut gic_icc_attr = kvm_device_attr { + group: KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr: ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | offset), // this needs the mpidr + addr: val as *const u32 as u64, + flags: 0, + }; + if set { + gic.set_device_attr(&gic_icc_attr) + .map_err(Error::SetDeviceAttribute)?; + } else { + gic.get_device_attr(&mut gic_icc_attr) + .map_err(Error::GetDeviceAttribute)?; + } + Ok(()) +} + +/// Get ICC registers. +pub fn get_icc_regs(gic: &Arc, gicr_typer: &[u64]) -> Result> { + let mut state: Vec = Vec::new(); + // We need this for the ICC_APR_EL1 registers. + let mut num_priority_bits = 0; + + for ix in gicr_typer { + let i = *ix; + for icc_offset in VGIC_ICC_REGS { + let val = 0; + if *icc_offset == SYS_ICC_CTLR_EL1 { + // calculate priority bits by reading the ctrl_el1 register. + icc_attr_access(gic, *icc_offset, i, &val, false)?; + // The priority bits are found in the ICC_CTLR_EL1 register (bits from 10:8). + // See page 194 from https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_ + // architecture_specification.pdf. + // Citation: + // "Priority bits. Read-only and writes are ignored. The number of priority bits + // implemented, minus one." + num_priority_bits = + ((val & ICC_CTLR_EL1_PRIBITS_MASK) >> ICC_CTLR_EL1_PRIBITS_SHIFT) + 1; + state.push(val); + } + // As per ARMv8 documentation: https://static.docs.arm.com/ihi0069/c/IHI0069C_ + // gic_architecture_specification.pdf + // page 178, + // ICC_AP0R1_EL1 is only implemented in implementations that support 6 or more bits of + // priority. + // ICC_AP0R2_EL1 and ICC_AP0R3_EL1 are only implemented in implementations that support + // 7 bits of priority. + else if *icc_offset == SYS_ICC_AP0R1_EL1 || *icc_offset == SYS_ICC_AP1R1_EL1 { + if num_priority_bits >= 6 { + icc_attr_access(gic, *icc_offset, i, &val, false)?; + state.push(val); + } + } else if *icc_offset == SYS_ICC_AP0R2_EL1 + || *icc_offset == SYS_ICC_AP0R3_EL1 + || *icc_offset == SYS_ICC_AP1R2_EL1 + || *icc_offset == SYS_ICC_AP1R3_EL1 + { + if num_priority_bits == 7 { + icc_attr_access(gic, *icc_offset, i, &val, false)?; + state.push(val); + } + } else { + icc_attr_access(gic, *icc_offset, i, &val, false)?; + state.push(val); + } + } + } + Ok(state) +} + +/// Set ICC registers. +pub fn set_icc_regs(gic: &Arc, gicr_typer: &[u64], state: &[u32]) -> Result<()> { + let mut num_priority_bits = 0; + let mut idx = 0; + for ix in gicr_typer { + let i = *ix; + for icc_offset in VGIC_ICC_REGS { + if *icc_offset == SYS_ICC_CTLR_EL1 { + let ctrl_el1 = state[idx]; + num_priority_bits = + ((ctrl_el1 & ICC_CTLR_EL1_PRIBITS_MASK) >> ICC_CTLR_EL1_PRIBITS_SHIFT) + 1; + } + if *icc_offset == SYS_ICC_AP0R1_EL1 || *icc_offset == SYS_ICC_AP1R1_EL1 { + if num_priority_bits >= 6 { + icc_attr_access(gic, *icc_offset, i, &state[idx], true)?; + idx += 1; + } + continue; + } + if *icc_offset == SYS_ICC_AP0R2_EL1 + || *icc_offset == SYS_ICC_AP0R3_EL1 + || *icc_offset == SYS_ICC_AP1R2_EL1 + || *icc_offset == SYS_ICC_AP1R3_EL1 + { + if num_priority_bits == 7 { + icc_attr_access(gic, *icc_offset, i, &state[idx], true)?; + idx += 1; + } + continue; + } + icc_attr_access(gic, *icc_offset, i, &state[idx], true)?; + idx += 1; + } + } + Ok(()) +} diff --git a/hypervisor/src/kvm/aarch64/gic/mod.rs b/hypervisor/src/kvm/aarch64/gic/mod.rs new file mode 100644 index 000000000..6b7a0cddc --- /dev/null +++ b/hypervisor/src/kvm/aarch64/gic/mod.rs @@ -0,0 +1,489 @@ +// Copyright 2022 Arm Limited (or its affiliates). All rights reserved. + +mod dist_regs; +mod icc_regs; +mod redist_regs; + +use crate::arch::aarch64::gic::{Error, Result, Vgic}; +use crate::kvm::kvm_bindings; +use crate::{CpuState, Device, Vm}; +use dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr}; +use icc_regs::{get_icc_regs, set_icc_regs}; +use redist_regs::{construct_gicr_typers, get_redist_regs, set_redist_regs}; +use serde::{Deserialize, Serialize}; +use std::any::Any; +use std::boxed::Box; +use std::convert::TryInto; +use std::sync::Arc; + +const GITS_CTLR: u32 = 0x0000; +const GITS_IIDR: u32 = 0x0004; +const GITS_CBASER: u32 = 0x0080; +const GITS_CWRITER: u32 = 0x0088; +const GITS_CREADR: u32 = 0x0090; +const GITS_BASER: u32 = 0x0100; + +/// Access an ITS device attribute. +/// +/// This is a helper function to get/set the ITS device attribute depending +/// the bool parameter `set` provided. +pub fn gicv3_its_attr_access( + its_device: &Arc, + group: u32, + attr: u32, + val: &u64, + set: bool, +) -> Result<()> { + let mut gicv3_its_attr = kvm_bindings::kvm_device_attr { + group, + attr: attr as u64, + addr: val as *const u64 as u64, + flags: 0, + }; + if set { + its_device + .set_device_attr(&gicv3_its_attr) + .map_err(Error::SetDeviceAttribute) + } else { + its_device + .get_device_attr(&mut gicv3_its_attr) + .map_err(Error::GetDeviceAttribute) + } +} + +/// Function that saves/restores ITS tables into guest RAM. +/// +/// The tables get flushed to guest RAM whenever the VM gets stopped. +pub fn gicv3_its_tables_access(its_device: &Arc, save: bool) -> Result<()> { + let attr = if save { + u64::from(kvm_bindings::KVM_DEV_ARM_ITS_SAVE_TABLES) + } else { + u64::from(kvm_bindings::KVM_DEV_ARM_ITS_RESTORE_TABLES) + }; + + let init_gic_attr = kvm_bindings::kvm_device_attr { + group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + attr, + addr: 0, + flags: 0, + }; + its_device + .set_device_attr(&init_gic_attr) + .map_err(Error::SetDeviceAttribute) +} + +pub struct KvmGicV3Its { + /// The hypervisor agnostic device for the GicV3 + device: Arc, + + /// The hypervisor agnostic device for the Its device + its_device: Option>, + + /// Vector holding values of GICR_TYPER for each vCPU + gicr_typers: Vec, + + /// GIC distributor address + dist_addr: u64, + + /// GIC distributor size + dist_size: u64, + + /// GIC distributors address + redists_addr: u64, + + /// GIC distributors size + redists_size: u64, + + /// GIC MSI address + msi_addr: u64, + + /// GIC MSI size + msi_size: u64, + + /// Number of CPUs handled by the device + vcpu_count: u64, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct Gicv3ItsState { + dist: Vec, + rdist: Vec, + icc: Vec, + // special register that enables interrupts and affinity routing + gicd_ctlr: u32, + its_ctlr: u64, + its_iidr: u64, + its_cbaser: u64, + its_cwriter: u64, + its_creadr: u64, + its_baser: [u64; 8], +} + +impl KvmGicV3Its { + /// Device trees specific constants + pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9; + + /// Returns the GIC version of the device + fn version() -> u32 { + kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3 + } + + fn device(&self) -> &Arc { + &self.device + } + + fn its_device(&self) -> Option<&Arc> { + self.its_device.as_ref() + } + + /// Setup the device-specific attributes + fn init_device_attributes(&mut self, vm: &dyn Vm, nr_irqs: u32) -> Result<()> { + // GicV3 part attributes + /* Setting up the distributor attribute. + We are placing the GIC below 1GB so we need to substract the size of the distributor. + */ + Self::set_device_attribute( + self.device(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST), + &self.dist_addr as *const u64 as u64, + 0, + )?; + + /* Setting up the redistributors' attribute. + We are calculating here the start of the redistributors address. We have one per CPU. + */ + Self::set_device_attribute( + self.device(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST), + &self.redists_addr as *const u64 as u64, + 0, + )?; + + // ITS part attributes + let mut its_device = kvm_bindings::kvm_create_device { + type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS, + fd: 0, + flags: 0, + }; + + let its_fd = vm + .create_device(&mut its_device) + .map_err(Error::CreateGic)?; + + Self::set_device_attribute( + &its_fd, + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE), + &self.msi_addr as *const u64 as u64, + 0, + )?; + + Self::set_device_attribute( + &its_fd, + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), + 0, + 0, + )?; + + self.set_its_device(Some(its_fd)); + + /* We need to tell the kernel how many irqs to support with this vgic. + * See the `layout` module for details. + */ + let nr_irqs_ptr = &nr_irqs as *const u32; + Self::set_device_attribute( + self.device(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, + nr_irqs_ptr as u64, + 0, + )?; + + /* Finalize the GIC. + * See https://code.woboq.org/linux/linux/virt/kvm/arm/vgic/vgic-kvm-device.c.html#211. + */ + Self::set_device_attribute( + self.device(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), + 0, + 0, + ) + } + + /// Create a KVM Vgic device + fn create_device(vm: &dyn Vm) -> Result> { + let mut gic_device = kvm_bindings::kvm_create_device { + type_: Self::version(), + fd: 0, + flags: 0, + }; + + vm.create_device(&mut gic_device).map_err(Error::CreateGic) + } + + /// Set a GIC device attribute + fn set_device_attribute( + device: &Arc, + group: u32, + attr: u64, + addr: u64, + flags: u32, + ) -> Result<()> { + let attr = kvm_bindings::kvm_device_attr { + flags, + group, + attr, + addr, + }; + device + .set_device_attr(&attr) + .map_err(Error::SetDeviceAttribute) + } + + /// Function that saves RDIST pending tables into guest RAM. + /// + /// The tables get flushed to guest RAM whenever the VM gets stopped. + pub fn save_pending_tables(vgic: &Arc) -> Result<()> { + let init_gic_attr = kvm_bindings::kvm_device_attr { + group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + attr: u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES), + addr: 0, + flags: 0, + }; + vgic.set_device_attr(&init_gic_attr) + .map_err(Error::SetDeviceAttribute) + } + + /// Method to initialize the GIC device + #[allow(clippy::new_ret_no_self)] + pub fn new( + vm: &dyn Vm, + vcpu_count: u64, + dist_addr: u64, + dist_size: u64, + redist_size: u64, + msi_size: u64, + nr_irqs: u32, + ) -> Result> { + let vgic = Self::create_device(vm)?; + let redists_size: u64 = redist_size * vcpu_count; + let redists_addr: u64 = dist_addr - redists_size; + let msi_addr: u64 = redists_addr - msi_size; + + let mut gic_device = Box::new(KvmGicV3Its { + device: vgic, + its_device: None, + gicr_typers: vec![0; vcpu_count.try_into().unwrap()], + dist_addr, + dist_size, + redists_addr, + redists_size, + msi_addr, + msi_size, + vcpu_count, + }); + + gic_device.init_device_attributes(vm, nr_irqs)?; + + Ok(gic_device) + } +} + +impl Vgic for KvmGicV3Its { + fn fdt_compatibility(&self) -> &str { + "arm,gic-v3" + } + + fn msi_compatible(&self) -> bool { + true + } + + fn msi_compatibility(&self) -> &str { + "arm,gic-v3-its" + } + + fn fdt_maint_irq(&self) -> u32 { + KvmGicV3Its::ARCH_GIC_V3_MAINT_IRQ + } + + fn vcpu_count(&self) -> u64 { + self.vcpu_count + } + + fn device_properties(&self) -> [u64; 4] { + [ + self.dist_addr, + self.dist_size, + self.redists_addr, + self.redists_size, + ] + } + + fn msi_properties(&self) -> [u64; 2] { + [self.msi_addr, self.msi_size] + } + + fn set_its_device(&mut self, its_device: Option>) { + self.its_device = its_device; + } + + fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) { + let gicr_typers = construct_gicr_typers(vcpu_states); + self.gicr_typers = gicr_typers; + } + + fn as_any_concrete_mut(&mut self) -> &mut dyn Any { + self + } + + /// Save the state of GICv3ITS. + fn state(&self, gicr_typers: &[u64]) -> Result { + let gicd_ctlr = read_ctlr(self.device())?; + + let dist_state = get_dist_regs(self.device())?; + + let rdist_state = get_redist_regs(self.device(), gicr_typers)?; + + let icc_state = get_icc_regs(self.device(), gicr_typers)?; + + let its_baser_state: [u64; 8] = [0; 8]; + for i in 0..8 { + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_BASER + i * 8, + &its_baser_state[i as usize], + false, + )?; + } + + let its_ctlr_state: u64 = 0; + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CTLR, + &its_ctlr_state, + false, + )?; + + let its_cbaser_state: u64 = 0; + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CBASER, + &its_cbaser_state, + false, + )?; + + let its_creadr_state: u64 = 0; + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CREADR, + &its_creadr_state, + false, + )?; + + let its_cwriter_state: u64 = 0; + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CWRITER, + &its_cwriter_state, + false, + )?; + + let its_iidr_state: u64 = 0; + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_IIDR, + &its_iidr_state, + false, + )?; + + Ok(Gicv3ItsState { + dist: dist_state, + rdist: rdist_state, + icc: icc_state, + gicd_ctlr, + its_ctlr: its_ctlr_state, + its_iidr: its_iidr_state, + its_cbaser: its_cbaser_state, + its_cwriter: its_cwriter_state, + its_creadr: its_creadr_state, + its_baser: its_baser_state, + }) + } + + /// Restore the state of GICv3ITS. + fn set_state(&mut self, gicr_typers: &[u64], state: &Gicv3ItsState) -> Result<()> { + write_ctlr(self.device(), state.gicd_ctlr)?; + + set_dist_regs(self.device(), &state.dist)?; + + set_redist_regs(self.device(), gicr_typers, &state.rdist)?; + + set_icc_regs(self.device(), gicr_typers, &state.icc)?; + + //Restore GICv3ITS registers + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_IIDR, + &state.its_iidr, + true, + )?; + + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CBASER, + &state.its_cbaser, + true, + )?; + + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CREADR, + &state.its_creadr, + true, + )?; + + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CWRITER, + &state.its_cwriter, + true, + )?; + + for i in 0..8 { + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_BASER + i * 8, + &state.its_baser[i as usize], + true, + )?; + } + + // Restore ITS tables + gicv3_its_tables_access(self.its_device().unwrap(), false)?; + + gicv3_its_attr_access( + self.its_device().unwrap(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS, + GITS_CTLR, + &state.its_ctlr, + true, + )?; + + Ok(()) + } +} diff --git a/hypervisor/src/kvm/aarch64/gic/redist_regs.rs b/hypervisor/src/kvm/aarch64/gic/redist_regs.rs new file mode 100644 index 000000000..b33ab8768 --- /dev/null +++ b/hypervisor/src/kvm/aarch64/gic/redist_regs.rs @@ -0,0 +1,207 @@ +// Copyright 2022 Arm Limited (or its affiliates). All rights reserved. +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::arch::aarch64::gic::{Error, Result}; +use crate::kvm::kvm_bindings::{kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS}; +use crate::{CpuState, Device}; +use std::sync::Arc; + +// Relevant redistributor registers that we want to save/restore. +const GICR_CTLR: u32 = 0x0000; +const GICR_STATUSR: u32 = 0x0010; +const GICR_WAKER: u32 = 0x0014; +const GICR_PROPBASER: u32 = 0x0070; +const GICR_PENDBASER: u32 = 0x0078; + +/* SGI and PPI Redistributor registers, offsets from RD_base */ +/* + * Redistributor frame offsets from RD_base which is actually SZ_ + */ +const GICR_SGI_OFFSET: u32 = 0x0001_0000; +const GICR_IGROUPR0: u32 = GICR_SGI_OFFSET + 0x0080; +const GICR_ICENABLER0: u32 = GICR_SGI_OFFSET + 0x0180; +const GICR_ISENABLER0: u32 = GICR_SGI_OFFSET + 0x0100; +const GICR_ISPENDR0: u32 = GICR_SGI_OFFSET + 0x0200; +const GICR_ICPENDR0: u32 = GICR_SGI_OFFSET + 0x0280; +const GICR_ISACTIVER0: u32 = GICR_SGI_OFFSET + 0x0300; +const GICR_ICACTIVER0: u32 = GICR_SGI_OFFSET + 0x0380; +const GICR_IPRIORITYR0: u32 = GICR_SGI_OFFSET + 0x0400; +const GICR_ICFGR0: u32 = GICR_SGI_OFFSET + 0x0C00; + +const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32; +const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64; + +/// This is how we represent the registers of a distributor. +/// It is relrvant their offset from the base address of the +/// distributor. +/// Each register has a different number +/// of bits_per_irq and is therefore variable length. +/// First 32 interrupts (0-32) are private to each CPU (SGIs and PPIs). +/// and so we save the first irq to identify between the type of the interrupt +/// that the respective register deals with. +struct RdistReg { + /// Offset from distributor address. + base: u32, + /// Length of the register. + length: u8, +} + +// All or at least the registers we are interested in are 32 bit, so +// we use a constant for size(u32). +const REG_SIZE: u8 = 4; + +// Creates a vgic redistributor register. +macro_rules! VGIC_RDIST_REG { + ($base:expr, $len:expr) => { + RdistReg { + base: $base, + length: $len, + } + }; +} + +// List with relevant distributor registers that we will be restoring. +static VGIC_RDIST_REGS: &[RdistReg] = &[ + VGIC_RDIST_REG!(GICR_STATUSR, 4), + VGIC_RDIST_REG!(GICR_WAKER, 4), + VGIC_RDIST_REG!(GICR_PROPBASER, 8), + VGIC_RDIST_REG!(GICR_PENDBASER, 8), + VGIC_RDIST_REG!(GICR_CTLR, 4), +]; + +// List with relevant distributor registers that we will be restoring. +static VGIC_SGI_REGS: &[RdistReg] = &[ + VGIC_RDIST_REG!(GICR_IGROUPR0, 4), + VGIC_RDIST_REG!(GICR_ICENABLER0, 4), + VGIC_RDIST_REG!(GICR_ISENABLER0, 4), + VGIC_RDIST_REG!(GICR_ICFGR0, 8), + VGIC_RDIST_REG!(GICR_ICPENDR0, 4), + VGIC_RDIST_REG!(GICR_ISPENDR0, 4), + VGIC_RDIST_REG!(GICR_ICACTIVER0, 4), + VGIC_RDIST_REG!(GICR_ISACTIVER0, 4), + VGIC_RDIST_REG!(GICR_IPRIORITYR0, 32), +]; + +fn redist_attr_access( + gic: &Arc, + offset: u32, + typer: u64, + val: &u32, + set: bool, +) -> Result<()> { + let mut gic_dist_attr = kvm_device_attr { + group: KVM_DEV_ARM_VGIC_GRP_REDIST_REGS, + attr: (typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | (offset as u64), // this needs the mpidr + addr: val as *const u32 as u64, + flags: 0, + }; + if set { + gic.set_device_attr(&gic_dist_attr) + .map_err(Error::SetDeviceAttribute)?; + } else { + gic.get_device_attr(&mut gic_dist_attr) + .map_err(Error::GetDeviceAttribute)?; + } + Ok(()) +} + +fn access_redists_aux( + gic: &Arc, + gicr_typer: &[u64], + state: &mut Vec, + reg_list: &[RdistReg], + idx: &mut usize, + set: bool, +) -> Result<()> { + for i in gicr_typer { + for rdreg in reg_list { + let mut base = rdreg.base; + let end = base + rdreg.length as u32; + + while base < end { + let mut val = 0; + if set { + val = state[*idx]; + redist_attr_access(gic, base, *i, &val, true)?; + *idx += 1; + } else { + redist_attr_access(gic, base, *i, &val, false)?; + state.push(val); + } + base += REG_SIZE as u32; + } + } + } + Ok(()) +} + +/// Get redistributor registers. +pub fn get_redist_regs(gic: &Arc, gicr_typer: &[u64]) -> Result> { + let mut state = Vec::new(); + let mut idx: usize = 0; + access_redists_aux( + gic, + gicr_typer, + &mut state, + VGIC_RDIST_REGS, + &mut idx, + false, + )?; + + access_redists_aux(gic, gicr_typer, &mut state, VGIC_SGI_REGS, &mut idx, false)?; + Ok(state) +} + +/// Set redistributor registers. +pub fn set_redist_regs(gic: &Arc, gicr_typer: &[u64], state: &[u32]) -> Result<()> { + let mut idx: usize = 0; + let mut mut_state = state.to_owned(); + access_redists_aux( + gic, + gicr_typer, + &mut mut_state, + VGIC_RDIST_REGS, + &mut idx, + true, + )?; + access_redists_aux( + gic, + gicr_typer, + &mut mut_state, + VGIC_SGI_REGS, + &mut idx, + true, + ) +} + +pub fn construct_gicr_typers(vcpu_states: &[CpuState]) -> Vec { + /* Pre-construct the GICR_TYPER: + * For our implementation: + * Top 32 bits are the affinity value of the associated CPU + * CommonLPIAff == 01 (redistributors with same Aff3 share LPI table) + * Processor_Number == CPU index starting from 0 + * DPGS == 0 (GICR_CTLR.DPG* not supported) + * Last == 1 if this is the last redistributor in a series of + * contiguous redistributor pages + * DirectLPI == 0 (direct injection of LPIs not supported) + * VLPIS == 0 (virtual LPIs not supported) + * PLPIS == 0 (physical LPIs not supported) + */ + let mut gicr_typers: Vec = Vec::new(); + for (index, state) in vcpu_states.iter().enumerate() { + let last = { + if index == vcpu_states.len() - 1 { + 1 + } else { + 0 + } + }; + //calculate affinity + let mut cpu_affid = state.mpidr & 1095233437695; + cpu_affid = ((cpu_affid & 0xFF00000000) >> 8) | (cpu_affid & 0xFFFFFF); + gicr_typers.push((cpu_affid << 32) | (1 << 24) | (index as u64) << 8 | (last << 4)); + } + + gicr_typers +} diff --git a/hypervisor/src/kvm/aarch64/mod.rs b/hypervisor/src/kvm/aarch64/mod.rs index e23eebade..903cdc052 100644 --- a/hypervisor/src/kvm/aarch64/mod.rs +++ b/hypervisor/src/kvm/aarch64/mod.rs @@ -8,9 +8,8 @@ // // -/// -/// Export generically-named wrappers of kvm-bindings for Unix-based platforms -/// +pub mod gic; + use crate::kvm::{KvmError, KvmResult}; use kvm_bindings::{ kvm_mp_state, kvm_one_reg, kvm_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index 4aeaa1e28..77cf4b85f 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -8,11 +8,15 @@ // // +#[cfg(target_arch = "aarch64")] +use crate::aarch64::gic::KvmGicV3Its; #[cfg(target_arch = "aarch64")] pub use crate::aarch64::{ - check_required_kvm_extensions, is_system_register, VcpuInit, VcpuKvmState as CpuState, - MPIDR_EL1, + check_required_kvm_extensions, gic::Gicv3ItsState as GicState, is_system_register, VcpuInit, + VcpuKvmState as CpuState, MPIDR_EL1, }; +#[cfg(target_arch = "aarch64")] +use crate::arch::aarch64::gic::Vgic; use crate::cpu; use crate::device; use crate::hypervisor; @@ -250,6 +254,30 @@ impl vm::Vm for KvmVm { }; Ok(Arc::new(vcpu)) } + #[cfg(target_arch = "aarch64")] + /// + /// Creates a virtual GIC device. + /// + fn create_vgic( + &self, + vcpu_count: u64, + dist_addr: u64, + dist_size: u64, + redist_size: u64, + msi_size: u64, + nr_irqs: u32, + ) -> vm::Result> { + KvmGicV3Its::new( + self, + vcpu_count, + dist_addr, + dist_size, + redist_size, + msi_size, + nr_irqs, + ) + .map_err(|e| vm::HypervisorVmError::CreateVgic(anyhow!("Vgic error {:?}", e))) + } /// /// Registers an event to be signaled whenever a certain address is written to. /// diff --git a/hypervisor/src/lib.rs b/hypervisor/src/lib.rs index 1e557a2e9..c3c422ea0 100644 --- a/hypervisor/src/lib.rs +++ b/hypervisor/src/lib.rs @@ -51,10 +51,10 @@ mod device; pub use cpu::{HypervisorCpuError, Vcpu, VmExit}; pub use device::{Device, HypervisorDeviceError}; pub use hypervisor::{Hypervisor, HypervisorError}; -#[cfg(all(feature = "kvm", target_arch = "aarch64"))] -pub use kvm::aarch64; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] pub use kvm::x86_64; +#[cfg(all(feature = "kvm", target_arch = "aarch64"))] +pub use kvm::{aarch64, GicState}; // Aliased types exposed from both hypervisors #[cfg(feature = "kvm")] pub use kvm::{ diff --git a/hypervisor/src/vm.rs b/hypervisor/src/vm.rs index cf3c49b69..d53cb1f0b 100644 --- a/hypervisor/src/vm.rs +++ b/hypervisor/src/vm.rs @@ -10,6 +10,8 @@ #[cfg(target_arch = "aarch64")] use crate::aarch64::VcpuInit; +#[cfg(target_arch = "aarch64")] +use crate::arch::aarch64::gic::Vgic; use crate::cpu::Vcpu; use crate::device::Device; #[cfg(feature = "kvm")] @@ -210,6 +212,11 @@ pub enum HypervisorVmError { /// #[error("Failed to initialize memory region TDX: {0}")] InitMemRegionTdx(#[source] std::io::Error), + /// + /// Create Vgic error + /// + #[error("Failed to create Vgic: {0}")] + CreateVgic(#[source] anyhow::Error), } /// /// Result type for returning from a function @@ -269,6 +276,17 @@ pub trait Vm: Send + Sync { fn unregister_irqfd(&self, fd: &EventFd, gsi: u32) -> Result<()>; /// Creates a new KVM vCPU file descriptor and maps the memory corresponding fn create_vcpu(&self, id: u8, vm_ops: Option>) -> Result>; + #[cfg(target_arch = "aarch64")] + fn create_vgic( + &self, + vcpu_count: u64, + dist_addr: u64, + dist_size: u64, + redist_size: u64, + msi_size: u64, + nr_irqs: u32, + ) -> Result>; + /// Registers an event to be signaled whenever a certain address is written to. fn register_ioevent( &self,