From e3e771727a5395a724c8e82df3b7526dd35efdab Mon Sep 17 00:00:00 2001 From: Michael Zhao Date: Thu, 16 Jul 2020 14:10:59 +0800 Subject: [PATCH] arch: Refactor GIC code to seperate KVM specific code Shrink GICDevice trait to contain hypervisor agnostic API's only, which are used in generating FDT. Move all KVM specific logic into KvmGICDevice trait. Signed-off-by: Michael Zhao --- arch/src/aarch64/gic.rs | 277 ++++++++++++++++++---------------- arch/src/aarch64/gicv2.rs | 209 ++++++++++++------------- arch/src/aarch64/gicv3.rs | 216 +++++++++++++------------- arch/src/aarch64/gicv3_its.rs | 231 ++++++++++++++-------------- arch/src/aarch64/mod.rs | 4 +- vmm/src/interrupt.rs | 2 +- vmm/src/vm.rs | 2 +- 7 files changed, 485 insertions(+), 456 deletions(-) diff --git a/arch/src/aarch64/gic.rs b/arch/src/aarch64/gic.rs index 560524ce5..5ebd1e581 100644 --- a/arch/src/aarch64/gic.rs +++ b/arch/src/aarch64/gic.rs @@ -1,12 +1,8 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use super::gicv2::GICv2; -use super::gicv3::GICv3; -use super::gicv3_its::GICv3ITS; use kvm_ioctls::DeviceFd; -use std::sync::Arc; -use std::{boxed::Box, result}; +use std::result; /// Errors thrown while setting up the GIC. #[derive(Debug)] @@ -14,31 +10,25 @@ pub enum Error { /// Error while calling KVM ioctl for setting up the global interrupt controller. CreateGIC(hypervisor::HypervisorVmError), /// Error while setting device attributes for the GIC. - SetDeviceAttribute(kvm_ioctls::Error), + SetDeviceAttribute(hypervisor::kvm_ioctls::Error), } type Result = result::Result; -/// Trait for GIC devices. -pub trait GICDevice: Send + Sync { +pub trait GICDevice { /// Returns the file descriptor of the GIC device fn device_fd(&self) -> &DeviceFd; - /// Returns an array with GIC device properties - fn device_properties(&self) -> &[u64]; - - /// Returns the number of vCPUs this GIC handles - fn vcpu_count(&self) -> u64; - /// Returns the fdt compatibility property of the device fn fdt_compatibility(&self) -> &str; /// Returns the maint_irq fdt property of the device fn fdt_maint_irq(&self) -> u32; - /// Returns the GIC version of the device - fn version() -> u32 - where - Self: Sized; + /// Returns an array with GIC device properties + fn device_properties(&self) -> &[u64]; + + /// Returns the number of vCPUs this GIC handles + fn vcpu_count(&self) -> u64; /// Returns whether the GIC device is MSI compatible or not fn msi_compatible(&self) -> bool { @@ -54,124 +44,145 @@ pub trait GICDevice: Send + Sync { fn msi_properties(&self) -> &[u64] { &[] } +} - /// Create the GIC device object - fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box - where - Self: Sized; +pub mod kvm { + use super::super::gicv2::kvm::KvmGICv2; + use super::super::gicv3::kvm::KvmGICv3; + use super::super::gicv3_its::kvm::KvmGICv3ITS; + use super::super::layout; + use super::GICDevice; + use super::Result; + use kvm_ioctls::DeviceFd; + use std::boxed::Box; + use std::sync::Arc; - /// Setup the device-specific attributes - fn init_device_attributes( + /// Trait for GIC devices. + pub trait KvmGICDevice: Send + Sync + GICDevice { + /// Returns the GIC version of the device + fn version() -> u32 + where + Self: Sized; + + /// Create the GIC device object + fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box + where + Self: Sized; + + /// Setup the device-specific attributes + fn init_device_attributes( + vm: &Arc, + gic_device: &Box, + ) -> Result<()> + where + Self: Sized; + + /// Initialize a GIC device + fn init_device(vm: &Arc) -> Result + where + Self: Sized, + { + let mut gic_device = kvm_bindings::kvm_create_device { + type_: Self::version(), + fd: 0, + flags: 0, + }; + + vm.create_device(&mut gic_device) + .map_err(super::Error::CreateGIC) + } + + /// Set a GIC device attribute + fn set_device_attribute( + fd: &DeviceFd, + group: u32, + attr: u64, + addr: u64, + flags: u32, + ) -> Result<()> + where + Self: Sized, + { + let attr = kvm_bindings::kvm_device_attr { + group: group, + attr: attr, + addr: addr, + flags: flags, + }; + fd.set_device_attr(&attr) + .map_err(super::Error::SetDeviceAttribute)?; + + Ok(()) + } + + /// Finalize the setup of a GIC device + fn finalize_device(gic_device: &Box) -> Result<()> + where + Self: Sized, + { + /* We need to tell the kernel how many irqs to support with this vgic. + * See the `layout` module for details. + */ + let nr_irqs: u32 = layout::IRQ_MAX - layout::IRQ_BASE + 1; + let nr_irqs_ptr = &nr_irqs as *const u32; + Self::set_device_attribute( + gic_device.device_fd(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, + nr_irqs_ptr as u64, + 0, + )?; + + /* Finalize the GIC. + * See https://code.woboq.org/linux/linux/virt/kvm/arm/vgic/vgic-kvm-device.c.html#211. + */ + Self::set_device_attribute( + gic_device.device_fd(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), + 0, + 0, + )?; + + Ok(()) + } + + /// Method to initialize the GIC device + fn new(vm: &Arc, vcpu_count: u64) -> Result> + where + Self: Sized, + { + let vgic_fd = Self::init_device(vm)?; + + let device = Self::create_device(vgic_fd, vcpu_count); + + Self::init_device_attributes(vm, &device)?; + + Self::finalize_device(&device)?; + + Ok(device) + } + } + + /// Create a GIC device. + /// + /// It will try to create by default a GICv3 device. If that fails it will try + /// to fall-back to a GICv2 device. + pub fn create_gic( vm: &Arc, - gic_device: &Box, - ) -> Result<()> - where - Self: Sized; - - /// Initialize a GIC device - fn init_device(vm: &Arc) -> Result - where - Self: Sized, - { - let mut gic_device = kvm_bindings::kvm_create_device { - type_: Self::version(), - fd: 0, - flags: 0, - }; - - vm.create_device(&mut gic_device).map_err(Error::CreateGIC) - } - - /// Set a GIC device attribute - fn set_device_attribute( - fd: &DeviceFd, - group: u32, - attr: u64, - addr: u64, - flags: u32, - ) -> Result<()> - where - Self: Sized, - { - let attr = kvm_bindings::kvm_device_attr { - group: group, - attr: attr, - addr: addr, - flags: flags, - }; - fd.set_device_attr(&attr) - .map_err(Error::SetDeviceAttribute)?; - - Ok(()) - } - - /// Finalize the setup of a GIC device - fn finalize_device(gic_device: &Box) -> Result<()> - where - Self: Sized, - { - /* We need to tell the kernel how many irqs to support with this vgic. - * See the `layout` module for details. - */ - let nr_irqs: u32 = super::layout::IRQ_MAX - super::layout::IRQ_BASE + 1; - let nr_irqs_ptr = &nr_irqs as *const u32; - Self::set_device_attribute( - gic_device.device_fd(), - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_NR_IRQS, - 0, - nr_irqs_ptr as u64, - 0, - )?; - - /* Finalize the GIC. - * See https://code.woboq.org/linux/linux/virt/kvm/arm/vgic/vgic-kvm-device.c.html#211. - */ - Self::set_device_attribute( - gic_device.device_fd(), - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, - u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), - 0, - 0, - )?; - - Ok(()) - } - - /// Method to initialize the GIC device - fn new(vm: &Arc, vcpu_count: u64) -> Result> - where - Self: Sized, - { - let vgic_fd = Self::init_device(vm)?; - - let device = Self::create_device(vgic_fd, vcpu_count); - - Self::init_device_attributes(vm, &device)?; - - Self::finalize_device(&device)?; - - Ok(device) - } -} - -/// Create a GIC device. -/// -/// It will try to create by default a GICv3 device. If that fails it will try -/// to fall-back to a GICv2 device. -pub fn create_gic( - vm: &Arc, - vcpu_count: u64, - its_required: bool, -) -> Result> { - if its_required { - GICv3ITS::new(vm, vcpu_count) - } else { - GICv3ITS::new(vm, vcpu_count).or_else(|_| { - debug!("Failed to create GICv3-ITS, will try GICv3 instead."); - GICv3::new(vm, vcpu_count).or_else(|_| { - debug!("Failed to create GICv3, will try GICv2 instead."); - GICv2::new(vm, vcpu_count) + vcpu_count: u64, + its_required: bool, + ) -> Result> { + if its_required { + KvmGICv3ITS::new(vm, vcpu_count) + } else { + KvmGICv3ITS::new(vm, vcpu_count).or_else(|_| { + debug!("Failed to create GICv3-ITS, will try GICv3 instead."); + KvmGICv3::new(vm, vcpu_count).or_else(|_| { + debug!("Failed to create GICv3, will try GICv2 instead."); + KvmGICv2::new(vm, vcpu_count) + }) }) - }) + } } } diff --git a/arch/src/aarch64/gicv2.rs b/arch/src/aarch64/gicv2.rs index afa6934e3..44b90afd6 100644 --- a/arch/src/aarch64/gicv2.rs +++ b/arch/src/aarch64/gicv2.rs @@ -1,116 +1,121 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use super::gic::{Error, GICDevice}; -use kvm_ioctls::DeviceFd; -use std::sync::Arc; -use std::{boxed::Box, result}; +pub mod kvm { + use super::super::gic::{Error, GICDevice}; + use std::{boxed::Box, result}; + type Result = result::Result; + use super::super::gic::kvm::KvmGICDevice; + use super::super::layout; + use kvm_ioctls::DeviceFd; + use std::sync::Arc; -type Result = result::Result; + /// Represent a GIC v2 device + pub struct KvmGICv2 { + /// The file descriptor for the KVM device + fd: DeviceFd, -/// Represent a GIC v2 device -pub struct GICv2 { - /// The file descriptor for the KVM device - fd: DeviceFd, + /// GIC device properties, to be used for setting up the fdt entry + properties: [u64; 4], - /// GIC device properties, to be used for setting up the fdt entry - properties: [u64; 4], - - /// Number of CPUs handled by the device - vcpu_count: u64, -} - -impl GICv2 { - // Unfortunately bindgen omits defines that are based on other defines. - // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel. - const KVM_VGIC_V2_DIST_SIZE: u64 = 0x1000; - const KVM_VGIC_V2_CPU_SIZE: u64 = 0x2000; - - // Device trees specific constants - const ARCH_GIC_V2_MAINT_IRQ: u32 = 8; - - /// Get the address of the GICv2 distributor. - const fn get_dist_addr() -> u64 { - super::layout::MAPPED_IO_START - GICv2::KVM_VGIC_V2_DIST_SIZE + /// Number of CPUs handled by the device + vcpu_count: u64, } - /// Get the size of the GIC_v2 distributor. - const fn get_dist_size() -> u64 { - GICv2::KVM_VGIC_V2_DIST_SIZE + impl KvmGICv2 { + // Unfortunately bindgen omits defines that are based on other defines. + // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel. + const KVM_VGIC_V2_DIST_SIZE: u64 = 0x1000; + const KVM_VGIC_V2_CPU_SIZE: u64 = 0x2000; + + // Device trees specific constants + const ARCH_GIC_V2_MAINT_IRQ: u32 = 8; + + /// Get the address of the GICv2 distributor. + const fn get_dist_addr() -> u64 { + layout::MAPPED_IO_START - KvmGICv2::KVM_VGIC_V2_DIST_SIZE + } + + /// Get the size of the GIC_v2 distributor. + const fn get_dist_size() -> u64 { + KvmGICv2::KVM_VGIC_V2_DIST_SIZE + } + + /// Get the address of the GIC_v2 CPU. + const fn get_cpu_addr() -> u64 { + KvmGICv2::get_dist_addr() - KvmGICv2::KVM_VGIC_V2_CPU_SIZE + } + + /// Get the size of the GIC_v2 CPU. + const fn get_cpu_size() -> u64 { + KvmGICv2::KVM_VGIC_V2_CPU_SIZE + } } - /// Get the address of the GIC_v2 CPU. - const fn get_cpu_addr() -> u64 { - GICv2::get_dist_addr() - GICv2::KVM_VGIC_V2_CPU_SIZE + impl GICDevice for KvmGICv2 { + fn device_fd(&self) -> &DeviceFd { + &self.fd + } + + fn device_properties(&self) -> &[u64] { + &self.properties + } + + fn fdt_compatibility(&self) -> &str { + "arm,gic-400" + } + + fn fdt_maint_irq(&self) -> u32 { + KvmGICv2::ARCH_GIC_V2_MAINT_IRQ + } + + fn vcpu_count(&self) -> u64 { + self.vcpu_count + } } - /// Get the size of the GIC_v2 CPU. - const fn get_cpu_size() -> u64 { - GICv2::KVM_VGIC_V2_CPU_SIZE - } -} - -impl GICDevice for GICv2 { - fn version() -> u32 { - kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V2 - } - - fn device_fd(&self) -> &DeviceFd { - &self.fd - } - - fn device_properties(&self) -> &[u64] { - &self.properties - } - - fn vcpu_count(&self) -> u64 { - self.vcpu_count - } - - fn fdt_compatibility(&self) -> &str { - "arm,gic-400" - } - - fn fdt_maint_irq(&self) -> u32 { - GICv2::ARCH_GIC_V2_MAINT_IRQ - } - - fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { - Box::new(GICv2 { - fd: fd, - properties: [ - GICv2::get_dist_addr(), - GICv2::get_dist_size(), - GICv2::get_cpu_addr(), - GICv2::get_cpu_size(), - ], - vcpu_count: vcpu_count, - }) - } - - fn init_device_attributes( - _vm: &Arc, - gic_device: &Box, - ) -> Result<()> { - /* Setting up the distributor attribute. - We are placing the GIC below 1GB so we need to substract the size of the distributor. */ - Self::set_device_attribute( - &gic_device.device_fd(), - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, - u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_DIST), - &GICv2::get_dist_addr() as *const u64 as u64, - 0, - )?; - - /* Setting up the CPU attribute. */ - Self::set_device_attribute( - &gic_device.device_fd(), - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, - u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_CPU), - &GICv2::get_cpu_addr() as *const u64 as u64, - 0, - )?; - - Ok(()) + impl KvmGICDevice for KvmGICv2 { + fn version() -> u32 { + kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V2 + } + + fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { + Box::new(KvmGICv2 { + fd: fd, + properties: [ + KvmGICv2::get_dist_addr(), + KvmGICv2::get_dist_size(), + KvmGICv2::get_cpu_addr(), + KvmGICv2::get_cpu_size(), + ], + vcpu_count: vcpu_count, + }) + } + + fn init_device_attributes( + _vm: &Arc, + gic_device: &Box, + ) -> Result<()> { + /* Setting up the distributor attribute. + We are placing the GIC below 1GB so we need to substract the size of the distributor. */ + Self::set_device_attribute( + &gic_device.device_fd(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_DIST), + &KvmGICv2::get_dist_addr() as *const u64 as u64, + 0, + )?; + + /* Setting up the CPU attribute. */ + Self::set_device_attribute( + &gic_device.device_fd(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_CPU), + &KvmGICv2::get_cpu_addr() as *const u64 as u64, + 0, + )?; + + Ok(()) + } } } diff --git a/arch/src/aarch64/gicv3.rs b/arch/src/aarch64/gicv3.rs index 08fcb1d3d..04ca58f8a 100644 --- a/arch/src/aarch64/gicv3.rs +++ b/arch/src/aarch64/gicv3.rs @@ -1,119 +1,125 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use super::gic::{Error, GICDevice}; -use kvm_ioctls::DeviceFd; -use std::sync::Arc; -use std::{boxed::Box, result}; +pub mod kvm { + use super::super::gic::kvm::KvmGICDevice; + use super::super::gic::{Error, GICDevice}; + use super::super::layout; + use kvm_ioctls::DeviceFd; + use std::sync::Arc; + use std::{boxed::Box, result}; + type Result = result::Result; -type Result = result::Result; + pub struct KvmGICv3 { + /// The file descriptor for the KVM device + fd: DeviceFd, -pub struct GICv3 { - /// The file descriptor for the KVM device - fd: DeviceFd, + /// GIC device properties, to be used for setting up the fdt entry + properties: [u64; 4], - /// GIC device properties, to be used for setting up the fdt entry - properties: [u64; 4], - - /// Number of CPUs handled by the device - vcpu_count: u64, -} - -impl GICv3 { - // Unfortunately bindgen omits defines that are based on other defines. - // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel. - pub const SZ_64K: u64 = 0x0001_0000; - const KVM_VGIC_V3_DIST_SIZE: u64 = GICv3::SZ_64K; - const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * GICv3::SZ_64K); - - // Device trees specific constants - pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9; - - /// Get the address of the GIC distributor. - pub fn get_dist_addr() -> u64 { - super::layout::MAPPED_IO_START - GICv3::KVM_VGIC_V3_DIST_SIZE + /// Number of CPUs handled by the device + vcpu_count: u64, } - /// Get the size of the GIC distributor. - pub fn get_dist_size() -> u64 { - GICv3::KVM_VGIC_V3_DIST_SIZE + impl KvmGICv3 { + // Unfortunately bindgen omits defines that are based on other defines. + // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel. + pub const SZ_64K: u64 = 0x0001_0000; + const KVM_VGIC_V3_DIST_SIZE: u64 = KvmGICv3::SZ_64K; + const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * KvmGICv3::SZ_64K); + + // Device trees specific constants + pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9; + + /// Get the address of the GIC distributor. + pub fn get_dist_addr() -> u64 { + layout::MAPPED_IO_START - KvmGICv3::KVM_VGIC_V3_DIST_SIZE + } + + /// Get the size of the GIC distributor. + pub fn get_dist_size() -> u64 { + KvmGICv3::KVM_VGIC_V3_DIST_SIZE + } + + /// Get the address of the GIC redistributors. + pub fn get_redists_addr(vcpu_count: u64) -> u64 { + KvmGICv3::get_dist_addr() - KvmGICv3::get_redists_size(vcpu_count) + } + + /// Get the size of the GIC redistributors. + pub fn get_redists_size(vcpu_count: u64) -> u64 { + vcpu_count * KvmGICv3::KVM_VGIC_V3_REDIST_SIZE + } } - /// Get the address of the GIC redistributors. - pub fn get_redists_addr(vcpu_count: u64) -> u64 { - GICv3::get_dist_addr() - GICv3::get_redists_size(vcpu_count) + impl GICDevice for KvmGICv3 { + fn device_fd(&self) -> &DeviceFd { + &self.fd + } + + fn fdt_compatibility(&self) -> &str { + "arm,gic-v3" + } + + fn fdt_maint_irq(&self) -> u32 { + KvmGICv3::ARCH_GIC_V3_MAINT_IRQ + } + + fn device_properties(&self) -> &[u64] { + &self.properties + } + + fn vcpu_count(&self) -> u64 { + self.vcpu_count + } } - /// Get the size of the GIC redistributors. - pub fn get_redists_size(vcpu_count: u64) -> u64 { - vcpu_count * GICv3::KVM_VGIC_V3_REDIST_SIZE - } -} - -impl GICDevice for GICv3 { - fn version() -> u32 { - kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3 - } - - fn device_fd(&self) -> &DeviceFd { - &self.fd - } - - fn device_properties(&self) -> &[u64] { - &self.properties - } - - fn vcpu_count(&self) -> u64 { - self.vcpu_count - } - - fn fdt_compatibility(&self) -> &str { - "arm,gic-v3" - } - - fn fdt_maint_irq(&self) -> u32 { - GICv3::ARCH_GIC_V3_MAINT_IRQ - } - - fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { - Box::new(GICv3 { - fd: fd, - properties: [ - GICv3::get_dist_addr(), - GICv3::get_dist_size(), - GICv3::get_redists_addr(vcpu_count), - GICv3::get_redists_size(vcpu_count), - ], - vcpu_count: vcpu_count, - }) - } - - fn init_device_attributes( - _vm: &Arc, - gic_device: &Box, - ) -> Result<()> { - /* Setting up the distributor attribute. - We are placing the GIC below 1GB so we need to substract the size of the distributor. - */ - Self::set_device_attribute( - &gic_device.device_fd(), - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, - u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST), - &GICv3::get_dist_addr() as *const u64 as u64, - 0, - )?; - - /* Setting up the redistributors' attribute. - We are calculating here the start of the redistributors address. We have one per CPU. - */ - Self::set_device_attribute( - &gic_device.device_fd(), - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, - u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST), - &GICv3::get_redists_addr(u64::from(gic_device.vcpu_count())) as *const u64 as u64, - 0, - )?; - - Ok(()) + impl KvmGICDevice for KvmGICv3 { + fn version() -> u32 { + kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3 + } + + fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { + Box::new(KvmGICv3 { + fd: fd, + properties: [ + KvmGICv3::get_dist_addr(), + KvmGICv3::get_dist_size(), + KvmGICv3::get_redists_addr(vcpu_count), + KvmGICv3::get_redists_size(vcpu_count), + ], + vcpu_count: vcpu_count, + }) + } + + fn init_device_attributes( + _vm: &Arc, + gic_device: &Box, + ) -> Result<()> { + /* Setting up the distributor attribute. + We are placing the GIC below 1GB so we need to substract the size of the distributor. + */ + Self::set_device_attribute( + &gic_device.device_fd(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST), + &KvmGICv3::get_dist_addr() as *const u64 as u64, + 0, + )?; + + /* Setting up the redistributors' attribute. + We are calculating here the start of the redistributors address. We have one per CPU. + */ + Self::set_device_attribute( + &gic_device.device_fd(), + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST), + &KvmGICv3::get_redists_addr(u64::from(gic_device.vcpu_count())) as *const u64 + as u64, + 0, + )?; + + Ok(()) + } } } diff --git a/arch/src/aarch64/gicv3_its.rs b/arch/src/aarch64/gicv3_its.rs index dcbdd474a..964834030 100644 --- a/arch/src/aarch64/gicv3_its.rs +++ b/arch/src/aarch64/gicv3_its.rs @@ -1,123 +1,130 @@ // Copyright 2020 ARM Limited // SPDX-License-Identifier: Apache-2.0 -use super::gic::{Error, GICDevice}; -use super::gicv3::GICv3; -use kvm_ioctls::DeviceFd; -use std::sync::Arc; -use std::{boxed::Box, result}; +pub mod kvm { + use std::sync::Arc; + use std::{boxed::Box, result}; + type Result = result::Result; + use super::super::gic::kvm::KvmGICDevice; + use super::super::gic::{Error, GICDevice}; + use super::super::gicv3::kvm::KvmGICv3; + use kvm_ioctls::DeviceFd; -type Result = result::Result; + pub struct KvmGICv3ITS { + /// The file descriptor for the KVM device + fd: DeviceFd, -pub struct GICv3ITS { - /// The file descriptor for the KVM device - fd: DeviceFd, + /// GIC device properties, to be used for setting up the fdt entry + gic_properties: [u64; 4], - /// GIC device properties, to be used for setting up the fdt entry - gic_properties: [u64; 4], + /// MSI device properties, to be used for setting up the fdt entry + msi_properties: [u64; 2], - /// MSI device properties, to be used for setting up the fdt entry - msi_properties: [u64; 2], - - /// Number of CPUs handled by the device - vcpu_count: u64, -} - -impl GICv3ITS { - const KVM_VGIC_V3_ITS_SIZE: u64 = (2 * GICv3::SZ_64K); - - fn get_msi_size() -> u64 { - GICv3ITS::KVM_VGIC_V3_ITS_SIZE + /// Number of CPUs handled by the device + vcpu_count: u64, } - fn get_msi_addr(vcpu_count: u64) -> u64 { - GICv3::get_redists_addr(vcpu_count) - GICv3ITS::get_msi_size() - } -} - -impl GICDevice for GICv3ITS { - fn version() -> u32 { - GICv3::version() - } - - fn device_fd(&self) -> &DeviceFd { - &self.fd - } - - fn device_properties(&self) -> &[u64] { - &self.gic_properties - } - - fn msi_properties(&self) -> &[u64] { - &self.msi_properties - } - - fn vcpu_count(&self) -> u64 { - self.vcpu_count - } - - fn fdt_compatibility(&self) -> &str { - "arm,gic-v3" - } - - fn msi_compatible(&self) -> bool { - true - } - - fn msi_compatiblility(&self) -> &str { - "arm,gic-v3-its" - } - - fn fdt_maint_irq(&self) -> u32 { - GICv3::ARCH_GIC_V3_MAINT_IRQ - } - - fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { - Box::new(GICv3ITS { - fd: fd, - gic_properties: [ - GICv3::get_dist_addr(), - GICv3::get_dist_size(), - GICv3::get_redists_addr(vcpu_count), - GICv3::get_redists_size(vcpu_count), - ], - msi_properties: [GICv3ITS::get_msi_addr(vcpu_count), GICv3ITS::get_msi_size()], - vcpu_count: vcpu_count, - }) - } - - fn init_device_attributes( - vm: &Arc, - gic_device: &Box, - ) -> Result<()> { - GICv3::init_device_attributes(vm, gic_device)?; - - let mut its_device = kvm_bindings::kvm_create_device { - type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS, - fd: 0, - flags: 0, - }; - - let its_fd = vm - .create_device(&mut its_device) - .map_err(Error::CreateGIC)?; - - Self::set_device_attribute( - &its_fd, - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, - u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE), - &GICv3ITS::get_msi_addr(u64::from(gic_device.vcpu_count())) as *const u64 as u64, - 0, - )?; - - Self::set_device_attribute( - &its_fd, - kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, - u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), - 0, - 0, - )?; - - Ok(()) + impl KvmGICv3ITS { + const KVM_VGIC_V3_ITS_SIZE: u64 = (2 * KvmGICv3::SZ_64K); + + fn get_msi_size() -> u64 { + KvmGICv3ITS::KVM_VGIC_V3_ITS_SIZE + } + + fn get_msi_addr(vcpu_count: u64) -> u64 { + KvmGICv3::get_redists_addr(vcpu_count) - KvmGICv3ITS::get_msi_size() + } + } + + impl GICDevice for KvmGICv3ITS { + fn device_fd(&self) -> &DeviceFd { + &self.fd + } + + fn fdt_compatibility(&self) -> &str { + "arm,gic-v3" + } + + fn msi_compatible(&self) -> bool { + true + } + + fn msi_compatiblility(&self) -> &str { + "arm,gic-v3-its" + } + + fn fdt_maint_irq(&self) -> u32 { + KvmGICv3::ARCH_GIC_V3_MAINT_IRQ + } + + fn msi_properties(&self) -> &[u64] { + &self.msi_properties + } + + fn device_properties(&self) -> &[u64] { + &self.gic_properties + } + + fn vcpu_count(&self) -> u64 { + self.vcpu_count + } + } + + impl KvmGICDevice for KvmGICv3ITS { + fn version() -> u32 { + KvmGICv3::version() + } + + fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { + Box::new(KvmGICv3ITS { + fd: fd, + gic_properties: [ + KvmGICv3::get_dist_addr(), + KvmGICv3::get_dist_size(), + KvmGICv3::get_redists_addr(vcpu_count), + KvmGICv3::get_redists_size(vcpu_count), + ], + msi_properties: [ + KvmGICv3ITS::get_msi_addr(vcpu_count), + KvmGICv3ITS::get_msi_size(), + ], + vcpu_count: vcpu_count, + }) + } + + fn init_device_attributes( + vm: &Arc, + gic_device: &Box, + ) -> Result<()> { + KvmGICv3::init_device_attributes(vm, gic_device)?; + + let mut its_device = kvm_bindings::kvm_create_device { + type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS, + fd: 0, + flags: 0, + }; + + let its_fd = vm + .create_device(&mut its_device) + .map_err(Error::CreateGIC)?; + + Self::set_device_attribute( + &its_fd, + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE), + &KvmGICv3ITS::get_msi_addr(u64::from(gic_device.vcpu_count())) as *const u64 as u64, + 0, + )?; + + Self::set_device_attribute( + &its_fd, + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), + 0, + 0, + )?; + + Ok(()) + } } } diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs index 5c344346e..64d6c7a61 100644 --- a/arch/src/aarch64/mod.rs +++ b/arch/src/aarch64/mod.rs @@ -148,8 +148,8 @@ pub fn configure_system( // If pci_space_address is present, it means PCI devices are used ("pci" feature enabled). // Then GITv3-ITS is required for MSI messaging. // Otherwise ("mmio" feature enabled), any version of GIC is OK. - let gic_device = - gic::create_gic(vm, vcpu_count, pci_space_address.is_some()).map_err(Error::SetupGIC)?; + let gic_device = gic::kvm::create_gic(vm, vcpu_count, pci_space_address.is_some()) + .map_err(Error::SetupGIC)?; fdt::create_fdt( guest_mem, diff --git a/vmm/src/interrupt.rs b/vmm/src/interrupt.rs index 429c7c578..f11e1bc7c 100644 --- a/vmm/src/interrupt.rs +++ b/vmm/src/interrupt.rs @@ -395,7 +395,7 @@ pub mod kvm { #[cfg(target_arch = "aarch64")] #[cfg(test)] mod tests { - use arch::aarch64::gic::create_gic; + use arch::aarch64::gic::kvm::create_gic; #[test] fn test_create_gic() { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index ad4bb31de..6d4cc44db 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1493,7 +1493,7 @@ mod tests { mod tests { use super::*; use arch::aarch64::fdt::create_fdt; - use arch::aarch64::gic::create_gic; + use arch::aarch64::gic::kvm::create_gic; use arch::aarch64::{layout, DeviceInfoForFDT}; use arch::DeviceType; use vm_memory::{GuestAddress, GuestMemoryMmap};