diff --git a/arch/Cargo.toml b/arch/Cargo.toml index 66ac1866b..877bd87f3 100644 --- a/arch/Cargo.toml +++ b/arch/Cargo.toml @@ -15,7 +15,6 @@ kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branc kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" } libc = "0.2.72" vm-memory = { version = "0.2.1", features = ["backend-mmap"] } - acpi_tables = { path = "../acpi_tables", optional = true } arch_gen = { path = "../arch_gen" } diff --git a/arch/src/aarch64/fdt.rs b/arch/src/aarch64/fdt.rs index 9df96c464..5b0f2d2db 100644 --- a/arch/src/aarch64/fdt.rs +++ b/arch/src/aarch64/fdt.rs @@ -23,8 +23,11 @@ use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, Gue // This is a value for uniquely identifying the FDT node declaring the interrupt controller. const GIC_PHANDLE: u32 = 1; +// This is a value for uniquely identifying the FDT node declaring the MSI controller. +const MSI_PHANDLE: u32 = 2; // This is a value for uniquely identifying the FDT node containing the clock definition. -const CLOCK_PHANDLE: u32 = 2; +const CLOCK_PHANDLE: u32 = 3; + // Read the documentation specified when appending the root node to the FDT. const ADDRESS_CELLS: u32 = 0x2; const SIZE_CELLS: u32 = 0x2; @@ -389,6 +392,17 @@ fn create_gic_node(fdt: &mut Vec, gic_device: &Box) -> Result let gic_intr_prop = generate_prop32(&gic_intr); append_property(fdt, "interrupts", &gic_intr_prop)?; + + if gic_device.msi_compatible() { + append_begin_node(fdt, "msic")?; + append_property_string(fdt, "compatible", gic_device.msi_compatiblility())?; + append_property_null(fdt, "msi-controller")?; + append_property_u32(fdt, "phandle", MSI_PHANDLE)?; + let msi_reg_prop = generate_prop64(gic_device.msi_properties()); + append_property(fdt, "reg", &msi_reg_prop)?; + append_end_node(fdt)?; + } + append_end_node(fdt)?; Ok(()) diff --git a/arch/src/aarch64/gic.rs b/arch/src/aarch64/gic.rs index 1ec924446..3e603b004 100644 --- a/arch/src/aarch64/gic.rs +++ b/arch/src/aarch64/gic.rs @@ -7,6 +7,7 @@ use std::{boxed::Box, result}; use super::gicv2::GICv2; use super::gicv3::GICv3; +use super::gicv3_its::GICv3ITS; /// Errors thrown while setting up the GIC. #[derive(Debug)] @@ -40,13 +41,31 @@ pub trait GICDevice: Send + Sync { where Self: Sized; + /// Returns whether the GIC device is MSI compatible or not + fn msi_compatible(&self) -> bool { + false + } + + /// Returns the MSI compatibility property of the device + fn msi_compatiblility(&self) -> &str { + "" + } + + /// Returns the MSI reg property of the device + fn msi_properties(&self) -> &[u64] { + &[] + } + /// Create the GIC device object fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box where Self: Sized; /// Setup the device-specific attributes - fn init_device_attributes(gic_device: &Box) -> Result<()> + fn init_device_attributes( + vm: &Arc, + gic_device: &Box, + ) -> Result<()> where Self: Sized; @@ -128,7 +147,7 @@ pub trait GICDevice: Send + Sync { let device = Self::create_device(vgic_fd, vcpu_count); - Self::init_device_attributes(&device)?; + Self::init_device_attributes(vm, &device)?; Self::finalize_device(&device)?; @@ -141,7 +160,8 @@ pub trait GICDevice: Send + Sync { /// It will try to create by default a GICv3 device. If that fails it will try /// to fall-back to a GICv2 device. pub fn create_gic(vm: &Arc, vcpu_count: u64) -> Result> { - GICv3::new(vm, vcpu_count).or_else(|_| GICv2::new(vm, vcpu_count)) + GICv3ITS::new(vm, vcpu_count) + .or_else(|_| GICv3::new(vm, vcpu_count).or_else(|_| GICv2::new(vm, vcpu_count))) } #[cfg(test)] diff --git a/arch/src/aarch64/gicv2.rs b/arch/src/aarch64/gicv2.rs index b68ab8c85..afa6934e3 100644 --- a/arch/src/aarch64/gicv2.rs +++ b/arch/src/aarch64/gicv2.rs @@ -1,11 +1,10 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use std::{boxed::Box, result}; - -use kvm_ioctls::DeviceFd; - use super::gic::{Error, GICDevice}; +use kvm_ioctls::DeviceFd; +use std::sync::Arc; +use std::{boxed::Box, result}; type Result = result::Result; @@ -89,7 +88,10 @@ impl GICDevice for GICv2 { }) } - fn init_device_attributes(gic_device: &Box) -> Result<()> { + fn init_device_attributes( + _vm: &Arc, + gic_device: &Box, + ) -> Result<()> { /* Setting up the distributor attribute. We are placing the GIC below 1GB so we need to substract the size of the distributor. */ Self::set_device_attribute( diff --git a/arch/src/aarch64/gicv3.rs b/arch/src/aarch64/gicv3.rs index 6c0d8c30c..08fcb1d3d 100644 --- a/arch/src/aarch64/gicv3.rs +++ b/arch/src/aarch64/gicv3.rs @@ -1,11 +1,10 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use std::{boxed::Box, result}; - -use kvm_ioctls::DeviceFd; - use super::gic::{Error, GICDevice}; +use kvm_ioctls::DeviceFd; +use std::sync::Arc; +use std::{boxed::Box, result}; type Result = result::Result; @@ -23,30 +22,30 @@ pub struct GICv3 { impl GICv3 { // Unfortunately bindgen omits defines that are based on other defines. // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel. - const SZ_64K: u64 = 0x0001_0000; + pub const SZ_64K: u64 = 0x0001_0000; const KVM_VGIC_V3_DIST_SIZE: u64 = GICv3::SZ_64K; const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * GICv3::SZ_64K); // Device trees specific constants - const ARCH_GIC_V3_MAINT_IRQ: u32 = 9; + pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9; /// Get the address of the GIC distributor. - fn get_dist_addr() -> u64 { + pub fn get_dist_addr() -> u64 { super::layout::MAPPED_IO_START - GICv3::KVM_VGIC_V3_DIST_SIZE } /// Get the size of the GIC distributor. - fn get_dist_size() -> u64 { + pub fn get_dist_size() -> u64 { GICv3::KVM_VGIC_V3_DIST_SIZE } /// Get the address of the GIC redistributors. - fn get_redists_addr(vcpu_count: u64) -> u64 { + pub fn get_redists_addr(vcpu_count: u64) -> u64 { GICv3::get_dist_addr() - GICv3::get_redists_size(vcpu_count) } /// Get the size of the GIC redistributors. - fn get_redists_size(vcpu_count: u64) -> u64 { + pub fn get_redists_size(vcpu_count: u64) -> u64 { vcpu_count * GICv3::KVM_VGIC_V3_REDIST_SIZE } } @@ -89,7 +88,10 @@ impl GICDevice for GICv3 { }) } - fn init_device_attributes(gic_device: &Box) -> Result<()> { + fn init_device_attributes( + _vm: &Arc, + gic_device: &Box, + ) -> Result<()> { /* Setting up the distributor attribute. We are placing the GIC below 1GB so we need to substract the size of the distributor. */ diff --git a/arch/src/aarch64/gicv3_its.rs b/arch/src/aarch64/gicv3_its.rs new file mode 100644 index 000000000..dcbdd474a --- /dev/null +++ b/arch/src/aarch64/gicv3_its.rs @@ -0,0 +1,123 @@ +// Copyright 2020 ARM Limited +// SPDX-License-Identifier: Apache-2.0 + +use super::gic::{Error, GICDevice}; +use super::gicv3::GICv3; +use kvm_ioctls::DeviceFd; +use std::sync::Arc; +use std::{boxed::Box, result}; + +type Result = result::Result; + +pub struct GICv3ITS { + /// The file descriptor for the KVM device + fd: DeviceFd, + + /// GIC device properties, to be used for setting up the fdt entry + gic_properties: [u64; 4], + + /// MSI device properties, to be used for setting up the fdt entry + msi_properties: [u64; 2], + + /// Number of CPUs handled by the device + vcpu_count: u64, +} + +impl GICv3ITS { + const KVM_VGIC_V3_ITS_SIZE: u64 = (2 * GICv3::SZ_64K); + + fn get_msi_size() -> u64 { + GICv3ITS::KVM_VGIC_V3_ITS_SIZE + } + + fn get_msi_addr(vcpu_count: u64) -> u64 { + GICv3::get_redists_addr(vcpu_count) - GICv3ITS::get_msi_size() + } +} + +impl GICDevice for GICv3ITS { + fn version() -> u32 { + GICv3::version() + } + + fn device_fd(&self) -> &DeviceFd { + &self.fd + } + + fn device_properties(&self) -> &[u64] { + &self.gic_properties + } + + fn msi_properties(&self) -> &[u64] { + &self.msi_properties + } + + fn vcpu_count(&self) -> u64 { + self.vcpu_count + } + + fn fdt_compatibility(&self) -> &str { + "arm,gic-v3" + } + + fn msi_compatible(&self) -> bool { + true + } + + fn msi_compatiblility(&self) -> &str { + "arm,gic-v3-its" + } + + fn fdt_maint_irq(&self) -> u32 { + GICv3::ARCH_GIC_V3_MAINT_IRQ + } + + fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box { + Box::new(GICv3ITS { + fd: fd, + gic_properties: [ + GICv3::get_dist_addr(), + GICv3::get_dist_size(), + GICv3::get_redists_addr(vcpu_count), + GICv3::get_redists_size(vcpu_count), + ], + msi_properties: [GICv3ITS::get_msi_addr(vcpu_count), GICv3ITS::get_msi_size()], + vcpu_count: vcpu_count, + }) + } + + fn init_device_attributes( + vm: &Arc, + gic_device: &Box, + ) -> Result<()> { + GICv3::init_device_attributes(vm, gic_device)?; + + let mut its_device = kvm_bindings::kvm_create_device { + type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS, + fd: 0, + flags: 0, + }; + + let its_fd = vm + .create_device(&mut its_device) + .map_err(Error::CreateGIC)?; + + Self::set_device_attribute( + &its_fd, + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR, + u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE), + &GICv3ITS::get_msi_addr(u64::from(gic_device.vcpu_count())) as *const u64 as u64, + 0, + )?; + + Self::set_device_attribute( + &its_fd, + kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL, + u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT), + 0, + 0, + )?; + + Ok(()) + } +} diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs index 6509d6871..73f131c0e 100644 --- a/arch/src/aarch64/mod.rs +++ b/arch/src/aarch64/mod.rs @@ -8,6 +8,7 @@ pub mod fdt; pub mod gic; mod gicv2; mod gicv3; +mod gicv3_its; /// Layout for this aarch64 system. pub mod layout; /// Logic for configuring aarch64 registers.