arch: aarch64: Add PCIe node in FDT for AArch64

Signed-off-by: Michael Zhao <michael.zhao@arm.com>

arch/src/aarch64/fdt.rs

@@ -17,7 +17,10 @@ use super::super::DeviceType;
 use super::super::InitramfsConfig;
 use super::get_fdt_addr;
 use super::gic::GICDevice;
-use super::layout::FDT_MAX_SIZE;
+use super::layout::{
+    FDT_MAX_SIZE, MEM_32BIT_DEVICES_SIZE, MEM_32BIT_DEVICES_START, PCI_MMCONFIG_SIZE,
+    PCI_MMCONFIG_START,
+};
 use crate::aarch64::fdt::Error::CstringFDTTransform;
 use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
 
@@ -94,6 +97,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
     device_info: &HashMap<(DeviceType, String), T>,
     gic_device: &Box<dyn GICDevice>,
     initrd: &Option<InitramfsConfig>,
+    pci_space_address: &Option<(u64, u64)>,
 ) -> Result<Vec<u8>> {
     // Alocate stuff necessary for the holding the blob.
     let mut fdt = vec![0; FDT_MAX_SIZE];
@@ -122,6 +126,9 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
     create_clock_node(&mut fdt)?;
     create_psci_node(&mut fdt)?;
     create_devices_node(&mut fdt, device_info)?;
+    if let Some((pci_device_base, pci_device_size)) = pci_space_address {
+        create_pci_nodes(&mut fdt, *pci_device_base, *pci_device_size)?;
+    }
 
     // End Header node.
     append_end_node(&mut fdt)?;
@@ -540,6 +547,49 @@ fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug>(
     Ok(())
 }
 
+fn create_pci_nodes(fdt: &mut Vec<u8>, pci_device_base: u64, pci_device_size: u64) -> Result<()> {
+    // Add node for PCIe controller.
+    // See Documentation/devicetree/bindings/pci/host-generic-pci.txt in the kernel
+    // and https://elinux.org/Device_Tree_Usage.
+    let ranges = generate_prop32(&[
+        // mmio addresses
+        0x2000000,                                // (ss = 10: 32-bit memory space)
+        (MEM_32BIT_DEVICES_START.0 >> 32) as u32, // PCI address
+        MEM_32BIT_DEVICES_START.0 as u32,
+        (MEM_32BIT_DEVICES_START.0 >> 32) as u32, // CPU address
+        MEM_32BIT_DEVICES_START.0 as u32,
+        (MEM_32BIT_DEVICES_SIZE >> 32) as u32, // size
+        MEM_32BIT_DEVICES_SIZE as u32,
+        // device addresses
+        0x3000000,                      // (ss = 11: 64-bit memory space)
+        (pci_device_base >> 32) as u32, // PCI address
+        pci_device_base as u32,
+        (pci_device_base >> 32) as u32, // CPU address
+        pci_device_base as u32,
+        (pci_device_size >> 32) as u32, // size
+        pci_device_size as u32,
+    ]);
+    let bus_range = generate_prop32(&[0, 0]); // Only bus 0
+    let reg = generate_prop64(&[PCI_MMCONFIG_START.0, PCI_MMCONFIG_SIZE]);
+
+    append_begin_node(fdt, "pci")?;
+    append_property_string(fdt, "compatible", "pci-host-ecam-generic")?;
+    append_property_string(fdt, "device_type", "pci")?;
+    append_property(fdt, "ranges", &ranges)?;
+    append_property(fdt, "bus-range", &bus_range)?;
+    append_property_u32(fdt, "#address-cells", 3)?;
+    append_property_u32(fdt, "#size-cells", 2)?;
+    append_property(fdt, "reg", &reg)?;
+    append_property_u32(fdt, "#interrupt-cells", 1)?;
+    append_property_null(fdt, "interrupt-map")?;
+    append_property_null(fdt, "interrupt-map-mask")?;
+    append_property_null(fdt, "dma-coherent")?;
+    append_property_u32(fdt, "msi-parent", MSI_PHANDLE)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -603,7 +653,7 @@ mod tests {
         let kvm = hypervisor::kvm::KvmHypervisor::new().unwrap();
         let hv: Arc<dyn hypervisor::Hypervisor> = Arc::new(kvm);
         let vm = hv.create_vm().unwrap();
-        let gic = create_gic(&vm, 1).unwrap();
+        let gic = create_gic(&vm, 1, false).unwrap();
         assert!(create_fdt(
             &mem,
             &CString::new("console=tty0").unwrap(),
@@ -611,6 +661,7 @@ mod tests {
             &dev_info,
             &gic,
             &None,
+            &None,
         )
         .is_ok())
     }

arch/src/aarch64/gic.rs

@@ -1,13 +1,12 @@
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-use kvm_ioctls::DeviceFd;
-use std::sync::Arc;
-use std::{boxed::Box, result};
-
 use super::gicv2::GICv2;
 use super::gicv3::GICv3;
 use super::gicv3_its::GICv3ITS;
+use kvm_ioctls::DeviceFd;
+use std::sync::Arc;
+use std::{boxed::Box, result};
 
 /// Errors thrown while setting up the GIC.
 #[derive(Debug)]
@@ -159,10 +158,18 @@ pub trait GICDevice: Send + Sync {
 ///
 /// It will try to create by default a GICv3 device. If that fails it will try
 /// to fall-back to a GICv2 device.
-pub fn create_gic(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GICDevice>> {
+pub fn create_gic(
+    vm: &Arc<dyn hypervisor::Vm>,
+    vcpu_count: u64,
+    its_required: bool,
+) -> Result<Box<dyn GICDevice>> {
+    if its_required {
+        GICv3ITS::new(vm, vcpu_count)
+    } else {
         GICv3ITS::new(vm, vcpu_count)
             .or_else(|_| GICv3::new(vm, vcpu_count).or_else(|_| GICv2::new(vm, vcpu_count)))
     }
+}
 
 #[cfg(test)]
 mod tests {
@@ -174,6 +181,6 @@ mod tests {
         let hv: Arc<dyn hypervisor::Hypervisor> = Arc::new(kvm);
         let vm = hv.create_vm().unwrap();
 
-        assert!(create_gic(&vm, 1).is_ok());
+        assert!(create_gic(&vm, 1, false).is_ok());
     }
 }

arch/src/aarch64/mod.rs

@@ -135,8 +135,6 @@ pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, Region
 ///
 /// * `guest_mem` - The memory to be used by the guest.
 /// * `num_cpus` - Number of virtual CPUs the guest will have.
-#[allow(clippy::too_many_arguments)]
-#[allow(unused_variables)]
 pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug>(
     vm: &Arc<dyn hypervisor::Vm>,
     guest_mem: &GuestMemoryMmap,
@@ -145,16 +143,22 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug>(
     vcpu_mpidr: Vec<u64>,
     device_info: &HashMap<(DeviceType, String), T>,
     initrd: &Option<super::InitramfsConfig>,
+    pci_space_address: &Option<(u64, u64)>,
 ) -> super::Result<()> {
-    let gic_device = gic::create_gic(vm, vcpu_count).map_err(Error::SetupGIC)?;
+    // If pci_space_address is present, it means PCI devices are used ("pci" feature enabled).
+    // Then GITv3-ITS is required for MSI messaging.
+    // Otherwise ("mmio" feature enabled), any version of GIC is OK.
+    let gic_device =
+        gic::create_gic(vm, vcpu_count, pci_space_address.is_some()).map_err(Error::SetupGIC)?;
 
-    let dtb = fdt::create_fdt(
+    fdt::create_fdt(
         guest_mem,
         cmdline_cstring,
         vcpu_mpidr,
         device_info,
         &gic_device,
         initrd,
+        pci_space_address,
     )
     .map_err(Error::SetupFDT)?;
 

vmm/src/vm.rs

@@ -60,9 +60,9 @@ use std::path::PathBuf;
 use std::sync::{Arc, Mutex, RwLock};
 use std::{result, str, thread};
 use url::Url;
+use vm_memory::{Address, GuestAddress, GuestAddressSpace};
 #[cfg(target_arch = "x86_64")]
-use vm_memory::{Address, Bytes, GuestMemoryMmap};
+use vm_memory::{Bytes, GuestMemoryMmap};
-use vm_memory::{GuestAddress, GuestAddressSpace};
 use vm_migration::{
     Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
     Transportable,
@@ -618,6 +618,31 @@ impl Vm {
             .get_device_info()
             .clone();
 
+        let pci_space: Option<(u64, u64)> = if cfg!(feature = "pci_support") {
+            let pci_space_start: GuestAddress = self
+                .memory_manager
+                .lock()
+                .as_ref()
+                .unwrap()
+                .start_of_device_area();
+
+            let pci_space_end: GuestAddress = self
+                .memory_manager
+                .lock()
+                .as_ref()
+                .unwrap()
+                .end_of_device_area();
+
+            let pci_space_size = pci_space_end
+                .checked_offset_from(pci_space_start)
+                .ok_or(Error::MemOverflow)?
+                + 1;
+
+            Some((pci_space_start.0, pci_space_size))
+        } else {
+            None
+        };
+
         arch::configure_system(
             &self.memory_manager.lock().as_ref().unwrap().vm,
             &mem,
@@ -626,6 +651,7 @@ impl Vm {
             vcpu_mpidrs,
             device_info,
             &None,
+            &pci_space,
         )
         .map_err(Error::ConfigureSystem)?;