From dcf6d9d7311f474e868343724a905ddb104ba9a8 Mon Sep 17 00:00:00 2001
From: Henry Wang <Henry.Wang@arm.com>
Date: Tue, 1 Sep 2020 16:07:15 +0800
Subject: [PATCH] device_manager: AArch64: add a field to set/get GIC device
 entity

In AArch64 systems, the state of GIC device can only be
retrieved from `KVM_GET_DEVICE_ATTR` ioctl. Therefore to implement
saving/restoring the GIC states, we need to make sure that the
GIC object (either the file descriptor or the device itself) can
be extracted after the VM is started.

This commit refactors the code of GIC creation by adding a new
field `gic_device_entity` in device manager and methods to set/get
this field. The GIC object can be therefore saved in the device
manager after calling `arch::configure_system`.

Signed-off-by: Henry Wang <Henry.Wang@arm.com>
---
 arch/src/aarch64/gic/mod.rs |  2 +-
 arch/src/aarch64/mod.rs     |  5 +++--
 vmm/src/device_manager.rs   | 17 +++++++++++++++++
 vmm/src/vm.rs               | 10 +++++++++-
 4 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/arch/src/aarch64/gic/mod.rs b/arch/src/aarch64/gic/mod.rs
index 63263a190..c39b79864 100644
--- a/arch/src/aarch64/gic/mod.rs
+++ b/arch/src/aarch64/gic/mod.rs
@@ -26,7 +26,7 @@ pub enum Error {
 }
 type Result<T> = result::Result<T, Error>;
 
-pub trait GICDevice {
+pub trait GICDevice: Send {
     /// Returns the hypervisor agnostic Device of the GIC device
     fn device(&self) -> &Arc<dyn hypervisor::Device>;
 
diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs
index 30559d349..7a7019ac0 100644
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -14,6 +14,7 @@ pub mod regs;
 pub use self::fdt::DeviceInfoForFDT;
 use crate::DeviceType;
 use crate::RegionType;
+use aarch64::gic::GICDevice;
 use hypervisor::kvm::kvm_bindings;
 use std::collections::HashMap;
 use std::ffi::CStr;
@@ -146,7 +147,7 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::Bui
     device_info: &HashMap<(DeviceType, String), T, S>,
     initrd: &Option<super::InitramfsConfig>,
     pci_space_address: &Option<(u64, u64)>,
-) -> super::Result<()> {
+) -> super::Result<Box<dyn GICDevice>> {
     // If pci_space_address is present, it means PCI devices are used ("pci" feature enabled).
     // Then GITv3-ITS is required for MSI messaging.
     // Otherwise ("mmio" feature enabled), any version of GIC is OK.
@@ -164,7 +165,7 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::Bui
     )
     .map_err(Error::SetupFDT)?;
 
-    Ok(())
+    Ok(gic_device)
 }
 
 /// Returns the memory address where the initramfs could be loaded.
diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs
index 66200ffda..4efa23a3b 100644
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -27,6 +27,8 @@ use crate::{device_node, DEVICE_MANAGER_SNAPSHOT_ID};
 use acpi_tables::{aml, aml::Aml};
 use anyhow::anyhow;
 #[cfg(target_arch = "aarch64")]
+use arch::aarch64::gic::GICDevice;
+#[cfg(target_arch = "aarch64")]
 use arch::aarch64::DeviceInfoForFDT;
 #[cfg(feature = "acpi")]
 use arch::layout;
@@ -727,6 +729,9 @@ pub struct DeviceManager {
     #[cfg(target_arch = "aarch64")]
     interrupt_controller: Option<Arc<Mutex<gic::Gic>>>,
 
+    #[cfg(target_arch = "aarch64")]
+    gic_device_entity: Option<Arc<Mutex<Box<dyn GICDevice>>>>,
+
     // Things to be added to the commandline (i.e. for virtio-mmio)
     cmdline_additions: Vec<String>,
 
@@ -853,6 +858,8 @@ impl DeviceManager {
             address_manager: Arc::clone(&address_manager),
             console: Arc::new(Console::default()),
             interrupt_controller: None,
+            #[cfg(target_arch = "aarch64")]
+            gic_device_entity: None,
             cmdline_additions: Vec::new(),
             #[cfg(feature = "acpi")]
             ged_notification_device: None,
@@ -1155,6 +1162,16 @@ impl DeviceManager {
         Ok(interrupt_controller)
     }
 
+    #[cfg(target_arch = "aarch64")]
+    pub fn set_gic_device_entity(&mut self, device_entity: Arc<Mutex<Box<dyn GICDevice>>>) {
+        self.gic_device_entity = Some(device_entity);
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    pub fn get_gic_device_entity(&self) -> Option<&Arc<Mutex<Box<dyn GICDevice>>>> {
+        self.gic_device_entity.as_ref()
+    }
+
     #[cfg(target_arch = "aarch64")]
     pub fn enable_interrupt_controller(&self) -> DeviceManagerResult<()> {
         if let Some(interrupt_controller) = &self.interrupt_controller {
diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs
index 6d83067db..31ec772d5 100644
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -801,7 +801,9 @@ impl Vm {
             None
         };
 
-        arch::configure_system(
+        // Call `configure_system` and pass the GIC devices out, so that
+        // we can register the GIC device to the device manager.
+        let gic_device = arch::configure_system(
             &self.memory_manager.lock().as_ref().unwrap().vm,
             &mem,
             &cmdline_cstring,
@@ -813,6 +815,12 @@ impl Vm {
         )
         .map_err(Error::ConfigureSystem)?;
 
+        // Update the GIC entity in device manager
+        self.device_manager
+            .lock()
+            .unwrap()
+            .set_gic_device_entity(Arc::new(Mutex::new(gic_device)));
+
         self.device_manager
             .lock()
             .unwrap()