From 46c60183cd79e611318b229116b371530529d803 Mon Sep 17 00:00:00 2001
From: Henry Wang <Henry.Wang@arm.com>
Date: Tue, 31 Aug 2021 22:31:15 -0400
Subject: [PATCH] arch, vmm: Implement GIC Pausable trait

This commit implements the GIC (including both GICv3 and GICv3ITS)
Pausable trait. The pause of device manager will trigger a "pause"
of GIC, where we flush GIC pending tables and ITS tables to the
guest RAM.

Signed-off-by: Henry Wang <Henry.Wang@arm.com>
---
 arch/src/aarch64/gic/gicv3.rs     | 14 ++++++++++----
 arch/src/aarch64/gic/gicv3_its.rs | 25 +++++++++++++++++--------
 vmm/src/device_manager.rs         | 23 +++++++++++++++++++++++
 3 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/arch/src/aarch64/gic/gicv3.rs b/arch/src/aarch64/gic/gicv3.rs
index 1620956dc..b94175af7 100644
--- a/arch/src/aarch64/gic/gicv3.rs
+++ b/arch/src/aarch64/gic/gicv3.rs
@@ -109,9 +109,6 @@ pub mod kvm {
 
         /// Save the state of GIC.
         fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3State> {
-            // Flush redistributors pending tables to guest RAM.
-            save_pending_tables(self.device()).map_err(Error::SavePendingTables)?;
-
             let gicd_ctlr =
                 read_ctlr(self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
 
@@ -255,7 +252,16 @@ pub mod kvm {
         }
     }
 
-    impl Pausable for KvmGicV3 {}
+    impl Pausable for KvmGicV3 {
+        fn pause(&mut self) -> std::result::Result<(), MigratableError> {
+            // Flush redistributors pending tables to guest RAM.
+            save_pending_tables(self.device()).map_err(|e| {
+                MigratableError::Pause(anyhow!("Could not save GICv3 GIC pending tables {:?}", e))
+            })?;
+
+            Ok(())
+        }
+    }
     impl Transportable for KvmGicV3 {}
     impl Migratable for KvmGicV3 {}
 }
diff --git a/arch/src/aarch64/gic/gicv3_its.rs b/arch/src/aarch64/gic/gicv3_its.rs
index b0ac01449..5f3d4db2b 100644
--- a/arch/src/aarch64/gic/gicv3_its.rs
+++ b/arch/src/aarch64/gic/gicv3_its.rs
@@ -191,9 +191,6 @@ pub mod kvm {
 
         /// Save the state of GICv3ITS.
         fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3ItsState> {
-            // Flush redistributors pending tables to guest RAM.
-            save_pending_tables(self.device()).map_err(Error::SavePendingTables)?;
-
             let gicd_ctlr =
                 read_ctlr(self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
 
@@ -206,10 +203,6 @@ pub mod kvm {
             let icc_state =
                 get_icc_regs(self.device(), gicr_typers).map_err(Error::SaveIccRegisters)?;
 
-            // Save GICv3ITS registers
-            gicv3_its_tables_access(self.its_device().unwrap(), true)
-                .map_err(Error::SaveITSTables)?;
-
             let its_baser_state: [u64; 8] = [0; 8];
             for i in 0..8 {
                 gicv3_its_attr_access(
@@ -501,7 +494,23 @@ pub mod kvm {
         }
     }
 
-    impl Pausable for KvmGicV3Its {}
+    impl Pausable for KvmGicV3Its {
+        fn pause(&mut self) -> std::result::Result<(), MigratableError> {
+            // Flush redistributors pending tables to guest RAM.
+            save_pending_tables(self.device()).map_err(|e| {
+                MigratableError::Pause(anyhow!(
+                    "Could not save GICv3ITS GIC pending tables {:?}",
+                    e
+                ))
+            })?;
+            // Flush ITS tables to guest RAM.
+            gicv3_its_tables_access(self.its_device().unwrap(), true).map_err(|e| {
+                MigratableError::Pause(anyhow!("Could not save GICv3ITS ITS tables {:?}", e))
+            })?;
+
+            Ok(())
+        }
+    }
     impl Transportable for KvmGicV3Its {}
     impl Migratable for KvmGicV3Its {}
 }
diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs
index 570e2ecc2..93999e1be 100644
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -29,6 +29,8 @@ use crate::{device_node, DEVICE_MANAGER_SNAPSHOT_ID};
 #[cfg(feature = "acpi")]
 use acpi_tables::{aml, aml::Aml};
 use anyhow::anyhow;
+#[cfg(target_arch = "aarch64")]
+use arch::aarch64::gic::gicv3_its::kvm::KvmGicV3Its;
 #[cfg(feature = "acpi")]
 use arch::layout;
 #[cfg(target_arch = "x86_64")]
@@ -4132,6 +4134,27 @@ impl Pausable for DeviceManager {
                 migratable.lock().unwrap().pause()?;
             }
         }
+        // On AArch64, the pause of device manager needs to trigger
+        // a "pause" of GIC, which will flush the GIC pending tables
+        // and ITS tables to guest RAM.
+        #[cfg(target_arch = "aarch64")]
+        {
+            let gic_device = Arc::clone(
+                self.get_interrupt_controller()
+                    .unwrap()
+                    .lock()
+                    .unwrap()
+                    .get_gic_device()
+                    .unwrap(),
+            );
+            gic_device
+                .lock()
+                .unwrap()
+                .as_any_concrete_mut()
+                .downcast_mut::<KvmGicV3Its>()
+                .unwrap()
+                .pause()?;
+        }
 
         Ok(())
     }