From 22a2a99e5f4b72221ec1732b35adf0df1b08b1b2 Mon Sep 17 00:00:00 2001
From: Jiangbo Wu <jiangbo.wu@intel.com>
Date: Mon, 21 Sep 2020 17:59:15 +0800
Subject: [PATCH] acpi: Add hotplug numa node

virtio-mem device would use 'VIRTIO_MEM_F_ACPI_PXM' to add memory to NUMA
node, which MUST be existed, otherwise it will be assigned to node id 0,
even if user specify different node id.

According ACPI spec about Memory Affinity Structure, system hardware
supports hot-add memory region using 'Hot Pluggable | Enabled' flags.

Signed-off-by: Jiangbo Wu <jiangbo.wu@intel.com>
---
 Cargo.lock      |  1 +
 vmm/Cargo.toml  |  1 +
 vmm/src/acpi.rs | 74 +++++++++++++++++++++++++++++++++----------------
 vmm/src/vm.rs   |  8 ++++++
 4 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 56d68f4eb..d5cc5706f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1611,6 +1611,7 @@ dependencies = [
  "anyhow",
  "arc-swap",
  "arch",
+ "bitflags 1.2.1",
  "block_util",
  "clap",
  "credibility",
diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml
index 38c1fabb5..a323bc90e 100644
--- a/vmm/Cargo.toml
+++ b/vmm/Cargo.toml
@@ -16,6 +16,7 @@ io_uring = ["virtio-devices/io_uring"]
 
 [dependencies]
 arc-swap = ">=0.4.4"
+bitflags = ">=1.2.1"
 clap = "2.33.3"
 acpi_tables = { path = "../acpi_tables", optional = true }
 anyhow = "1.0"
diff --git a/vmm/src/acpi.rs b/vmm/src/acpi.rs
index 2143e6f24..1366b02c1 100644
--- a/vmm/src/acpi.rs
+++ b/vmm/src/acpi.rs
@@ -12,7 +12,9 @@ use acpi_tables::{
     sdt::{GenericAddress, SDT},
 };
 use arch::layout;
+use bitflags::bitflags;
 use std::sync::{Arc, Mutex};
+use vm_memory::GuestRegionMmap;
 use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryMmap, GuestMemoryRegion};
 
 #[repr(packed)]
@@ -54,6 +56,42 @@ struct ProcessorLocalX2ApicAffinity {
     _reserved2: u32,
 }
 
+bitflags! {
+    pub struct MemAffinityFlags: u32 {
+        const NOFLAGS = 0;
+        const ENABLE = 0b1;
+        const HOTPLUGGABLE = 0b10;
+        const NON_VOLATILE = 0b100;
+    }
+}
+
+impl MemoryAffinity {
+    fn from_region(
+        region: &Arc<GuestRegionMmap>,
+        proximity_domain: u32,
+        flags: MemAffinityFlags,
+    ) -> Self {
+        let base_addr = region.start_addr().raw_value();
+        let base_addr_lo = (base_addr & 0xffff_ffff) as u32;
+        let base_addr_hi = (base_addr >> 32) as u32;
+        let length = region.len() as u64;
+        let length_lo = (length & 0xffff_ffff) as u32;
+        let length_hi = (length >> 32) as u32;
+
+        MemoryAffinity {
+            type_: 1,
+            length: 40,
+            proximity_domain,
+            base_addr_lo,
+            base_addr_hi,
+            length_lo,
+            length_hi,
+            flags: flags.bits(),
+            ..Default::default()
+        }
+    }
+}
+
 pub fn create_dsdt_table(
     device_manager: &Arc<Mutex<DeviceManager>>,
     cpu_manager: &Arc<Mutex<CpuManager>>,
@@ -170,31 +208,19 @@ pub fn create_acpi_tables(
             let proximity_domain = *node_id as u32;
 
             for region in node.memory_regions() {
-                let base_addr = region.start_addr().raw_value();
-                let base_addr_lo = (base_addr & 0xffff_ffff) as u32;
-                let base_addr_hi = (base_addr >> 32) as u32;
-                let length = region.len() as u64;
-                let length_lo = (length & 0xffff_ffff) as u32;
-                let length_hi = (length >> 32) as u32;
-
-                // Flags
-                // - Enabled = 1 (bit 0)
-                // - Hot Pluggable = 0 (bit 1)
-                // - NonVolatile = 0 (bit 2)
-                // - Reserved bits 3-31
-                let flags = 1;
-
-                srat.append(MemoryAffinity {
-                    type_: 1,
-                    length: 40,
+                srat.append(MemoryAffinity::from_region(
+                    region,
                     proximity_domain,
-                    base_addr_lo,
-                    base_addr_hi,
-                    length_lo,
-                    length_hi,
-                    flags,
-                    ..Default::default()
-                });
+                    MemAffinityFlags::ENABLE,
+                ))
+            }
+
+            for region in node.hotplug_regions() {
+                srat.append(MemoryAffinity::from_region(
+                    region,
+                    proximity_domain,
+                    MemAffinityFlags::ENABLE | MemAffinityFlags::HOTPLUGGABLE,
+                ))
             }
 
             for cpu in node.cpus() {
diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs
index a4f5be344..6d83067db 100644
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -218,6 +218,7 @@ pub type Result<T> = result::Result<T, Error>;
 #[derive(Clone, Default)]
 pub struct NumaNode {
     memory_regions: Vec<Arc<GuestRegionMmap>>,
+    hotplug_regions: Vec<Arc<GuestRegionMmap>>,
     cpus: Vec<u8>,
     distances: BTreeMap<u32, u8>,
     memory_zones: Vec<String>,
@@ -228,6 +229,10 @@ impl NumaNode {
         &self.memory_regions
     }
 
+    pub fn hotplug_regions(&self) -> &Vec<Arc<GuestRegionMmap>> {
+        &self.hotplug_regions
+    }
+
     pub fn cpus(&self) -> &Vec<u8> {
         &self.cpus
     }
@@ -411,6 +416,9 @@ impl Vm {
                     for memory_zone in memory_zones.iter() {
                         if let Some(mm_zone) = mm_zones.get(memory_zone) {
                             node.memory_regions.extend(mm_zone.regions().clone());
+                            if let Some(virtiomem_zone) = mm_zone.virtio_mem_zone() {
+                                node.hotplug_regions.push(virtiomem_zone.region().clone());
+                            }
                             node.memory_zones.push(memory_zone.clone());
                         } else {
                             error!("Unknown memory zone '{}'", memory_zone);