acpi: Add hotplug numa node

virtio-mem device would use 'VIRTIO_MEM_F_ACPI_PXM' to add memory to NUMA
node, which MUST be existed, otherwise it will be assigned to node id 0,
even if user specify different node id.

According ACPI spec about Memory Affinity Structure, system hardware
supports hot-add memory region using 'Hot Pluggable | Enabled' flags.

Signed-off-by: Jiangbo Wu <jiangbo.wu@intel.com>

Cargo.lock

@@ -1611,6 +1611,7 @@ dependencies = [
  "anyhow",
  "arc-swap",
  "arch",
+ "bitflags 1.2.1",
  "block_util",
  "clap",
  "credibility",

vmm/Cargo.toml

@@ -16,6 +16,7 @@ io_uring = ["virtio-devices/io_uring"]
 
 [dependencies]
 arc-swap = ">=0.4.4"
+bitflags = ">=1.2.1"
 clap = "2.33.3"
 acpi_tables = { path = "../acpi_tables", optional = true }
 anyhow = "1.0"

vmm/src/acpi.rs

@@ -12,7 +12,9 @@ use acpi_tables::{
     sdt::{GenericAddress, SDT},
 };
 use arch::layout;
+use bitflags::bitflags;
 use std::sync::{Arc, Mutex};
+use vm_memory::GuestRegionMmap;
 use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryMmap, GuestMemoryRegion};
 
 #[repr(packed)]
@@ -54,6 +56,42 @@ struct ProcessorLocalX2ApicAffinity {
     _reserved2: u32,
 }
 
+bitflags! {
+    pub struct MemAffinityFlags: u32 {
+        const NOFLAGS = 0;
+        const ENABLE = 0b1;
+        const HOTPLUGGABLE = 0b10;
+        const NON_VOLATILE = 0b100;
+    }
+}
+
+impl MemoryAffinity {
+    fn from_region(
+        region: &Arc<GuestRegionMmap>,
+        proximity_domain: u32,
+        flags: MemAffinityFlags,
+    ) -> Self {
+        let base_addr = region.start_addr().raw_value();
+        let base_addr_lo = (base_addr & 0xffff_ffff) as u32;
+        let base_addr_hi = (base_addr >> 32) as u32;
+        let length = region.len() as u64;
+        let length_lo = (length & 0xffff_ffff) as u32;
+        let length_hi = (length >> 32) as u32;
+
+        MemoryAffinity {
+            type_: 1,
+            length: 40,
+            proximity_domain,
+            base_addr_lo,
+            base_addr_hi,
+            length_lo,
+            length_hi,
+            flags: flags.bits(),
+            ..Default::default()
+        }
+    }
+}
+
 pub fn create_dsdt_table(
     device_manager: &Arc<Mutex<DeviceManager>>,
     cpu_manager: &Arc<Mutex<CpuManager>>,
@@ -170,31 +208,19 @@ pub fn create_acpi_tables(
             let proximity_domain = *node_id as u32;
 
             for region in node.memory_regions() {
-                let base_addr = region.start_addr().raw_value();
-                let base_addr_lo = (base_addr & 0xffff_ffff) as u32;
-                let base_addr_hi = (base_addr >> 32) as u32;
-                let length = region.len() as u64;
-                let length_lo = (length & 0xffff_ffff) as u32;
-                let length_hi = (length >> 32) as u32;
-
-                // Flags
-                // - Enabled = 1 (bit 0)
-                // - Hot Pluggable = 0 (bit 1)
-                // - NonVolatile = 0 (bit 2)
-                // - Reserved bits 3-31
-                let flags = 1;
-
-                srat.append(MemoryAffinity {
-                    type_: 1,
-                    length: 40,
+                srat.append(MemoryAffinity::from_region(
+                    region,
                     proximity_domain,
-                    base_addr_lo,
-                    base_addr_hi,
-                    length_lo,
-                    length_hi,
-                    flags,
-                    ..Default::default()
-                });
+                    MemAffinityFlags::ENABLE,
+                ))
+            }
+
+            for region in node.hotplug_regions() {
+                srat.append(MemoryAffinity::from_region(
+                    region,
+                    proximity_domain,
+                    MemAffinityFlags::ENABLE | MemAffinityFlags::HOTPLUGGABLE,
+                ))
             }
 
             for cpu in node.cpus() {

vmm/src/vm.rs

@@ -218,6 +218,7 @@ pub type Result<T> = result::Result<T, Error>;
 #[derive(Clone, Default)]
 pub struct NumaNode {
     memory_regions: Vec<Arc<GuestRegionMmap>>,
+    hotplug_regions: Vec<Arc<GuestRegionMmap>>,
     cpus: Vec<u8>,
     distances: BTreeMap<u32, u8>,
     memory_zones: Vec<String>,
@@ -228,6 +229,10 @@ impl NumaNode {
         &self.memory_regions
     }
 
+    pub fn hotplug_regions(&self) -> &Vec<Arc<GuestRegionMmap>> {
+        &self.hotplug_regions
+    }
+
     pub fn cpus(&self) -> &Vec<u8> {
         &self.cpus
     }
@@ -411,6 +416,9 @@ impl Vm {
                     for memory_zone in memory_zones.iter() {
                         if let Some(mm_zone) = mm_zones.get(memory_zone) {
                             node.memory_regions.extend(mm_zone.regions().clone());
+                            if let Some(virtiomem_zone) = mm_zone.virtio_mem_zone() {
+                                node.hotplug_regions.push(virtiomem_zone.region().clone());
+                            }
                             node.memory_zones.push(memory_zone.clone());
                         } else {
                             error!("Unknown memory zone '{}'", memory_zone);