From 5a0a4bc505f00b820c6055b820aaa0ac1d625c5e Mon Sep 17 00:00:00 2001
From: Henry Wang <Henry.Wang@arm.com>
Date: Fri, 6 Aug 2021 19:41:33 -0400
Subject: [PATCH] arch: Add optional `distance-map` node to FDT

The optional device tree node distance-map describes the relative
distance (memory latency) between all NUMA nodes.

Signed-off-by: Henry Wang <Henry.Wang@arm.com>
---
 arch/src/aarch64/fdt.rs | 46 +++++++++++++++++++++++++++++++++++++++++
 arch/src/aarch64/mod.rs |  6 +++---
 vmm/src/vm.rs           |  2 ++
 3 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/arch/src/aarch64/fdt.rs b/arch/src/aarch64/fdt.rs
index 1800e3297..36e392c57 100644
--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -6,6 +6,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.
 
+use crate::NumaNodes;
 use byteorder::{BigEndian, ByteOrder};
 use std::cmp;
 use std::collections::HashMap;
@@ -85,6 +86,7 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     gic_device: &dyn GicDevice,
     initrd: &Option<InitramfsConfig>,
     pci_space_address: &(u64, u64),
+    numa_nodes: &NumaNodes,
 ) -> FdtWriterResult<Vec<u8>> {
     // Allocate stuff necessary for the holding the blob.
     let mut fdt = FdtWriter::new().unwrap();
@@ -112,6 +114,9 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     create_psci_node(&mut fdt)?;
     create_devices_node(&mut fdt, device_info)?;
     create_pci_nodes(&mut fdt, pci_space_address.0, pci_space_address.1)?;
+    if numa_nodes.len() > 1 {
+        create_distance_map_node(&mut fdt, numa_nodes)?;
+    }
 
     // End Header node.
     fdt.end_node(root_node)?;
@@ -561,6 +566,47 @@ fn create_pci_nodes(
     Ok(())
 }
 
+fn create_distance_map_node(fdt: &mut FdtWriter, numa_nodes: &NumaNodes) -> FdtWriterResult<()> {
+    let distance_map_node = fdt.begin_node("distance-map")?;
+    fdt.property_string("compatible", "numa-distance-map-v1")?;
+    // Construct the distance matrix.
+    // 1. We use the word entry to describe a distance from a node to
+    // its destination, e.g. 0 -> 1 = 20 is described as <0 1 20>.
+    // 2. Each entry represents distance from first node to second node.
+    // The distances are equal in either direction.
+    // 3. The distance from a node to self (local distance) is represented
+    // with value 10 and all internode distance should be represented with
+    // a value greater than 10.
+    // 4. distance-matrix should have entries in lexicographical ascending
+    // order of nodes.
+    let mut distance_matrix = Vec::new();
+    for numa_node_idx in 0..numa_nodes.len() {
+        let numa_node = numa_nodes.get(&(numa_node_idx as u32));
+        for dest_numa_node in 0..numa_node.unwrap().distances.len() + 1 {
+            if numa_node_idx == dest_numa_node {
+                distance_matrix.push(numa_node_idx as u32);
+                distance_matrix.push(dest_numa_node as u32);
+                distance_matrix.push(10_u32);
+                continue;
+            }
+
+            distance_matrix.push(numa_node_idx as u32);
+            distance_matrix.push(dest_numa_node as u32);
+            distance_matrix.push(
+                *numa_node
+                    .unwrap()
+                    .distances
+                    .get(&(dest_numa_node as u32))
+                    .unwrap() as u32,
+            );
+        }
+    }
+    fdt.property_array_u32("distance-matrix", distance_matrix.as_ref())?;
+    fdt.end_node(distance_map_node)?;
+
+    Ok(())
+}
+
 // Parse the DTB binary and print for debugging
 pub fn print_fdt(dtb: &[u8]) {
     match fdt_parser::Fdt::new(dtb) {
diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs
index d2f786c61..fe83c6da0 100644
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -14,9 +14,7 @@ pub mod regs;
 pub mod uefi;
 
 pub use self::fdt::DeviceInfoForFdt;
-use crate::DeviceType;
-use crate::GuestMemoryMmap;
-use crate::RegionType;
+use crate::{DeviceType, GuestMemoryMmap, NumaNodes, RegionType};
 use gic::GicDevice;
 use log::{log_enabled, Level};
 use std::collections::HashMap;
@@ -145,6 +143,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
     initrd: &Option<super::InitramfsConfig>,
     pci_space_address: &(u64, u64),
     gic_device: &dyn GicDevice,
+    numa_nodes: &NumaNodes,
 ) -> super::Result<()> {
     let fdt_final = fdt::create_fdt(
         guest_mem,
@@ -155,6 +154,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
         gic_device,
         initrd,
         pci_space_address,
+        numa_nodes,
     )
     .map_err(|_| Error::SetupFdt)?;
 
diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs
index 127750501..4f34e1682 100644
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -1102,6 +1102,7 @@ impl Vm {
             &initramfs_config,
             &pci_space,
             &*gic_device,
+            &self.numa_nodes,
         )
         .map_err(Error::ConfigureSystem)?;
 
@@ -2632,6 +2633,7 @@ mod tests {
             &*gic,
             &None,
             &(0x1_0000_0000, 0x1_0000),
+            &BTreeMap::new(),
         )
         .is_ok())
     }