vmm, virtio-devices: mem: Don't use MADV_DONTNEED on hugepages

This commit introduces a new information to the VirtioMemZone structure
in order to know if the memory zone is backed by hugepages.

Based on this new information, the virtio-mem device is now able to
determine if madvise(MADV_DONTNEED) should be performed or not. The
madvise documentation specifies that MADV_DONTNEED advice will fail if
the memory range has been allocated with some hugepages.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
Signed-off-by: Hui Zhu <teawater@antfin.com>

virtio-devices/src/mem.rs

@@ -415,10 +415,12 @@ struct MemEpollHandler {
     queue_evt: EventFd,
     kill_evt: EventFd,
     pause_evt: EventFd,
+    hugepages: bool,
 }
 
 impl MemEpollHandler {
     fn discard_memory_range(&self, offset: u64, size: u64) -> Result<(), Error> {
+        // Use fallocate if the memory region is backed by a file.
         if let Some(fd) = self.host_fd {
             let res = unsafe {
                 libc::fallocate64(
@@ -434,17 +436,22 @@ impl MemEpollHandler {
                 return Err(Error::DiscardMemoryRange(err));
             }
         }
-        let res = unsafe {
-            libc::madvise(
-                (self.host_addr + offset) as *mut libc::c_void,
-                size as libc::size_t,
-                libc::MADV_DONTNEED,
-            )
-        };
-        if res != 0 {
-            let err = io::Error::last_os_error();
-            error!("Advising kernel about pages range failed: {}", err);
-            return Err(Error::DiscardMemoryRange(err));
+
+        // Only use madvise if the memory region is not allocated with
+        // hugepages.
+        if !self.hugepages {
+            let res = unsafe {
+                libc::madvise(
+                    (self.host_addr + offset) as *mut libc::c_void,
+                    size as libc::size_t,
+                    libc::MADV_DONTNEED,
+                )
+            };
+            if res != 0 {
+                let err = io::Error::last_os_error();
+                error!("Advising kernel about pages range failed: {}", err);
+                return Err(Error::DiscardMemoryRange(err));
+            }
         }
 
         Ok(())
@@ -667,6 +674,7 @@ pub struct Mem {
     host_fd: Option<RawFd>,
     config: Arc<Mutex<VirtioMemConfig>>,
     seccomp_action: SeccompAction,
+    hugepages: bool,
 }
 
 impl Mem {
@@ -678,6 +686,7 @@ impl Mem {
         seccomp_action: SeccompAction,
         numa_node_id: Option<u16>,
         initial_size: u64,
+        hugepages: bool,
     ) -> io::Result<Mem> {
         let region_len = region.len();
 
@@ -748,6 +757,7 @@ impl Mem {
             host_fd,
             config: Arc::new(Mutex::new(config)),
             seccomp_action,
+            hugepages,
         })
     }
 }
@@ -827,6 +837,7 @@ impl VirtioDevice for Mem {
             queue_evt: queue_evts.remove(0),
             kill_evt,
             pause_evt,
+            hugepages: self.hugepages,
         };
 
         handler

vmm/src/device_manager.rs

@@ -2347,6 +2347,7 @@ impl DeviceManager {
                         self.seccomp_action.clone(),
                         node_id,
                         virtio_mem_zone.hotplugged_size(),
+                        virtio_mem_zone.hugepages(),
                     )
                     .map_err(DeviceManagerError::CreateVirtioMem)?,
                 ));

vmm/src/memory_manager.rs

@@ -72,6 +72,7 @@ pub struct VirtioMemZone {
     region: Arc<GuestRegionMmap>,
     resize_handler: virtio_devices::Resize,
     hotplugged_size: u64,
+    hugepages: bool,
 }
 
 impl VirtioMemZone {
@@ -84,6 +85,9 @@ impl VirtioMemZone {
     pub fn hotplugged_size(&self) -> u64 {
         self.hotplugged_size
     }
+    pub fn hugepages(&self) -> bool {
+        self.hugepages
+    }
 }
 
 #[derive(Default)]
@@ -684,6 +688,7 @@ impl MemoryManager {
                             resize_handler: virtio_devices::Resize::new()
                                 .map_err(Error::EventFdFail)?,
                             hotplugged_size: zone.hotplugged_size.unwrap_or(0),
+                            hugepages: zone.hugepages,
                         });
 
                         start_of_device_area = start_addr