vmm: add GuestDebuggable trait

It's useful to dump the guest, which named coredump so that crash
tool can be used to analysize it when guest hung up.

Let's add GuestDebuggable trait and Coredumpxxx error to support
coredump firstly.

Signed-off-by: Yi Wang <wang.yi59@zte.com.cn>
Co-authored-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

vmm/Cargo.toml

@@ -10,6 +10,7 @@ amx = []
 cmos = ["devices/cmos"]
 fwdebug = ["devices/fwdebug"]
 gdb = ["kvm"]
+guest_debug = ["kvm"]
 kvm = ["hypervisor/kvm", "vfio-ioctls/kvm", "vm-device/kvm", "pci/kvm"]
 mshv = ["hypervisor/mshv", "virtio-devices/mshv", "vfio-ioctls/mshv", "vm-device/mshv", "pci/mshv"]
 tdx = ["arch/tdx", "hypervisor/tdx"]

vmm/src/coredump.rs

@@ -0,0 +1,43 @@
+// Copyright © 2022 ZTE Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs::File;
+
+#[derive(Clone)]
+pub struct CoredumpMemoryRegion {
+    pub mem_offset_in_elf: u64,
+    pub mem_size: u64,
+}
+
+#[derive(Clone)]
+pub struct CoredumpMemoryRegions {
+    pub ram_maps: std::collections::BTreeMap<u64, CoredumpMemoryRegion>,
+}
+
+/// Platform information
+#[derive(Default)]
+pub struct DumpState {
+    pub elf_note_size: isize,
+    pub elf_phdr_num: u16,
+    pub elf_sh_info: u32,
+    pub mem_offset: u64,
+    pub mem_info: Option<CoredumpMemoryRegions>,
+    pub file: Option<File>,
+}
+
+#[derive(Debug)]
+pub enum GuestDebuggableError {
+    Coredump(anyhow::Error),
+    CoredumpFile(std::io::Error),
+}
+
+pub trait GuestDebuggable: vm_migration::Pausable {
+    fn coredump(
+        &mut self,
+        _destination_url: &str,
+    ) -> std::result::Result<(), GuestDebuggableError> {
+        Ok(())
+    }
+}

vmm/src/lib.rs

@@ -52,6 +52,8 @@ mod acpi;
 pub mod api;
 mod clone3;
 pub mod config;
+#[cfg(feature = "guest_debug")]
+mod coredump;
 pub mod cpu;
 pub mod device_manager;
 pub mod device_tree;

vmm/src/memory_manager.rs

@@ -5,6 +5,10 @@
 #[cfg(target_arch = "x86_64")]
 use crate::config::SgxEpcConfig;
 use crate::config::{HotplugMethod, MemoryConfig, MemoryZoneConfig};
+#[cfg(feature = "guest_debug")]
+use crate::coredump::{CoredumpMemoryRegion, CoredumpMemoryRegions};
+#[cfg(feature = "guest_debug")]
+use crate::coredump::{DumpState, GuestDebuggable, GuestDebuggableError};
 use crate::migration::url_to_path;
 use crate::MEMORY_MANAGER_SNAPSHOT_ID;
 use crate::{GuestMemoryMmap, GuestRegionMmap};
@@ -18,6 +22,8 @@ use devices::ioapic;
 #[cfg(target_arch = "x86_64")]
 use libc::{MAP_NORESERVE, MAP_POPULATE, MAP_SHARED, PROT_READ, PROT_WRITE};
 use serde::{Deserialize, Serialize};
+#[cfg(feature = "guest_debug")]
+use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::ffi;
@@ -1846,6 +1852,31 @@ impl MemoryManager {
     pub fn acpi_address(&self) -> Option<GuestAddress> {
         self.acpi_address
     }
+
+    pub fn num_guest_ram_mappings(&self) -> u32 {
+        self.guest_ram_mappings.len() as u32
+    }
+
+    #[cfg(feature = "guest_debug")]
+    pub fn coredump_memory_regions(&self, mem_offset: u64) -> CoredumpMemoryRegions {
+        let mut mapping_sorted_by_gpa = self.guest_ram_mappings.clone();
+        mapping_sorted_by_gpa.sort_by_key(|m| m.gpa);
+
+        let mut mem_offset_in_elf = mem_offset;
+        let mut ram_maps = BTreeMap::new();
+        for mapping in mapping_sorted_by_gpa.iter() {
+            ram_maps.insert(
+                mapping.gpa,
+                CoredumpMemoryRegion {
+                    mem_offset_in_elf,
+                    mem_size: mapping.size,
+                },
+            );
+            mem_offset_in_elf += mapping.size;
+        }
+
+        CoredumpMemoryRegions { ram_maps }
+    }
 }
 
 struct MemoryNotify {

vmm/src/migration.rs

@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+#[cfg(feature = "guest_debug")]
+use crate::coredump::GuestDebuggableError;
 use crate::{
     config::VmConfig,
     vm::{VmSnapshot, VM_SNAPSHOT_ID},
@@ -32,6 +34,18 @@ pub fn url_to_path(url: &str) -> std::result::Result<PathBuf, MigratableError> {
     Ok(path)
 }
 
+#[cfg(feature = "guest_debug")]
+pub fn url_to_file(url: &str) -> std::result::Result<PathBuf, GuestDebuggableError> {
+    let file: PathBuf = url
+        .strip_prefix("file://")
+        .ok_or_else(|| {
+            GuestDebuggableError::Coredump(anyhow!("Could not extract file from URL: {}", url))
+        })
+        .map(|s| s.into())?;
+
+    Ok(file)
+}
+
 pub fn recv_vm_config(source_url: &str) -> std::result::Result<VmConfig, MigratableError> {
     let mut vm_config_path = url_to_path(source_url)?;
 

vmm/src/vm.rs

@@ -16,6 +16,10 @@ use crate::config::{
     add_to_config, DeviceConfig, DiskConfig, FsConfig, HotplugMethod, NetConfig, PmemConfig,
     UserDeviceConfig, ValidationError, VdpaConfig, VmConfig, VsockConfig,
 };
+#[cfg(feature = "guest_debug")]
+use crate::coredump::DumpState;
+#[cfg(feature = "guest_debug")]
+use crate::coredump::{GuestDebuggable, GuestDebuggableError};
 use crate::cpu;
 use crate::device_manager::{Console, DeviceManager, DeviceManagerError, PtyPair};
 use crate::device_tree::DeviceTree;
@@ -24,6 +28,8 @@ use crate::gdb::{Debuggable, DebuggableError, GdbRequestPayload, GdbResponsePayl
 use crate::memory_manager::{
     Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData,
 };
+#[cfg(feature = "guest_debug")]
+use crate::migration::url_to_file;
 use crate::migration::{get_vm_snapshot, url_to_path, SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE};
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::GuestMemoryMmap;
@@ -51,6 +57,8 @@ use devices::AcpiNotificationFlags;
 use gdbstub_arch::x86::reg::X86_64CoreRegs;
 use hypervisor::{HypervisorVmError, VmOps};
 use linux_loader::cmdline::Cmdline;
+#[cfg(feature = "guest_debug")]
+use linux_loader::elf;
 #[cfg(target_arch = "x86_64")]
 use linux_loader::loader::elf::PvhBootCapability::PvhEntryPresent;
 #[cfg(target_arch = "aarch64")]
@@ -72,6 +80,8 @@ use std::io::{self, Read, Write};
 use std::io::{Seek, SeekFrom};
 #[cfg(feature = "tdx")]
 use std::mem;
+#[cfg(feature = "guest_debug")]
+use std::mem::size_of;
 use std::num::Wrapping;
 use std::ops::Deref;
 use std::os::unix::net::UnixStream;
@@ -294,6 +304,10 @@ pub enum Error {
     #[cfg(target_arch = "x86_64")]
     #[error("Error joining kernel loading thread")]
     KernelLoadThreadJoin(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
+
+    #[cfg(feature = "guest_debug")]
+    #[error("Error coredumping VM: {0:?}")]
+    Coredump(GuestDebuggableError),
 }
 pub type Result<T> = result::Result<T, Error>;
 
@@ -2527,6 +2541,54 @@ impl Vm {
         }
         Ok(GdbResponsePayload::CommandComplete)
     }
+
+    #[cfg(feature = "guest_debug")]
+    fn get_dump_state(
+        &mut self,
+        destination_url: &str,
+    ) -> std::result::Result<DumpState, GuestDebuggableError> {
+        let nr_cpus = self.config.lock().unwrap().cpus.boot_vcpus as u32;
+        let elf_note_size = self.get_note_size(NoteDescType::ElfAndVmmDesc, nr_cpus) as isize;
+        let mut elf_phdr_num = 1 as u16;
+        let elf_sh_info = 0;
+        let coredump_file_path = url_to_file(destination_url)?;
+        let mapping_num = self.memory_manager.lock().unwrap().num_guest_ram_mappings();
+
+        if mapping_num < UINT16_MAX - 2 {
+            elf_phdr_num += mapping_num as u16;
+        } else {
+            panic!("mapping num beyond 65535 not supported");
+        }
+        let coredump_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create_new(true)
+            .open(coredump_file_path)
+            .map_err(|e| GuestDebuggableError::Coredump(e.into()))?;
+
+        let mem_offset = self.coredump_get_mem_offset(elf_phdr_num, elf_note_size);
+        let mem_data = self
+            .memory_manager
+            .lock()
+            .unwrap()
+            .coredump_memory_regions(mem_offset);
+
+        Ok(DumpState {
+            elf_note_size,
+            elf_phdr_num,
+            elf_sh_info,
+            mem_offset,
+            mem_info: Some(mem_data),
+            file: Some(coredump_file),
+        })
+    }
+
+    #[cfg(feature = "guest_debug")]
+    fn coredump_get_mem_offset(&self, phdr_num: u16, note_size: isize) -> u64 {
+        size_of::<elf::Elf64_Ehdr>() as u64
+            + note_size as u64
+            + size_of::<elf::Elf64_Phdr>() as u64 * phdr_num as u64
+    }
 }
 
 impl Pausable for Vm {
@@ -2955,6 +3017,36 @@ impl Debuggable for Vm {
     }
 }
 
+#[cfg(feature = "guest_debug")]
+pub const UINT16_MAX: u32 = 65535;
+
+#[cfg(feature = "guest_debug")]
+impl GuestDebuggable for Vm {
+    fn coredump(&mut self, destination_url: &str) -> std::result::Result<(), GuestDebuggableError> {
+        event!("vm", "coredumping");
+
+        #[cfg(feature = "tdx")]
+        {
+            if self.config.lock().unwrap().tdx.is_some() {
+                return Err(GuestDebuggableError::Coredump(anyhow!(
+                    "Coredump not possible with TDX VM"
+                )));
+            }
+        }
+
+        let current_state = self.get_state().unwrap();
+        if current_state != VmState::Paused {
+            return Err(GuestDebuggableError::Coredump(anyhow!(
+                "Trying to coredump while VM is running"
+            )));
+        }
+
+        let coredump_state = self.get_dump_state(destination_url)?;
+
+        Ok(())
+    }
+}
+
 #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
 #[cfg(test)]
 mod tests {