vmm: Implement GDB event handler to enable --gdb flag

This commit adds event fds and the event handler to send/receive
requests and responses from the GDB thread. It also adds `--gdb` flag to
enable GDB stub feature.

Signed-off-by: Akira Moroo <retrage01@gmail.com>

src/main.rs

@@ -32,6 +32,9 @@ use vmm_sys_util::signal::block_signal;
 enum Error {
     #[error("Failed to create API EventFd: {0}")]
     CreateApiEventFd(#[source] std::io::Error),
+    #[cfg(feature = "gdb")]
+    #[error("Failed to create Debug EventFd: {0}")]
+    CreateDebugEventFd(#[source] std::io::Error),
     #[cfg_attr(
         feature = "kvm",
         error("Failed to open hypervisor interface (is /dev/kvm available?): {0}")
@@ -65,6 +68,12 @@ enum Error {
     BareEventMonitor,
     #[error("Error doing event monitor I/O: {0}")]
     EventMonitorIo(std::io::Error),
+    #[cfg(feature = "gdb")]
+    #[error("Error parsing --gdb: {0}")]
+    ParsingGdb(option_parser::OptionParserError),
+    #[cfg(feature = "gdb")]
+    #[error("Error parsing --gdb: path required")]
+    BareGdb,
     #[error("Error creating log file: {0}")]
     LogFileCreation(std::io::Error),
     #[error("Error setting up logger: {0}")]
@@ -376,6 +385,15 @@ fn create_app<'a>(
             .group("vm-config"),
     );
 
+    #[cfg(feature = "gdb")]
+    let app = app.arg(
+        Arg::new("gdb")
+            .long("gdb")
+            .help("GDB socket (UNIX domain socket): path=</path/to/a/file>")
+            .takes_value(true)
+            .group("vmm-config"),
+    );
+
     #[cfg(feature = "tdx")]
     let app = app.arg(
         Arg::new("tdx")
@@ -513,6 +531,26 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
     event!("vmm", "starting");
 
     let hypervisor = hypervisor::new().map_err(Error::CreateHypervisor)?;
+
+    #[cfg(feature = "gdb")]
+    let gdb_socket_path = if let Some(gdb_config) = cmd_arguments.value_of("gdb") {
+        let mut parser = OptionParser::new();
+        parser.add("path");
+        parser.parse(gdb_config).map_err(Error::ParsingGdb)?;
+
+        if parser.is_set("path") {
+            Some(std::path::PathBuf::from(parser.get("path").unwrap()))
+        } else {
+            return Err(Error::BareGdb);
+        }
+    } else {
+        None
+    };
+    #[cfg(feature = "gdb")]
+    let debug_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::CreateDebugEventFd)?;
+    #[cfg(feature = "gdb")]
+    let vm_debug_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::CreateDebugEventFd)?;
+
     let vmm_thread = vmm::start_vmm_thread(
         env!("CARGO_PKG_VERSION").to_string(),
         &api_socket_path,
@@ -520,6 +558,12 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
         api_evt.try_clone().unwrap(),
         http_sender,
         api_request_receiver,
+        #[cfg(feature = "gdb")]
+        gdb_socket_path,
+        #[cfg(feature = "gdb")]
+        debug_evt.try_clone().unwrap(),
+        #[cfg(feature = "gdb")]
+        vm_debug_evt.try_clone().unwrap(),
         &seccomp_action,
         hypervisor,
     )
@@ -687,6 +731,8 @@ mod unit_tests {
             watchdog: false,
             #[cfg(feature = "tdx")]
             tdx: None,
+            #[cfg(feature = "gdb")]
+            gdb: false,
             platform: None,
         };
 

vmm/src/config.rs

@@ -323,6 +323,8 @@ pub struct VmParams<'a> {
     pub watchdog: bool,
     #[cfg(feature = "tdx")]
     pub tdx: Option<&'a str>,
+    #[cfg(feature = "gdb")]
+    pub gdb: bool,
     pub platform: Option<&'a str>,
 }
 
@@ -355,6 +357,8 @@ impl<'a> VmParams<'a> {
         let platform = args.value_of("platform");
         #[cfg(feature = "tdx")]
         let tdx = args.value_of("tdx");
+        #[cfg(feature = "gdb")]
+        let gdb = args.is_present("gdb");
         VmParams {
             cpus,
             memory,
@@ -379,6 +383,8 @@ impl<'a> VmParams<'a> {
             watchdog,
             #[cfg(feature = "tdx")]
             tdx,
+            #[cfg(feature = "gdb")]
+            gdb,
             platform,
         }
     }
@@ -2109,6 +2115,8 @@ pub struct VmConfig {
     pub watchdog: bool,
     #[cfg(feature = "tdx")]
     pub tdx: Option<TdxConfig>,
+    #[cfg(feature = "gdb")]
+    pub gdb: bool,
     pub platform: Option<PlatformConfig>,
 }
 
@@ -2423,6 +2431,9 @@ impl VmConfig {
         #[cfg(feature = "tdx")]
         let tdx = vm_params.tdx.map(TdxConfig::parse).transpose()?;
 
+        #[cfg(feature = "gdb")]
+        let gdb = vm_params.gdb;
+
         let config = VmConfig {
             cpus: CpusConfig::parse(vm_params.cpus)?,
             memory: MemoryConfig::parse(vm_params.memory, vm_params.memory_zones)?,
@@ -2447,6 +2458,8 @@ impl VmConfig {
             watchdog: vm_params.watchdog,
             #[cfg(feature = "tdx")]
             tdx,
+            #[cfg(feature = "gdb")]
+            gdb,
             platform,
         };
         config.validate().map_err(Error::Validation)?;
@@ -3063,6 +3076,8 @@ mod tests {
             watchdog: false,
             #[cfg(feature = "tdx")]
             tdx: None,
+            #[cfg(feature = "gdb")]
+            gdb: false,
             platform: None,
         };
 

vmm/src/cpu.rs

@@ -14,7 +14,7 @@
 use crate::config::CpusConfig;
 use crate::device_manager::DeviceManager;
 #[cfg(feature = "gdb")]
-use crate::gdb::{Debuggable, DebuggableError};
+use crate::gdb::{get_raw_tid, Debuggable, DebuggableError};
 use crate::memory_manager::MemoryManager;
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 #[cfg(target_arch = "x86_64")]
@@ -405,6 +405,8 @@ pub struct CpuManager {
     exit_evt: EventFd,
     #[cfg_attr(target_arch = "aarch64", allow(dead_code))]
     reset_evt: EventFd,
+    #[cfg(feature = "gdb")]
+    vm_debug_evt: EventFd,
     vcpu_states: Vec<VcpuState>,
     selected_cpu: u8,
     vcpus: Vec<Arc<Mutex<Vcpu>>>,
@@ -557,6 +559,7 @@ impl CpuManager {
         vm: Arc<dyn hypervisor::Vm>,
         exit_evt: EventFd,
         reset_evt: EventFd,
+        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
         seccomp_action: SeccompAction,
         vmmops: Arc<dyn VmmOps>,
@@ -635,6 +638,8 @@ impl CpuManager {
             vcpu_states,
             exit_evt,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            vm_debug_evt,
             selected_cpu: 0,
             vcpus: Vec::with_capacity(usize::from(config.max_vcpus)),
             seccomp_action,
@@ -773,6 +778,8 @@ impl CpuManager {
     ) -> Result<()> {
         let reset_evt = self.reset_evt.try_clone().unwrap();
         let exit_evt = self.exit_evt.try_clone().unwrap();
+        #[cfg(feature = "gdb")]
+        let vm_debug_evt = self.vm_debug_evt.try_clone().unwrap();
         let panic_exit_evt = self.exit_evt.try_clone().unwrap();
         let vcpu_kill_signalled = self.vcpus_kill_signalled.clone();
         let vcpu_pause_signalled = self.vcpus_pause_signalled.clone();
@@ -904,6 +911,16 @@ impl CpuManager {
                             // vcpu.run() returns false on a triple-fault so trigger a reset
                             match vcpu.run() {
                                 Ok(run) => match run {
+                                    #[cfg(all(target_arch = "x86_64", feature = "kvm"))]
+                                    VmExit::Debug => {
+                                        info!("VmExit::Debug");
+                                        #[cfg(feature = "gdb")]
+                                        {
+                                            vcpu_pause_signalled.store(true, Ordering::SeqCst);
+                                            let raw_tid = get_raw_tid(vcpu_id as usize);
+                                            vm_debug_evt.write(raw_tid as u64).unwrap();
+                                        }
+                                    }
                                     #[cfg(target_arch = "x86_64")]
                                     VmExit::IoapicEoi(vector) => {
                                         if let Some(interrupt_controller) =

vmm/src/gdb.rs

@@ -488,7 +488,7 @@ impl run_blocking::BlockingEventLoop for GdbEventLoop {
     }
 }
 
-pub fn gdb_thread(mut gdbstub: GdbStub, path: &str) {
+pub fn gdb_thread(mut gdbstub: GdbStub, path: &std::path::Path) {
     let listener = match UnixListener::bind(path) {
         Ok(s) => s,
         Err(e) => {
@@ -496,7 +496,7 @@ pub fn gdb_thread(mut gdbstub: GdbStub, path: &str) {
             return;
         }
     };
-    info!("Waiting for a GDB connection on {}...", path);
+    info!("Waiting for a GDB connection on {}...", path.display());
 
     let (stream, addr) = match listener.accept() {
         Ok(v) => v,

vmm/src/lib.rs

@@ -147,6 +147,20 @@ pub enum Error {
     /// Error binding API server socket
     #[error("Error creation API server's socket {0:?}")]
     CreateApiServerSocket(#[source] io::Error),
+
+    #[cfg(feature = "gdb")]
+    #[error("Failed to start the GDB thread: {0}")]
+    GdbThreadSpawn(io::Error),
+
+    /// GDB request receive error
+    #[cfg(feature = "gdb")]
+    #[error("Error receiving GDB request: {0}")]
+    GdbRequestRecv(#[source] RecvError),
+
+    /// GDB response send error
+    #[cfg(feature = "gdb")]
+    #[error("Error sending GDB request: {0}")]
+    GdbResponseSend(#[source] SendError<gdb::GdbResponse>),
 }
 pub type Result<T> = result::Result<T, Error>;
 
@@ -157,6 +171,7 @@ pub enum EpollDispatch {
     Reset = 1,
     Api = 2,
     ActivateVirtioDevices = 3,
+    Debug = 4,
     Unknown,
 }
 
@@ -168,6 +183,7 @@ impl From<u64> for EpollDispatch {
             1 => Reset,
             2 => Api,
             3 => ActivateVirtioDevices,
+            4 => Debug,
             _ => Unknown,
         }
     }
@@ -229,6 +245,7 @@ impl Serialize for PciDeviceInfo {
     }
 }
 
+#[allow(unused_variables)]
 #[allow(clippy::too_many_arguments)]
 pub fn start_vmm_thread(
     vmm_version: String,
@@ -237,9 +254,19 @@ pub fn start_vmm_thread(
     api_event: EventFd,
     api_sender: Sender<ApiRequest>,
     api_receiver: Receiver<ApiRequest>,
+    #[cfg(feature = "gdb")] debug_path: Option<PathBuf>,
+    #[cfg(feature = "gdb")] debug_event: EventFd,
+    #[cfg(feature = "gdb")] vm_debug_event: EventFd,
     seccomp_action: &SeccompAction,
     hypervisor: Arc<dyn hypervisor::Hypervisor>,
 ) -> Result<thread::JoinHandle<Result<()>>> {
+    #[cfg(feature = "gdb")]
+    let (gdb_sender, gdb_receiver) = std::sync::mpsc::channel();
+    #[cfg(feature = "gdb")]
+    let gdb_debug_event = debug_event.try_clone().map_err(Error::EventFdClone)?;
+    #[cfg(feature = "gdb")]
+    let gdb_vm_debug_event = vm_debug_event.try_clone().map_err(Error::EventFdClone)?;
+
     let http_api_event = api_event.try_clone().map_err(Error::EventFdClone)?;
 
     // Retrieve seccomp filter
@@ -261,12 +288,20 @@ pub fn start_vmm_thread(
                 let mut vmm = Vmm::new(
                     vmm_version.to_string(),
                     api_event,
+                    #[cfg(feature = "gdb")]
+                    debug_event,
+                    #[cfg(feature = "gdb")]
+                    vm_debug_event,
                     vmm_seccomp_action,
                     hypervisor,
                     exit_evt,
                 )?;
 
-                vmm.control_loop(Arc::new(api_receiver))
+                vmm.control_loop(
+                    Arc::new(api_receiver),
+                    #[cfg(feature = "gdb")]
+                    Arc::new(gdb_receiver),
+                )
             })
             .map_err(Error::VmmThreadSpawn)?
     };
@@ -289,6 +324,16 @@ pub fn start_vmm_thread(
             exit_evt,
         )?;
     }
+
+    #[cfg(feature = "gdb")]
+    if let Some(debug_path) = debug_path {
+        let target = gdb::GdbStub::new(gdb_sender, gdb_debug_event, gdb_vm_debug_event);
+        thread::Builder::new()
+            .name("gdb".to_owned())
+            .spawn(move || gdb::gdb_thread(target, &debug_path))
+            .map_err(Error::GdbThreadSpawn)?;
+    }
+
     Ok(thread)
 }
 
@@ -305,6 +350,10 @@ pub struct Vmm {
     exit_evt: EventFd,
     reset_evt: EventFd,
     api_evt: EventFd,
+    #[cfg(feature = "gdb")]
+    debug_evt: EventFd,
+    #[cfg(feature = "gdb")]
+    vm_debug_evt: EventFd,
     version: String,
     vm: Option<Vm>,
     vm_config: Option<Arc<Mutex<VmConfig>>>,
@@ -317,6 +366,8 @@ impl Vmm {
     fn new(
         vmm_version: String,
         api_evt: EventFd,
+        #[cfg(feature = "gdb")] debug_evt: EventFd,
+        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
         seccomp_action: SeccompAction,
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
         exit_evt: EventFd,
@@ -341,11 +392,20 @@ impl Vmm {
             .add_event(&api_evt, EpollDispatch::Api)
             .map_err(Error::Epoll)?;
 
+        #[cfg(feature = "gdb")]
+        epoll
+            .add_event(&debug_evt, EpollDispatch::Debug)
+            .map_err(Error::Epoll)?;
+
         Ok(Vmm {
             epoll,
             exit_evt,
             reset_evt,
             api_evt,
+            #[cfg(feature = "gdb")]
+            debug_evt,
+            #[cfg(feature = "gdb")]
+            vm_debug_evt,
             version: vmm_version,
             vm: None,
             vm_config: None,
@@ -376,6 +436,11 @@ impl Vmm {
         if self.vm.is_none() {
             let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?;
             let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?;
+            #[cfg(feature = "gdb")]
+            let vm_debug_evt = self
+                .vm_debug_evt
+                .try_clone()
+                .map_err(VmError::EventFdClone)?;
             let activate_evt = self
                 .activate_evt
                 .try_clone()
@@ -386,6 +451,8 @@ impl Vmm {
                     Arc::clone(vm_config),
                     exit_evt,
                     reset_evt,
+                    #[cfg(feature = "gdb")]
+                    vm_debug_evt,
                     &self.seccomp_action,
                     self.hypervisor.clone(),
                     activate_evt,
@@ -462,6 +529,11 @@ impl Vmm {
 
         let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?;
         let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?;
+        #[cfg(feature = "gdb")]
+        let debug_evt = self
+            .vm_debug_evt
+            .try_clone()
+            .map_err(VmError::EventFdClone)?;
         let activate_evt = self
             .activate_evt
             .try_clone()
@@ -472,6 +544,8 @@ impl Vmm {
             vm_config,
             exit_evt,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            debug_evt,
             Some(source_url),
             restore_cfg.prefault,
             &self.seccomp_action,
@@ -520,6 +594,11 @@ impl Vmm {
 
             let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?;
             let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?;
+            #[cfg(feature = "gdb")]
+            let debug_evt = self
+                .vm_debug_evt
+                .try_clone()
+                .map_err(VmError::EventFdClone)?;
             let activate_evt = self
                 .activate_evt
                 .try_clone()
@@ -535,6 +614,8 @@ impl Vmm {
                 config,
                 exit_evt,
                 reset_evt,
+                #[cfg(feature = "gdb")]
+                debug_evt,
                 &self.seccomp_action,
                 self.hypervisor.clone(),
                 activate_evt,
@@ -930,6 +1011,10 @@ impl Vmm {
         let reset_evt = self.reset_evt.try_clone().map_err(|e| {
             MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e))
         })?;
+        #[cfg(feature = "gdb")]
+        let debug_evt = self.vm_debug_evt.try_clone().map_err(|e| {
+            MigratableError::MigrateReceive(anyhow!("Error cloning debug EventFd: {}", e))
+        })?;
         let activate_evt = self.activate_evt.try_clone().map_err(|e| {
             MigratableError::MigrateReceive(anyhow!("Error cloning activate EventFd: {}", e))
         })?;
@@ -939,6 +1024,8 @@ impl Vmm {
             self.vm_config.clone().unwrap(),
             exit_evt,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            debug_evt,
             &self.seccomp_action,
             self.hypervisor.clone(),
             activate_evt,
@@ -1421,7 +1508,11 @@ impl Vmm {
         })
     }
 
-    fn control_loop(&mut self, api_receiver: Arc<Receiver<ApiRequest>>) -> Result<()> {
+    fn control_loop(
+        &mut self,
+        api_receiver: Arc<Receiver<ApiRequest>>,
+        #[cfg(feature = "gdb")] gdb_receiver: Arc<Receiver<gdb::GdbRequest>>,
+    ) -> Result<()> {
         const EPOLL_EVENTS_LEN: usize = 100;
 
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
@@ -1689,6 +1780,28 @@ impl Vmm {
                             }
                         }
                     }
+                    #[cfg(feature = "gdb")]
+                    EpollDispatch::Debug => {
+                        // Consume the event.
+                        self.debug_evt.read().map_err(Error::EventFdRead)?;
+
+                        // Read from the API receiver channel
+                        let gdb_request = gdb_receiver.recv().map_err(Error::GdbRequestRecv)?;
+
+                        let response = if let Some(ref mut vm) = self.vm {
+                            vm.debug_request(&gdb_request.payload, gdb_request.cpu_id)
+                        } else {
+                            Err(VmError::VmNotRunning)
+                        }
+                        .map_err(gdb::Error::Vm);
+
+                        gdb_request
+                            .sender
+                            .send(response)
+                            .map_err(Error::GdbResponseSend)?;
+                    }
+                    #[cfg(not(feature = "gdb"))]
+                    EpollDispatch::Debug => {}
                 }
             }
         }
@@ -1713,6 +1826,10 @@ mod unit_tests {
         Vmm::new(
             "dummy".to_string(),
             EventFd::new(EFD_NONBLOCK).unwrap(),
+            #[cfg(feature = "gdb")]
+            EventFd::new(EFD_NONBLOCK).unwrap(),
+            #[cfg(feature = "gdb")]
+            EventFd::new(EFD_NONBLOCK).unwrap(),
             SeccompAction::Allow,
             hypervisor::new().unwrap(),
             EventFd::new(EFD_NONBLOCK).unwrap(),
@@ -1778,6 +1895,8 @@ mod unit_tests {
             watchdog: false,
             #[cfg(feature = "tdx")]
             tdx: None,
+            #[cfg(feature = "gdb")]
+            gdb: false,
             platform: None,
         }))
     }

vmm/src/vm.rs

@@ -552,6 +552,7 @@ impl Vm {
         vm: Arc<dyn hypervisor::Vm>,
         exit_evt: EventFd,
         reset_evt: EventFd,
+        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
         seccomp_action: &SeccompAction,
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
         activate_evt: EventFd,
@@ -575,6 +576,9 @@ impl Vm {
         #[cfg(not(feature = "tdx"))]
         let force_iommu = false;
 
+        #[cfg(feature = "gdb")]
+        let stop_on_boot = config.lock().unwrap().gdb;
+        #[cfg(not(feature = "gdb"))]
         let stop_on_boot = false;
 
         let device_manager = DeviceManager::new(
@@ -624,6 +628,8 @@ impl Vm {
             vm.clone(),
             exit_evt_clone,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            vm_debug_evt,
             hypervisor.clone(),
             seccomp_action.clone(),
             vm_ops,
@@ -763,6 +769,7 @@ impl Vm {
         config: Arc<Mutex<VmConfig>>,
         exit_evt: EventFd,
         reset_evt: EventFd,
+        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
         seccomp_action: &SeccompAction,
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
         activate_evt: EventFd,
@@ -817,6 +824,8 @@ impl Vm {
             vm,
             exit_evt,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            vm_debug_evt,
             seccomp_action,
             hypervisor,
             activate_evt,
@@ -840,6 +849,7 @@ impl Vm {
         vm_config: Arc<Mutex<VmConfig>>,
         exit_evt: EventFd,
         reset_evt: EventFd,
+        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
         source_url: Option<&str>,
         prefault: bool,
         seccomp_action: &SeccompAction,
@@ -888,6 +898,8 @@ impl Vm {
             vm,
             exit_evt,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            vm_debug_evt,
             seccomp_action,
             hypervisor,
             activate_evt,
@@ -900,6 +912,7 @@ impl Vm {
         config: Arc<Mutex<VmConfig>>,
         exit_evt: EventFd,
         reset_evt: EventFd,
+        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
         seccomp_action: &SeccompAction,
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
         activate_evt: EventFd,
@@ -939,6 +952,8 @@ impl Vm {
             vm,
             exit_evt,
             reset_evt,
+            #[cfg(feature = "gdb")]
+            vm_debug_evt,
             seccomp_action,
             hypervisor,
             activate_evt,