From f1c47056381bd71f8a408747dae4c73ac3d6ac09 Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Sun, 20 Feb 2022 12:17:50 +0900 Subject: [PATCH] vmm: Add `Debuggable` trait implementation This commit adds initial gdb.rs implementation for `Debuggable` trait to describe a debuggable component. Some part of the trait bound implementations is based on the crosvm GDB stub code [1]. [1] https://github.com/google/crosvm/blob/main/src/gdb.rs Signed-off-by: Akira Moroo --- Cargo.lock | 47 +++++++++ Cargo.toml | 1 + fuzz/Cargo.lock | 47 +++++++++ vmm/Cargo.toml | 3 + vmm/src/cpu.rs | 255 ++++++++++++++++++++++++++++++++++++++++++++++++ vmm/src/gdb.rs | 53 ++++++++++ vmm/src/lib.rs | 2 + vmm/src/vm.rs | 100 +++++++++++++++++++ 8 files changed, 508 insertions(+) create mode 100644 vmm/src/gdb.rs diff --git a/Cargo.lock b/Cargo.lock index bfbe2ebd1..07e2ac2c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,6 +276,30 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b643857cf70949306b81d7e92cb9d47add673868edac9863c4a49c42feaf3f1e" +[[package]] +name = "gdbstub" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686fd5c0d799433870f0ba2ec2bf97697b1369aa8ce71559e2eaafb186a9c5f2" +dependencies = [ + "bitflags", + "cfg-if", + "log", + "managed", + "num-traits", + "paste", +] + +[[package]] +name = "gdbstub_arch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d34d4fe3f10e9741e20f9b576d980a1f3f4918ba4f629ef3dbf932ac1b55db" +dependencies = [ + "gdbstub", + "num-traits", +] + [[package]] name = "getrandom" version = "0.2.5" @@ -475,6 +499,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "memchr" version = "2.4.1" @@ -542,6 +572,15 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + [[package]] name = "openssl-src" version = "111.17.0+1.1.1m" @@ -603,6 +642,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + [[package]] name = "pci" version = "0.1.0" @@ -1328,6 +1373,8 @@ dependencies = [ "devices", "epoll", "event_monitor", + "gdbstub", + "gdbstub_arch", "hypervisor", "lazy_static", "libc", diff --git a/Cargo.toml b/Cargo.toml index 7135784ed..6e91c9d1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ common = ["acpi", "cmos", "fwdebug"] acpi = ["vmm/acpi"] cmos = ["vmm/cmos"] fwdebug = ["vmm/fwdebug"] +gdb = ["vmm/gdb"] kvm = ["vmm/kvm"] mshv = ["vmm/mshv"] tdx = ["vmm/tdx"] diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index e988963b0..7d6f9f0f2 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -244,6 +244,30 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b643857cf70949306b81d7e92cb9d47add673868edac9863c4a49c42feaf3f1e" +[[package]] +name = "gdbstub" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686fd5c0d799433870f0ba2ec2bf97697b1369aa8ce71559e2eaafb186a9c5f2" +dependencies = [ + "bitflags", + "cfg-if", + "log", + "managed", + "num-traits", + "paste", +] + +[[package]] +name = "gdbstub_arch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d34d4fe3f10e9741e20f9b576d980a1f3f4918ba4f629ef3dbf932ac1b55db" +dependencies = [ + "gdbstub", + "num-traits", +] + [[package]] name = "getrandom" version = "0.2.5" @@ -386,6 +410,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "memchr" version = "2.4.1" @@ -428,6 +458,15 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.9.0" @@ -447,6 +486,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + [[package]] name = "pci" version = "0.1.0" @@ -904,6 +949,8 @@ dependencies = [ "devices", "epoll", "event_monitor", + "gdbstub", + "gdbstub_arch", "hypervisor", "lazy_static", "libc", diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index 416fec6bd..8954dcb22 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -9,6 +9,7 @@ default = [] acpi = ["acpi_tables","devices/acpi", "arch/acpi"] cmos = ["devices/cmos"] fwdebug = ["devices/fwdebug"] +gdb = ["kvm"] kvm = ["hypervisor/kvm", "vfio-ioctls/kvm", "vm-device/kvm", "pci/kvm"] mshv = ["hypervisor/mshv", "virtio-devices/mshv", "vfio-ioctls/mshv", "vm-device/mshv", "pci/mshv"] tdx = ["arch/tdx", "hypervisor/tdx"] @@ -24,6 +25,8 @@ clap = "3.1.1" devices = { path = "../devices" } epoll = "4.3.1" event_monitor = { path = "../event_monitor" } +gdbstub = "0.6.0" +gdbstub_arch = "0.2.0" hypervisor = { path = "../hypervisor" } lazy_static = "1.4.0" libc = "0.2.119" diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index be5239ff3..57fc6133a 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -13,6 +13,8 @@ use crate::config::CpusConfig; use crate::device_manager::DeviceManager; +#[cfg(feature = "gdb")] +use crate::gdb::{Debuggable, DebuggableError}; use crate::memory_manager::MemoryManager; use crate::seccomp_filters::{get_seccomp_filter, Thread}; #[cfg(target_arch = "x86_64")] @@ -26,8 +28,12 @@ use arch::EntryPoint; #[cfg(any(target_arch = "aarch64", feature = "acpi"))] use arch::NumaNodes; use devices::interrupt_controller::InterruptController; +#[cfg(all(target_arch = "x86_64", feature = "gdb"))] +use gdbstub_arch::x86::reg::{X86SegmentRegs, X86_64CoreRegs}; #[cfg(target_arch = "aarch64")] use hypervisor::kvm::kvm_bindings; +#[cfg(all(target_arch = "x86_64", feature = "gdb"))] +use hypervisor::x86_64::{SpecialRegisters, StandardRegisters}; #[cfg(target_arch = "x86_64")] use hypervisor::CpuId; use hypervisor::{vm::VmmOps, CpuState, HypervisorCpuError, VmExit}; @@ -111,6 +117,14 @@ pub enum Error { /// Failed scheduling the thread on the expected CPU set. ScheduleCpuSet, + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + /// Error on debug related CPU ops. + CpuDebug(hypervisor::HypervisorCpuError), + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + /// Failed to translate guest virtual address. + TranslateVirtualAddress(hypervisor::HypervisorCpuError), } pub type Result = result::Result; @@ -1357,6 +1371,61 @@ impl CpuManager { pptt.update_checksum(); pptt } + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + fn get_regs(&self, cpu_id: u8) -> Result { + self.vcpus[usize::from(cpu_id)] + .lock() + .unwrap() + .vcpu + .get_regs() + .map_err(Error::CpuDebug) + } + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + fn set_regs(&self, cpu_id: u8, regs: &StandardRegisters) -> Result<()> { + self.vcpus[usize::from(cpu_id)] + .lock() + .unwrap() + .vcpu + .set_regs(regs) + .map_err(Error::CpuDebug) + } + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + fn get_sregs(&self, cpu_id: u8) -> Result { + self.vcpus[usize::from(cpu_id)] + .lock() + .unwrap() + .vcpu + .get_sregs() + .map_err(Error::CpuDebug) + } + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + fn set_sregs(&self, cpu_id: u8, sregs: &SpecialRegisters) -> Result<()> { + self.vcpus[usize::from(cpu_id)] + .lock() + .unwrap() + .vcpu + .set_sregs(sregs) + .map_err(Error::CpuDebug) + } + + #[cfg(all(target_arch = "x86_64", feature = "gdb"))] + fn translate_gva(&self, cpu_id: u8, gva: u64) -> Result { + let (gpa, _) = self.vcpus[usize::from(cpu_id)] + .lock() + .unwrap() + .vcpu + .translate_gva(gva, /* flags: unused */ 0) + .map_err(Error::TranslateVirtualAddress)?; + Ok(gpa) + } + + pub fn vcpus_paused(&self) -> bool { + self.vcpus_pause_signalled.load(Ordering::SeqCst) + } } #[cfg(feature = "acpi")] @@ -1732,6 +1801,192 @@ impl Snapshottable for CpuManager { impl Transportable for CpuManager {} impl Migratable for CpuManager {} +#[cfg(feature = "gdb")] +impl Debuggable for CpuManager { + #[cfg(feature = "kvm")] + fn set_guest_debug( + &self, + cpu_id: usize, + addrs: &[GuestAddress], + singlestep: bool, + ) -> std::result::Result<(), DebuggableError> { + self.vcpus[cpu_id] + .lock() + .unwrap() + .vcpu + .set_guest_debug(addrs, singlestep) + .map_err(DebuggableError::SetDebug) + } + + fn debug_pause(&mut self) -> std::result::Result<(), DebuggableError> { + Ok(()) + } + + fn debug_resume(&mut self) -> std::result::Result<(), DebuggableError> { + Ok(()) + } + + #[cfg(target_arch = "x86_64")] + fn read_regs(&self, cpu_id: usize) -> std::result::Result { + // General registers: RAX, RBX, RCX, RDX, RSI, RDI, RBP, RSP, r8-r15 + let gregs = self + .get_regs(cpu_id as u8) + .map_err(DebuggableError::ReadRegs)?; + let regs = [ + gregs.rax, gregs.rbx, gregs.rcx, gregs.rdx, gregs.rsi, gregs.rdi, gregs.rbp, gregs.rsp, + gregs.r8, gregs.r9, gregs.r10, gregs.r11, gregs.r12, gregs.r13, gregs.r14, gregs.r15, + ]; + + // GDB exposes 32-bit eflags instead of 64-bit rflags. + // https://github.com/bminor/binutils-gdb/blob/master/gdb/features/i386/64bit-core.xml + let eflags = gregs.rflags as u32; + let rip = gregs.rip; + + // Segment registers: CS, SS, DS, ES, FS, GS + let sregs = self + .get_sregs(cpu_id as u8) + .map_err(DebuggableError::ReadRegs)?; + let segments = X86SegmentRegs { + cs: sregs.cs.selector as u32, + ss: sregs.ss.selector as u32, + ds: sregs.ds.selector as u32, + es: sregs.es.selector as u32, + fs: sregs.fs.selector as u32, + gs: sregs.gs.selector as u32, + }; + + // TODO: Add other registers + + Ok(X86_64CoreRegs { + regs, + eflags, + rip, + segments, + ..Default::default() + }) + } + + #[cfg(target_arch = "x86_64")] + fn write_regs( + &self, + cpu_id: usize, + regs: &X86_64CoreRegs, + ) -> std::result::Result<(), DebuggableError> { + let orig_gregs = self + .get_regs(cpu_id as u8) + .map_err(DebuggableError::ReadRegs)?; + let gregs = StandardRegisters { + rax: regs.regs[0], + rbx: regs.regs[1], + rcx: regs.regs[2], + rdx: regs.regs[3], + rsi: regs.regs[4], + rdi: regs.regs[5], + rbp: regs.regs[6], + rsp: regs.regs[7], + r8: regs.regs[8], + r9: regs.regs[9], + r10: regs.regs[10], + r11: regs.regs[11], + r12: regs.regs[12], + r13: regs.regs[13], + r14: regs.regs[14], + r15: regs.regs[15], + rip: regs.rip, + // Update the lower 32-bit of rflags. + rflags: (orig_gregs.rflags & !(u32::MAX as u64)) | (regs.eflags as u64), + }; + + self.set_regs(cpu_id as u8, &gregs) + .map_err(DebuggableError::WriteRegs)?; + + // Segment registers: CS, SS, DS, ES, FS, GS + // Since GDB care only selectors, we call get_sregs() first. + let mut sregs = self + .get_sregs(cpu_id as u8) + .map_err(DebuggableError::ReadRegs)?; + sregs.cs.selector = regs.segments.cs as u16; + sregs.ss.selector = regs.segments.ss as u16; + sregs.ds.selector = regs.segments.ds as u16; + sregs.es.selector = regs.segments.es as u16; + sregs.fs.selector = regs.segments.fs as u16; + sregs.gs.selector = regs.segments.gs as u16; + + self.set_sregs(cpu_id as u8, &sregs) + .map_err(DebuggableError::WriteRegs)?; + + // TODO: Add other registers + + Ok(()) + } + + #[cfg(target_arch = "x86_64")] + fn read_mem( + &self, + cpu_id: usize, + vaddr: GuestAddress, + len: usize, + ) -> std::result::Result, DebuggableError> { + let mut buf = vec![0; len]; + let mut total_read = 0_u64; + + while total_read < len as u64 { + let gaddr = vaddr.0 + total_read; + let paddr = match self.translate_gva(cpu_id as u8, gaddr) { + Ok(paddr) => paddr, + Err(_) if gaddr == u64::MIN => gaddr, // Silently return GVA as GPA if GVA == 0. + Err(e) => return Err(DebuggableError::TranslateGva(e)), + }; + let psize = arch::PAGE_SIZE as u64; + let read_len = std::cmp::min(len as u64 - total_read, psize - (paddr & (psize - 1))); + self.vmmops + .guest_mem_read( + paddr, + &mut buf[total_read as usize..total_read as usize + read_len as usize], + ) + .map_err(DebuggableError::ReadMem)?; + total_read += read_len; + } + Ok(buf) + } + + #[cfg(target_arch = "x86_64")] + fn write_mem( + &self, + cpu_id: usize, + vaddr: &GuestAddress, + data: &[u8], + ) -> std::result::Result<(), DebuggableError> { + let mut total_written = 0_u64; + + while total_written < data.len() as u64 { + let gaddr = vaddr.0 + total_written; + let paddr = match self.translate_gva(cpu_id as u8, gaddr) { + Ok(paddr) => paddr, + Err(_) if gaddr == u64::MIN => gaddr, // Silently return GVA as GPA if GVA == 0. + Err(e) => return Err(DebuggableError::TranslateGva(e)), + }; + let psize = arch::PAGE_SIZE as u64; + let write_len = std::cmp::min( + data.len() as u64 - total_written, + psize - (paddr & (psize - 1)), + ); + self.vmmops + .guest_mem_write( + paddr, + &data[total_written as usize..total_written as usize + write_len as usize], + ) + .map_err(DebuggableError::WriteMem)?; + total_written += write_len; + } + Ok(()) + } + + fn active_vcpus(&self) -> usize { + self.present_vcpus() as usize + } +} + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] #[cfg(test)] mod tests { diff --git a/vmm/src/gdb.rs b/vmm/src/gdb.rs new file mode 100644 index 000000000..21b0a33ed --- /dev/null +++ b/vmm/src/gdb.rs @@ -0,0 +1,53 @@ +// Copyright 2022 Akira Moroo. +// Portions Copyright 2020 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE-BSD-3-Clause file. +// +// SPDX-License-Identifier: BSD-3-Clause + +#[cfg(target_arch = "x86_64")] +use gdbstub_arch::x86::reg::X86_64CoreRegs as CoreRegs; +use vm_memory::GuestAddress; + +#[derive(Debug)] +pub enum DebuggableError { + SetDebug(hypervisor::HypervisorCpuError), + Pause(vm_migration::MigratableError), + Resume(vm_migration::MigratableError), + ReadRegs(crate::cpu::Error), + WriteRegs(crate::cpu::Error), + ReadMem(hypervisor::HypervisorVmError), + WriteMem(hypervisor::HypervisorVmError), + TranslateGva(crate::cpu::Error), + PoisonedState, +} + +pub trait Debuggable: vm_migration::Pausable { + fn set_guest_debug( + &self, + cpu_id: usize, + addrs: &[GuestAddress], + singlestep: bool, + ) -> Result<(), DebuggableError>; + fn debug_pause(&mut self) -> std::result::Result<(), DebuggableError>; + fn debug_resume(&mut self) -> std::result::Result<(), DebuggableError>; + fn read_regs(&self, cpu_id: usize) -> std::result::Result; + fn write_regs( + &self, + cpu_id: usize, + regs: &CoreRegs, + ) -> std::result::Result<(), DebuggableError>; + fn read_mem( + &self, + cpu_id: usize, + vaddr: GuestAddress, + len: usize, + ) -> std::result::Result, DebuggableError>; + fn write_mem( + &self, + cpu_id: usize, + vaddr: &GuestAddress, + data: &[u8], + ) -> std::result::Result<(), DebuggableError>; + fn active_vcpus(&self) -> usize; +} diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index f319c6a8d..e2e1b4594 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -57,6 +57,8 @@ pub mod config; pub mod cpu; pub mod device_manager; pub mod device_tree; +#[cfg(feature = "gdb")] +mod gdb; pub mod interrupt; pub mod memory_manager; pub mod migration; diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 6dc4f7b09..360072ef0 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -20,6 +20,8 @@ use crate::config::{ use crate::cpu; use crate::device_manager::{self, Console, DeviceManager, DeviceManagerError, PtyPair}; use crate::device_tree::DeviceTree; +#[cfg(feature = "gdb")] +use crate::gdb::{Debuggable, DebuggableError}; use crate::memory_manager::{ Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, }; @@ -41,6 +43,8 @@ use arch::PciSpaceInfo; #[cfg(any(target_arch = "aarch64", feature = "acpi"))] use arch::{NumaNode, NumaNodes}; use devices::AcpiNotificationFlags; +#[cfg(all(target_arch = "x86_64", feature = "gdb"))] +use gdbstub_arch::x86::reg::X86_64CoreRegs; use hypervisor::vm::{HypervisorVmError, VmmOps}; use linux_loader::cmdline::Cmdline; #[cfg(target_arch = "x86_64")] @@ -2727,6 +2731,102 @@ impl Migratable for Vm { } } +#[cfg(feature = "gdb")] +impl Debuggable for Vm { + fn set_guest_debug( + &self, + cpu_id: usize, + addrs: &[GuestAddress], + singlestep: bool, + ) -> std::result::Result<(), DebuggableError> { + self.cpu_manager + .lock() + .unwrap() + .set_guest_debug(cpu_id, addrs, singlestep) + } + + fn debug_pause(&mut self) -> std::result::Result<(), DebuggableError> { + if !self.cpu_manager.lock().unwrap().vcpus_paused() { + self.pause().map_err(DebuggableError::Pause)?; + } + let mut state = self + .state + .try_write() + .map_err(|_| DebuggableError::PoisonedState)?; + *state = VmState::BreakPoint; + Ok(()) + } + + fn debug_resume(&mut self) -> std::result::Result<(), DebuggableError> { + if !self.cpu_manager.lock().unwrap().vcpus_paused() { + self.cpu_manager + .lock() + .unwrap() + .start_boot_vcpus() + .map_err(|e| { + DebuggableError::Resume(MigratableError::Resume(anyhow!( + "Could not start boot vCPUs: {:?}", + e + ))) + })?; + } else { + self.resume().map_err(DebuggableError::Resume)?; + } + let mut state = self + .state + .try_write() + .map_err(|_| DebuggableError::PoisonedState)?; + *state = VmState::Running; + Ok(()) + } + + fn read_regs(&self, cpu_id: usize) -> std::result::Result { + self.cpu_manager.lock().unwrap().read_regs(cpu_id) + } + + fn write_regs( + &self, + cpu_id: usize, + regs: &X86_64CoreRegs, + ) -> std::result::Result<(), DebuggableError> { + self.cpu_manager.lock().unwrap().write_regs(cpu_id, regs) + } + + fn read_mem( + &self, + cpu_id: usize, + vaddr: GuestAddress, + len: usize, + ) -> std::result::Result, DebuggableError> { + self.cpu_manager + .lock() + .unwrap() + .read_mem(cpu_id, vaddr, len) + } + + fn write_mem( + &self, + cpu_id: usize, + vaddr: &GuestAddress, + data: &[u8], + ) -> std::result::Result<(), DebuggableError> { + self.cpu_manager + .lock() + .unwrap() + .write_mem(cpu_id, vaddr, data) + } + + fn active_vcpus(&self) -> usize { + let active_vcpus = self.cpu_manager.lock().unwrap().active_vcpus(); + if active_vcpus > 0 { + active_vcpus + } else { + // The VM is not booted yet. Report boot_vcpus() instead. + self.cpu_manager.lock().unwrap().boot_vcpus() as usize + } + } +} + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] #[cfg(test)] mod tests {