From 0921cfb8f8af8f9f2c2a2662b216b56c2140b334 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Thu, 28 Feb 2019 15:26:30 +0100 Subject: [PATCH] vmm: Basic Vcpu implementation Signed-off-by: Samuel Ortiz --- Cargo.lock | 11 +++ vmm/Cargo.toml | 2 + vmm/src/lib.rs | 4 +- vmm/src/vm.rs | 201 ++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 213 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 530fdeacf..d22576ea3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,8 +149,18 @@ dependencies = [ "arch 0.1.0", "kvm-bindings 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvm-ioctls 0.1.0 (git+https://github.com/rust-vmm/kvm-ioctls)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", "linux-loader 0.1.0 (git+https://github.com/sameo/linux-loader)", "vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)", + "vmm-sys-util 0.1.0 (git+https://github.com/sameo/vmm-sys-util)", +] + +[[package]] +name = "vmm-sys-util" +version = "0.1.0" +source = "git+https://github.com/sameo/vmm-sys-util#766db444eb9ac315ce7139dff9f56e6e9fd2471f" +dependencies = [ + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -190,6 +200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)" = "" +"checksum vmm-sys-util 0.1.0 (git+https://github.com/sameo/vmm-sys-util)" = "" "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index cebeaa156..c05bd1c0f 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -8,7 +8,9 @@ edition = "2018" arch = { path = "../arch" } kvm-bindings = "0.1" kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls" } +libc = ">=0.2.39" linux-loader = { git = "https://github.com/sameo/linux-loader" } +vmm-sys-util = { git = "https://github.com/sameo/vmm-sys-util" } [dependencies.vm-memory] git = "https://github.com/rust-vmm/vm-memory" diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index b56ea5ac2..1431d2930 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -26,8 +26,8 @@ pub fn boot_kernel(kernel: &Path) -> Result<()> { let vmm = Vmm::new()?; let mut vm = Vm::new(&vmm.kvm, kernel)?; - vm.load_kernel()?; - vm.start()?; + let entry = vm.load_kernel()?; + vm.start(entry)?; Ok(()) } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index ddc2f5c22..ba7ddfb49 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -5,22 +5,28 @@ extern crate arch; extern crate kvm_ioctls; +extern crate libc; extern crate linux_loader; extern crate vm_memory; +extern crate vmm_sys_util; use kvm_bindings::kvm_userspace_memory_region; use kvm_ioctls::*; +use libc::{c_void, siginfo_t}; use linux_loader::cmdline; use linux_loader::loader::KernelLoader; use std::ffi::CString; use std::fs::File; use std::path::Path; +use std::sync::{Arc, Barrier}; use std::{io, result, str, thread}; use vm_memory::{ Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestUsize, MmapError, }; +use vmm_sys_util::signal::register_signal_handler; +const VCPU_RTSIG_OFFSET: i32 = 0; const DEFAULT_CMDLINE: &str = "console=ttyS0,115200n8 init=/init tsc=reliable no_timer_check cryptomgr.notests"; const CMDLINE_OFFSET: GuestAddress = GuestAddress(0x20000); @@ -48,9 +54,92 @@ pub enum Error { /// Cannot load the command line in memory CmdLine, + + /// Cannot open the VCPU file descriptor. + VcpuFd(io::Error), + + /// Cannot run the VCPUs. + VcpuRun(io::Error), + + /// Cannot spawn a new vCPU thread. + VcpuSpawn(io::Error), + + #[cfg(target_arch = "x86_64")] + /// Cannot set the local interruption due to bad configuration. + LocalIntConfiguration(arch::x86_64::interrupts::Error), + + #[cfg(target_arch = "x86_64")] + /// Error configuring the MSR registers + MSRSConfiguration(arch::x86_64::regs::Error), + + #[cfg(target_arch = "x86_64")] + /// Error configuring the general purpose registers + REGSConfiguration(arch::x86_64::regs::Error), + + #[cfg(target_arch = "x86_64")] + /// Error configuring the special registers + SREGSConfiguration(arch::x86_64::regs::Error), + + #[cfg(target_arch = "x86_64")] + /// Error configuring the floating point related registers + FPUConfiguration(arch::x86_64::regs::Error), } pub type Result = result::Result; +/// A wrapper around creating and using a kvm-based VCPU. +pub struct Vcpu { + // #[cfg(target_arch = "x86_64")] + // cpuid: CpuId, + fd: VcpuFd, + id: u8, +} + +impl Vcpu { + /// Constructs a new VCPU for `vm`. + /// + /// # Arguments + /// + /// * `id` - Represents the CPU number between [0, max vcpus). + /// * `vm` - The virtual machine this vcpu will get attached to. + pub fn new(id: u8, vm: &Vm) -> Result { + let kvm_vcpu = vm.fd.create_vcpu(id).map_err(Error::VcpuFd)?; + // Initially the cpuid per vCPU is the one supported by this VM. + Ok(Vcpu { fd: kvm_vcpu, id }) + } + + /// Configures a x86_64 specific vcpu and should be called once per vcpu from the vcpu's thread. + /// + /// # Arguments + /// + /// * `machine_config` - Specifies necessary info used for the CPUID configuration. + /// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts. + /// * `vm` - The virtual machine this vcpu will get attached to. + pub fn configure(&mut self, kernel_start_addr: GuestAddress, vm: &Vm) -> Result<()> { + arch::x86_64::regs::setup_msrs(&self.fd).map_err(Error::MSRSConfiguration)?; + // Safe to unwrap because this method is called after the VM is configured + let vm_memory = vm.get_memory(); + arch::x86_64::regs::setup_regs( + &self.fd, + kernel_start_addr.raw_value(), + arch::x86_64::layout::BOOT_STACK_POINTER.raw_value(), + arch::x86_64::layout::ZERO_PAGE_START.raw_value(), + ) + .map_err(Error::REGSConfiguration)?; + arch::x86_64::regs::setup_fpu(&self.fd).map_err(Error::FPUConfiguration)?; + arch::x86_64::regs::setup_sregs(vm_memory, &self.fd).map_err(Error::SREGSConfiguration)?; + arch::x86_64::interrupts::set_lint(&self.fd).map_err(Error::LocalIntConfiguration)?; + Ok(()) + } + + /// Runs the VCPU until it exits, returning the reason. + /// + /// Note that the state of the VCPU and associated VM must be setup first for this to do + /// anything useful. + pub fn run(&self) -> Result { + self.fd.run().map_err(Error::VcpuRun) + } +} + struct VmConfig<'a> { kernel_path: &'a Path, cmdline: Option, @@ -122,6 +211,9 @@ impl<'a> Vm<'a> { fd.set_tss_address(arch::x86_64::layout::KVM_TSS_ADDRESS.raw_value() as usize) .map_err(Error::VmSetup)?; + // Create IRQ chip + fd.create_irq_chip().map_err(Error::VmSetup)?; + Ok(Vm { fd, kernel, @@ -133,9 +225,7 @@ impl<'a> Vm<'a> { pub fn load_kernel(&mut self) -> Result { let cmdline = self.config.cmdline.clone().ok_or(Error::CmdLine)?; - let cmdline_cstring = CString::new(cmdline).map_err(|_| Error::CmdLine)?; - let entry_addr = linux_loader::loader::Elf::load( &self.memory, None, @@ -164,9 +254,114 @@ impl<'a> Vm<'a> { Ok(entry_addr.kernel_load) } - pub fn start(&mut self) -> Result<()> { + pub fn start(&mut self, entry_addr: GuestAddress) -> Result<()> { + let vcpu_count = self.config.vcpu_count; + + let mut vcpus = Vec::with_capacity(vcpu_count as usize); + let vcpu_thread_barrier = Arc::new(Barrier::new((vcpu_count + 1) as usize)); + + for cpu_id in 0..vcpu_count { + println!("Starting VCPU {:?}", cpu_id); + let mut vcpu = Vcpu::new(cpu_id, &self)?; + let vcpu_thread_barrier = vcpu_thread_barrier.clone(); + + vcpu.configure(entry_addr, &self)?; + + vcpus.push( + thread::Builder::new() + .name(format!("cloud-hypervisor_vcpu{}", vcpu.id)) + .spawn(move || { + unsafe { + extern "C" fn handle_signal(_: i32, _: *mut siginfo_t, _: *mut c_void) { + } + // This uses an async signal safe handler to kill the vcpu handles. + register_signal_handler( + VCPU_RTSIG_OFFSET, + vmm_sys_util::signal::SignalHandler::Siginfo(handle_signal), + true, + 0, + ) + .expect("Failed to register vcpu signal handler"); + } + + vcpu_thread_barrier.wait(); + + loop { + match vcpu.run() { + Ok(run) => match run { + VcpuExit::IoIn(addr, data) => { + println!( + "IO in -- addr: {:#x} data [{:?}]", + addr, + str::from_utf8(&data).unwrap() + ); + } + VcpuExit::IoOut(addr, data) => { + println!( + "IO out -- addr: {:#x} data [{:?}]", + addr, + str::from_utf8(&data).unwrap() + ); + } + VcpuExit::MmioRead(_addr, _data) => {} + VcpuExit::MmioWrite(_addr, _data) => {} + VcpuExit::Unknown => {} + VcpuExit::Exception => {} + VcpuExit::Hypercall => {} + VcpuExit::Debug => {} + VcpuExit::Hlt => { + println!("HLT"); + } + VcpuExit::IrqWindowOpen => {} + VcpuExit::Shutdown => {} + VcpuExit::FailEntry => {} + VcpuExit::Intr => {} + VcpuExit::SetTpr => {} + VcpuExit::TprAccess => {} + VcpuExit::S390Sieic => {} + VcpuExit::S390Reset => {} + VcpuExit::Dcr => {} + VcpuExit::Nmi => {} + VcpuExit::InternalError => {} + VcpuExit::Osi => {} + VcpuExit::PaprHcall => {} + VcpuExit::S390Ucontrol => {} + VcpuExit::Watchdog => {} + VcpuExit::S390Tsch => {} + VcpuExit::Epr => {} + VcpuExit::SystemEvent => {} + VcpuExit::S390Stsi => {} + VcpuExit::IoapicEoi => {} + VcpuExit::Hyperv => {} + }, + Err(e) => { + println! {"VCPU {:?} error {:?}", cpu_id, e} + break; + } + } + } + }) + .map_err(Error::VcpuSpawn)?, + ); + } + + vcpu_thread_barrier.wait(); Ok(()) } + + /// Gets a reference to the guest memory owned by this VM. + /// + /// Note that `GuestMemory` does not include any device memory that may have been added after + /// this VM was constructed. + pub fn get_memory(&self) -> &GuestMemoryMmap { + &self.memory + } + + /// Gets a reference to the kvm file descriptor owned by this VM. + /// + pub fn get_fd(&self) -> &VmFd { + &self.fd + } } #[allow(unused)]