From 4d16ca8ae79675a6792f23f16179e080b65eebe4 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 15 Jul 2019 11:42:40 +0200 Subject: [PATCH] vmm: Support direct device assignment With the VFIO crate, we can now support directly assigned PCI devices into cloud-hypervisor guests. We support assigning multiple host devices, through the --device command line parameter. This parameter takes the host device sysfs path. Fixes: #60 Signed-off-by: Samuel Ortiz --- Cargo.lock | 25 ++++++++++++++++++++ src/main.rs | 9 +++++++ vmm/Cargo.toml | 1 + vmm/src/config.rs | 25 ++++++++++++++++++++ vmm/src/vm.rs | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index cdc0d1fe7..0f1086e66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -982,6 +982,30 @@ name = "vec_map" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "vfio" +version = "0.0.1" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "devices 0.1.0", + "kvm-bindings 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvm-ioctls 0.2.0 (git+https://github.com/rust-vmm/kvm-ioctls)", + "libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pci 0.1.0", + "vfio-bindings 0.0.1", + "vm-allocator 0.1.0", + "vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)", + "vmm-sys-util 0.1.0 (git+https://github.com/rust-vmm/vmm-sys-util)", +] + +[[package]] +name = "vfio-bindings" +version = "0.0.1" +dependencies = [ + "vmm-sys-util 0.1.0 (git+https://github.com/rust-vmm/vmm-sys-util)", +] + [[package]] name = "vhost_rs" version = "0.1.0" @@ -1049,6 +1073,7 @@ dependencies = [ "net_util 0.1.0", "pci 0.1.0", "qcow 0.1.0", + "vfio 0.0.1", "vm-allocator 0.1.0", "vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)", "vm-virtio 0.1.0", diff --git a/src/main.rs b/src/main.rs index be5888fdf..db1d6e0c5 100755 --- a/src/main.rs +++ b/src/main.rs @@ -100,6 +100,13 @@ fn main() { .help("Control (virtio) console: off|tty|file=/path/to/a/file") .default_value("tty"), ) + .arg( + Arg::with_name("device") + .long("device") + .help("Direct device assignment parameter") + .takes_value(true) + .min_values(1), + ) .get_matches(); // These .unwrap()s cannot fail as there is a default value defined @@ -118,6 +125,7 @@ fn main() { let console = cmd_arguments.value_of("console").unwrap(); let fs: Option> = cmd_arguments.values_of("fs").map(|x| x.collect()); let pmem: Option> = cmd_arguments.values_of("pmem").map(|x| x.collect()); + let devices: Option> = cmd_arguments.values_of("device").map(|x| x.collect()); let vm_config = match config::VmConfig::parse(config::VmParams { cpus, @@ -131,6 +139,7 @@ fn main() { pmem, serial, console, + devices, }) { Ok(config) => config, Err(e) => { diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index f175f7376..3546de34a 100755 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -15,6 +15,7 @@ log = "*" net_util = { path = "../net_util" } pci = {path = "../pci"} qcow = { path = "../qcow" } +vfio = { path = "../vfio" } vm-virtio = { path = "../vm-virtio" } vm-allocator = { path = "../vm-allocator" } vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" } diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 77f06d0df..8181cc231 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -69,6 +69,7 @@ pub struct VmParams<'a> { pub pmem: Option>, pub serial: &'a str, pub console: &'a str, + pub devices: Option>, } fn parse_size(size: &str) -> Result { @@ -379,6 +380,19 @@ impl<'a> ConsoleConfig<'a> { } } +#[derive(Debug)] +pub struct DeviceConfig<'a> { + pub path: &'a Path, +} + +impl<'a> DeviceConfig<'a> { + pub fn parse(device: &'a str) -> Result { + Ok(DeviceConfig { + path: Path::new(device), + }) + } +} + pub struct VmConfig<'a> { pub cpus: CpusConfig, pub memory: MemoryConfig<'a>, @@ -391,6 +405,7 @@ pub struct VmConfig<'a> { pub pmem: Option>>, pub serial: ConsoleConfig<'a>, pub console: ConsoleConfig<'a>, + pub devices: Option>>, } impl<'a> VmConfig<'a> { @@ -437,6 +452,15 @@ impl<'a> VmConfig<'a> { return Err(Error::ParseTTYParam); } + let mut devices: Option> = None; + if let Some(device_list) = &vm_params.devices { + let mut device_config_list = Vec::new(); + for item in device_list.iter() { + device_config_list.push(DeviceConfig::parse(item)?); + } + devices = Some(device_config_list); + } + Ok(VmConfig { cpus: CpusConfig::parse(vm_params.cpus)?, memory: MemoryConfig::parse(vm_params.memory)?, @@ -449,6 +473,7 @@ impl<'a> VmConfig<'a> { pmem, serial, console, + devices, }) } } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 86e627229..27b8d16b5 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -16,6 +16,7 @@ extern crate kvm_ioctls; extern crate libc; extern crate linux_loader; extern crate net_util; +extern crate vfio; extern crate vm_allocator; extern crate vm_memory; extern crate vm_virtio; @@ -45,6 +46,7 @@ use std::os::unix::io::{AsRawFd, RawFd}; use std::ptr::null_mut; use std::sync::{Arc, Barrier, Mutex}; use std::{result, str, thread}; +use vfio::{VfioDevice, VfioPciDevice, VfioPciError}; use vm_allocator::{GsiApic, SystemAllocator}; use vm_memory::guest_memory::FileOffset; use vm_memory::{ @@ -245,6 +247,15 @@ pub enum DeviceManagerError { /// Error creating console output file ConsoleOutputFileOpen(io::Error), + + /// Cannot create a VFIO device + VfioCreate(vfio::VfioError), + + /// Cannot create a VFIO PCI device + VfioPciCreate(vfio::VfioPciError), + + /// Failed to map VFIO MMIO region. + VfioMapRegion(VfioPciError), } pub type DeviceManagerResult = result::Result; @@ -513,6 +524,7 @@ impl DeviceManager { vm_cfg: &VmConfig, msi_capable: bool, userspace_ioapic: bool, + mem_slots: u32, ) -> DeviceManagerResult { let mut io_bus = devices::Bus::new(); let mut mmio_bus = devices::Bus::new(); @@ -608,6 +620,16 @@ impl DeviceManager { &interrupt_info, )?; + DeviceManager::add_vfio_devices( + memory.clone(), + allocator, + vm_fd, + &vm_cfg, + &mut pci, + &mut buses, + mem_slots, + )?; + let pci = Arc::new(Mutex::new(pci)); Ok(DeviceManager { @@ -924,6 +946,43 @@ impl DeviceManager { Ok(()) } + fn add_vfio_devices( + memory: GuestMemoryMmap, + allocator: &mut SystemAllocator, + vm_fd: &Arc, + vm_cfg: &VmConfig, + pci: &mut PciConfigIo, + buses: &mut BusInfo, + mem_slots: u32, + ) -> DeviceManagerResult<()> { + if let Some(device_list_cfg) = &vm_cfg.devices { + for device_cfg in device_list_cfg.iter() { + let vfio_device = VfioDevice::new(device_cfg.path, vm_fd, memory.clone()) + .map_err(DeviceManagerError::VfioCreate)?; + + let mut vfio_pci_device = VfioPciDevice::new(vm_fd, allocator, vfio_device) + .map_err(DeviceManagerError::VfioPciCreate)?; + + let bars = vfio_pci_device + .allocate_bars(allocator) + .map_err(DeviceManagerError::AllocateBars)?; + + vfio_pci_device + .map_mmio_regions(vm_fd, mem_slots) + .map_err(DeviceManagerError::VfioMapRegion)?; + + let vfio_pci_device = Arc::new(Mutex::new(vfio_pci_device)); + + pci.add_device(vfio_pci_device.clone()) + .map_err(DeviceManagerError::AddPciDevice)?; + + pci.register_mapping(vfio_pci_device.clone(), buses.io, buses.mmio, bars) + .map_err(DeviceManagerError::AddPciDevice)?; + } + } + Ok(()) + } + fn add_virtio_pci_device( virtio_device: Box, memory: GuestMemoryMmap, @@ -1322,6 +1381,7 @@ impl<'a> Vm<'a> { &config, msi_capable, userspace_ioapic, + arch_mem_regions.len() as u32, ) .map_err(Error::DeviceManager)?;