From 1099f0726bce9b08aa12e1877e5ed52e946f84d4 Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Wed, 11 Sep 2019 17:25:07 +0100 Subject: [PATCH] vmm: Add MMIO support Add (non-default) support for using MMIO for virtio devices. This can be tested by: cargo build --no-default-features --features "mmio" All necessary options will be included injected into the kernel commandline. Fixes: #243 Signed-off-by: Rob Bradford --- Cargo.toml | 1 + vmm/Cargo.toml | 1 + vmm/src/device_manager.rs | 127 ++++++++++++++++++++++++++++++++------ 3 files changed, 109 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aee1a7ca0..f62a412f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ lazy_static= "1.4.0" default = ["acpi", "pci"] acpi = ["vmm/acpi"] pci = ["vmm/pci_support"] +mmio = ["vmm/mmio_support"] # Integration tests require a special environment to run in integration_tests = [] diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index 32af1d5bb..6491b01d0 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" default = [] acpi = ["acpi_tables","devices/acpi", "arch/acpi"] pci_support = ["pci", "vfio", "vm-virtio/pci_support"] +mmio_support = ["vm-virtio/mmio_support"] [dependencies] acpi_tables = { path = "../acpi_tables", optional = true } diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index cfbc5bdeb..f0f9a1c8e 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -36,6 +36,8 @@ use std::sync::{Arc, Mutex, RwLock}; #[cfg(feature = "pci_support")] use vfio::{VfioDevice, VfioPciDevice, VfioPciError}; use vm_allocator::SystemAllocator; +#[cfg(feature = "mmio_support")] +use vm_memory::GuestAddress; use vm_memory::{Address, GuestMemoryMmap, GuestUsize}; #[cfg(feature = "pci_support")] use vm_virtio::transport::VirtioPciDevice; @@ -46,6 +48,9 @@ use vmm_sys_util::eventfd::EventFd; const IOAPIC_RANGE_ADDR: u64 = 0xfec0_0000; const IOAPIC_RANGE_SIZE: u64 = 0x20; +#[cfg(feature = "mmio_support")] +const MMIO_LEN: u64 = 0x1000; + /// Errors associated with device manager #[derive(Debug)] pub enum DeviceManagerError { @@ -403,31 +408,56 @@ impl DeviceManager { #[allow(unused_mut)] let mut cmdline_additions = Vec::new(); - #[cfg(feature = "pci_support")] - { - let pci_root = PciRoot::new(None); - let mut pci = PciConfigIo::new(pci_root); + if cfg!(feature = "pci_support") { + #[cfg(feature = "pci_support")] + { + let pci_root = PciRoot::new(None); + let mut pci = PciConfigIo::new(pci_root); - for device in virtio_devices { - DeviceManager::add_virtio_pci_device( - device, - vm_info.memory, - allocator, - vm_info.vm_fd, - &mut pci, - &mut buses, - &interrupt_info, + for device in virtio_devices { + DeviceManager::add_virtio_pci_device( + device, + vm_info.memory, + allocator, + vm_info.vm_fd, + &mut pci, + &mut buses, + &interrupt_info, + )?; + } + + DeviceManager::add_vfio_devices( + vm_info, allocator, &mut pci, &mut buses, mem_slots, )?; + let pci = Arc::new(Mutex::new(pci)); + io_bus + .insert(pci, 0xcf8, 0x8) + .map_err(DeviceManagerError::BusError)?; + } + } else if cfg!(feature = "mmio_support") { + #[cfg(feature = "mmio_support")] + { + for device in virtio_devices { + if let Some(addr) = + allocator.allocate_mmio_addresses(None, MMIO_LEN, Some(MMIO_LEN)) + { + DeviceManager::add_virtio_mmio_device( + device, + vm_info.memory, + allocator, + vm_info.vm_fd, + &mut buses, + &interrupt_info, + addr, + &mut cmdline_additions, + )?; + } else { + error!("Unable to allocate MMIO address!"); + } + } } - - DeviceManager::add_vfio_devices(vm_info, allocator, &mut pci, &mut buses, mem_slots)?; - let pci = Arc::new(Mutex::new(pci)); - io_bus - .insert(pci, 0xcf8, 0x8) - .map_err(DeviceManagerError::BusError)?; } - let mut dm = DeviceManager { io_bus, mmio_bus, @@ -934,6 +964,63 @@ impl DeviceManager { Ok(()) } + #[allow(clippy::too_many_arguments)] + #[cfg(feature = "mmio_support")] + fn add_virtio_mmio_device( + virtio_device: Box, + memory: &Arc>, + allocator: &mut SystemAllocator, + vm_fd: &Arc, + buses: &mut BusInfo, + interrupt_info: &InterruptInfo, + mmio_base: GuestAddress, + cmdline_additions: &mut Vec, + ) -> DeviceManagerResult<()> { + let mut mmio_device = vm_virtio::transport::MmioDevice::new(memory.clone(), virtio_device) + .map_err(DeviceManagerError::VirtioDevice)?; + + for (i, queue_evt) in mmio_device.queue_evts().iter().enumerate() { + let io_addr = IoEventAddress::Mmio( + mmio_base.0 + u64::from(vm_virtio::transport::NOTIFY_REG_OFFSET), + ); + vm_fd + .register_ioevent(queue_evt.as_raw_fd(), &io_addr, i as u32) + .map_err(DeviceManagerError::RegisterIoevent)?; + } + + let irq_num = allocator + .allocate_irq() + .ok_or(DeviceManagerError::AllocateIrq)?; + + let interrupt: Box = if let Some(ioapic) = interrupt_info.ioapic { + Box::new(UserIoapicIrq::new(ioapic.clone(), irq_num as usize)) + } else { + let irqfd = EventFd::new(EFD_NONBLOCK).map_err(DeviceManagerError::EventFd)?; + + vm_fd + .register_irqfd(irqfd.as_raw_fd(), irq_num as u32) + .map_err(DeviceManagerError::Irq)?; + + Box::new(KernelIoapicIrq::new(irqfd)) + }; + + mmio_device.assign_interrupt(interrupt); + + buses + .mmio + .insert(Arc::new(Mutex::new(mmio_device)), mmio_base.0, MMIO_LEN) + .map_err(DeviceManagerError::BusError)?; + + cmdline_additions.push(format!( + "virtio_mmio.device={}K@0x{:08x}:{}", + MMIO_LEN / 1024, + mmio_base.0, + irq_num + )); + + Ok(()) + } + fn register_devices(&mut self) -> DeviceManagerResult<()> { if self.console.serial.is_some() { // Insert serial device