From 53b2e19934abfe2920258e586e627498653b2371 Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Fri, 30 Jul 2021 14:48:58 +0000 Subject: [PATCH] vmm: Add support for hotplugging user devices Signed-off-by: Rob Bradford --- vmm/src/api/http.rs | 4 ++++ vmm/src/api/http_endpoint.rs | 17 ++++++++++++----- vmm/src/api/mod.rs | 19 +++++++++++++++++++ vmm/src/device_manager.rs | 22 ++++++++++++++++++++++ vmm/src/lib.rs | 25 ++++++++++++++++++++++++- vmm/src/vm.rs | 35 +++++++++++++++++++++++++++++++++-- 6 files changed, 114 insertions(+), 8 deletions(-) diff --git a/vmm/src/api/http.rs b/vmm/src/api/http.rs index 78f06d888..3ce3e6147 100644 --- a/vmm/src/api/http.rs +++ b/vmm/src/api/http.rs @@ -77,6 +77,9 @@ pub enum HttpError { /// Could not add a device to a VM VmAddDevice(ApiError), + /// Could not add a user device to the VM + VmAddUserDevice(ApiError), + /// Could not remove a device from a VM VmRemoveDevice(ApiError), @@ -207,6 +210,7 @@ lazy_static! { }; r.routes.insert(endpoint!("/vm.add-device"), Box::new(VmActionHandler::new(VmAction::AddDevice(Arc::default())))); + r.routes.insert(endpoint!("/vm.add-user-device"), Box::new(VmActionHandler::new(VmAction::AddUserDevice(Arc::default())))); r.routes.insert(endpoint!("/vm.add-disk"), Box::new(VmActionHandler::new(VmAction::AddDisk(Arc::default())))); r.routes.insert(endpoint!("/vm.add-fs"), Box::new(VmActionHandler::new(VmAction::AddFs(Arc::default())))); r.routes.insert(endpoint!("/vm.add-net"), Box::new(VmActionHandler::new(VmAction::AddNet(Arc::default())))); diff --git a/vmm/src/api/http_endpoint.rs b/vmm/src/api/http_endpoint.rs index fb7f4fcb3..68d8ab639 100644 --- a/vmm/src/api/http_endpoint.rs +++ b/vmm/src/api/http_endpoint.rs @@ -5,11 +5,11 @@ use crate::api::http::{error_response, EndpointHandler, HttpError}; use crate::api::{ - vm_add_device, vm_add_disk, vm_add_fs, vm_add_net, vm_add_pmem, vm_add_vsock, vm_boot, - vm_counters, vm_create, vm_delete, vm_info, vm_pause, vm_power_button, vm_reboot, - vm_receive_migration, vm_remove_device, vm_resize, vm_resize_zone, vm_restore, vm_resume, - vm_send_migration, vm_shutdown, vm_snapshot, vmm_ping, vmm_shutdown, ApiRequest, VmAction, - VmConfig, + vm_add_device, vm_add_disk, vm_add_fs, vm_add_net, vm_add_pmem, vm_add_user_device, + vm_add_vsock, vm_boot, vm_counters, vm_create, vm_delete, vm_info, vm_pause, vm_power_button, + vm_reboot, vm_receive_migration, vm_remove_device, vm_resize, vm_resize_zone, vm_restore, + vm_resume, vm_send_migration, vm_shutdown, vm_snapshot, vmm_ping, vmm_shutdown, ApiRequest, + VmAction, VmConfig, }; use crate::config::NetConfig; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; @@ -127,6 +127,13 @@ impl EndpointHandler for VmActionHandler { ) .map_err(HttpError::VmAddVsock), + AddUserDevice(_) => vm_add_user_device( + api_notifier, + api_sender, + Arc::new(serde_json::from_slice(body.raw())?), + ) + .map_err(HttpError::VmAddUserDevice), + RemoveDevice(_) => vm_remove_device( api_notifier, api_sender, diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 41dc0c89f..f43296c0a 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -34,6 +34,7 @@ pub use self::http::start_http_path_thread; pub mod http; pub mod http_endpoint; +use crate::config::UserDeviceConfig; use crate::config::{ DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, VmConfig, VsockConfig, }; @@ -109,6 +110,9 @@ pub enum ApiError { /// The device could not be added to the VM. VmAddDevice(VmError), + /// The user device could not be added to the VM. + VmAddUserDevice(VmError), + /// The device could not be removed from the VM. VmRemoveDevice(VmError), @@ -269,6 +273,9 @@ pub enum ApiRequest { /// Add a device to the VM. VmAddDevice(Arc, Sender), + /// Add a user device to the VM. + VmAddUserDevice(Arc, Sender), + /// Remove a device from the VM. VmRemoveDevice(Arc, Sender), @@ -364,6 +371,9 @@ pub enum VmAction { /// Add vsock AddVsock(Arc), + /// Add user device + AddUserDevice(Arc), + /// Remove VFIO device RemoveDevice(Arc), @@ -411,6 +421,7 @@ fn vm_action( AddPmem(v) => ApiRequest::VmAddPmem(v, response_sender), AddNet(v) => ApiRequest::VmAddNet(v, response_sender), AddVsock(v) => ApiRequest::VmAddVsock(v, response_sender), + AddUserDevice(v) => ApiRequest::VmAddUserDevice(v, response_sender), RemoveDevice(v) => ApiRequest::VmRemoveDevice(v, response_sender), Resize(v) => ApiRequest::VmResize(v, response_sender), ResizeZone(v) => ApiRequest::VmResizeZone(v, response_sender), @@ -572,6 +583,14 @@ pub fn vm_add_device( vm_action(api_evt, api_sender, VmAction::AddDevice(data)) } +pub fn vm_add_user_device( + api_evt: EventFd, + api_sender: Sender, + data: Arc, +) -> ApiResult> { + vm_action(api_evt, api_sender, VmAction::AddUserDevice(data)) +} + pub fn vm_remove_device( api_evt: EventFd, api_sender: Sender, diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index de213767c..7966552c9 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -3317,6 +3317,28 @@ impl DeviceManager { }) } + pub fn add_user_device( + &mut self, + device_cfg: &mut UserDeviceConfig, + ) -> DeviceManagerResult { + let pci = if let Some(pci_bus) = &self.pci_bus { + Arc::clone(pci_bus) + } else { + return Err(DeviceManagerError::NoPciBus); + }; + + let (device_id, device_name) = + self.add_vfio_user_device(&mut pci.lock().unwrap(), device_cfg)?; + + // Update the PCIU bitmap + self.pci_devices_up |= 1 << (device_id >> 3); + + Ok(PciDeviceInfo { + id: device_name, + bdf: device_id, + }) + } + pub fn remove_device(&mut self, id: String) -> DeviceManagerResult<()> { // The node can be directly a PCI node in case the 'id' refers to a // VFIO device or a virtio-pci one. diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 22f6a1320..6866f3749 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -20,7 +20,8 @@ use crate::api::{ VmSendMigrationData, VmmPingResponse, }; use crate::config::{ - DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, VmConfig, VsockConfig, + DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, UserDeviceConfig, + VmConfig, VsockConfig, }; use crate::migration::{get_vm_snapshot, recv_vm_snapshot}; use crate::seccomp_filters::{get_seccomp_filter, Thread}; @@ -645,6 +646,21 @@ impl Vmm { } } + fn vm_add_user_device( + &mut self, + device_cfg: UserDeviceConfig, + ) -> result::Result, VmError> { + if let Some(ref mut vm) = self.vm { + let info = vm.add_user_device(device_cfg).map_err(|e| { + error!("Error when adding new user device to the VM: {:?}", e); + e + })?; + serde_json::to_vec(&info).map_err(VmError::SerializeJson) + } else { + Err(VmError::VmNotRunning) + } + } + fn vm_remove_device(&mut self, id: String) -> result::Result<(), VmError> { if let Some(ref mut vm) = self.vm { if let Err(e) = vm.remove_device(id) { @@ -1414,6 +1430,13 @@ impl Vmm { .map(ApiResponsePayload::VmAction); sender.send(response).map_err(Error::ApiResponseSend)?; } + ApiRequest::VmAddUserDevice(add_device_data, sender) => { + let response = self + .vm_add_user_device(add_device_data.as_ref().clone()) + .map_err(ApiError::VmAddUserDevice) + .map(ApiResponsePayload::VmAction); + sender.send(response).map_err(Error::ApiResponseSend)?; + } ApiRequest::VmRemoveDevice(remove_device_data, sender) => { let response = self .vm_remove_device(remove_device_data.id.clone()) diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 4f34e1682..60cf55460 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -14,8 +14,8 @@ #[cfg(any(target_arch = "aarch64", feature = "acpi"))] use crate::config::NumaConfig; use crate::config::{ - DeviceConfig, DiskConfig, FsConfig, HotplugMethod, NetConfig, PmemConfig, ValidationError, - VmConfig, VsockConfig, + DeviceConfig, DiskConfig, FsConfig, HotplugMethod, NetConfig, PmemConfig, UserDeviceConfig, + ValidationError, VmConfig, VsockConfig, }; use crate::cpu; use crate::device_manager::{ @@ -1345,6 +1345,37 @@ impl Vm { Ok(pci_device_info) } + pub fn add_user_device(&mut self, mut device_cfg: UserDeviceConfig) -> Result { + { + // Validate on a clone of the config + let mut config = self.config.lock().unwrap().clone(); + Self::add_to_config(&mut config.user_devices, device_cfg.clone()); + config.validate().map_err(Error::ConfigValidation)?; + } + + let pci_device_info = self + .device_manager + .lock() + .unwrap() + .add_user_device(&mut device_cfg) + .map_err(Error::DeviceManager)?; + + // Update VmConfig by adding the new device. This is important to + // ensure the device would be created in case of a reboot. + { + let mut config = self.config.lock().unwrap(); + Self::add_to_config(&mut config.user_devices, device_cfg); + } + + self.device_manager + .lock() + .unwrap() + .notify_hotplug(AcpiNotificationFlags::PCI_DEVICES_CHANGED) + .map_err(Error::DeviceManager)?; + + Ok(pci_device_info) + } + pub fn remove_device(&mut self, _id: String) -> Result<()> { self.device_manager .lock()