From 3fea5f539655b63bc2306f0e37dea1f3911efb5d Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 11 Mar 2022 11:32:53 +0100 Subject: [PATCH] vmm: Add support for hotplugging a vDPA device Signed-off-by: Sebastien Boeuf --- docs/api.md | 1 + vmm/src/api/http.rs | 4 ++ vmm/src/api/http_endpoint.rs | 15 ++-- vmm/src/api/mod.rs | 22 +++++- vmm/src/api/openapi/cloud-hypervisor.yaml | 23 ++++++- vmm/src/device_manager.rs | 5 ++ vmm/src/lib.rs | 83 ++++++++++++++++++++++- vmm/src/vm.rs | 31 ++++++++- 8 files changed, 175 insertions(+), 9 deletions(-) diff --git a/docs/api.md b/docs/api.md index 7fff6f310..5ffd07194 100644 --- a/docs/api.md +++ b/docs/api.md @@ -100,6 +100,7 @@ Add fs device to the VM | `/vm.add-fs` | `/schemas/FsConfig` Add pmem device to the VM | `/vm.add-pmem` | `/schemas/PmemConfig` | `/schemas/PciDeviceInfo` | The VM is booted Add network device to the VM | `/vm.add-net` | `/schemas/NetConfig` | `/schemas/PciDeviceInfo` | The VM is booted Add userspace PCI device to the VM | `/vm.add-user-device`| `/schemas/VmAddUserDevice`| `/schemas/PciDeviceInfo` | The VM is booted +Add vdpa device to the VM | `/vm.add-vdpa` | `/schemas/VdpaConfig` | `/schemas/PciDeviceInfo` | The VM is booted Add vsock device to the VM | `/vm.add-vsock` | `/schemas/VsockConfig` | `/schemas/PciDeviceInfo` | The VM is booted Remove device from the VM | `/vm.remove-device` | `/schemas/VmRemoveDevice` | N/A | The VM is booted Dump the VM counters | `/vm.counters` | N/A | `/schemas/VmCounters` | The VM is booted diff --git a/vmm/src/api/http.rs b/vmm/src/api/http.rs index 897ed220b..9fa86e4a6 100644 --- a/vmm/src/api/http.rs +++ b/vmm/src/api/http.rs @@ -102,6 +102,9 @@ pub enum HttpError { /// Could not add a network device to a VM VmAddNet(ApiError), + /// Could not add a vDPA device to a VM + VmAddVdpa(ApiError), + /// Could not add a vsock device to a VM VmAddVsock(ApiError), @@ -220,6 +223,7 @@ lazy_static! { r.routes.insert(endpoint!("/vm.add-fs"), Box::new(VmActionHandler::new(VmAction::AddFs(Arc::default())))); r.routes.insert(endpoint!("/vm.add-net"), Box::new(VmActionHandler::new(VmAction::AddNet(Arc::default())))); r.routes.insert(endpoint!("/vm.add-pmem"), Box::new(VmActionHandler::new(VmAction::AddPmem(Arc::default())))); + r.routes.insert(endpoint!("/vm.add-vdpa"), Box::new(VmActionHandler::new(VmAction::AddVdpa(Arc::default())))); r.routes.insert(endpoint!("/vm.add-vsock"), Box::new(VmActionHandler::new(VmAction::AddVsock(Arc::default())))); r.routes.insert(endpoint!("/vm.boot"), Box::new(VmActionHandler::new(VmAction::Boot))); r.routes.insert(endpoint!("/vm.counters"), Box::new(VmActionHandler::new(VmAction::Counters))); diff --git a/vmm/src/api/http_endpoint.rs b/vmm/src/api/http_endpoint.rs index 67861a718..2ba17fd56 100644 --- a/vmm/src/api/http_endpoint.rs +++ b/vmm/src/api/http_endpoint.rs @@ -6,10 +6,10 @@ use crate::api::http::{error_response, EndpointHandler, HttpError}; use crate::api::{ vm_add_device, vm_add_disk, vm_add_fs, vm_add_net, vm_add_pmem, vm_add_user_device, - vm_add_vsock, vm_boot, vm_counters, vm_create, vm_delete, vm_info, vm_pause, vm_power_button, - vm_reboot, vm_receive_migration, vm_remove_device, vm_resize, vm_resize_zone, vm_restore, - vm_resume, vm_send_migration, vm_shutdown, vm_snapshot, vmm_ping, vmm_shutdown, ApiRequest, - VmAction, VmConfig, + vm_add_vdpa, vm_add_vsock, vm_boot, vm_counters, vm_create, vm_delete, vm_info, vm_pause, + vm_power_button, vm_reboot, vm_receive_migration, vm_remove_device, vm_resize, vm_resize_zone, + vm_restore, vm_resume, vm_send_migration, vm_shutdown, vm_snapshot, vmm_ping, vmm_shutdown, + ApiRequest, VmAction, VmConfig, }; use crate::config::NetConfig; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; @@ -121,6 +121,13 @@ impl EndpointHandler for VmActionHandler { .map_err(HttpError::VmAddNet) } + AddVdpa(_) => vm_add_vdpa( + api_notifier, + api_sender, + Arc::new(serde_json::from_slice(body.raw())?), + ) + .map_err(HttpError::VmAddVdpa), + AddVsock(_) => vm_add_vsock( api_notifier, api_sender, diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 7c4452d2c..19b29536c 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -34,9 +34,9 @@ pub use self::http::start_http_path_thread; pub mod http; pub mod http_endpoint; -use crate::config::UserDeviceConfig; use crate::config::{ - DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, VmConfig, VsockConfig, + DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, UserDeviceConfig, + VdpaConfig, VmConfig, VsockConfig, }; use crate::device_tree::DeviceTree; use crate::vm::{Error as VmError, VmState}; @@ -134,6 +134,9 @@ pub enum ApiError { /// The network device could not be added to the VM. VmAddNet(VmError), + /// The vDPA device could not be added to the VM. + VmAddVdpa(VmError), + /// The vsock device could not be added to the VM. VmAddVsock(VmError), @@ -294,6 +297,9 @@ pub enum ApiRequest { /// Add a network device to the VM. VmAddNet(Arc, Sender), + /// Add a vDPA device to the VM. + VmAddVdpa(Arc, Sender), + /// Add a vsock device to the VM. VmAddVsock(Arc, Sender), @@ -371,6 +377,9 @@ pub enum VmAction { /// Add network AddNet(Arc), + /// Add vdpa + AddVdpa(Arc), + /// Add vsock AddVsock(Arc), @@ -423,6 +432,7 @@ fn vm_action( AddFs(v) => ApiRequest::VmAddFs(v, response_sender), AddPmem(v) => ApiRequest::VmAddPmem(v, response_sender), AddNet(v) => ApiRequest::VmAddNet(v, response_sender), + AddVdpa(v) => ApiRequest::VmAddVdpa(v, response_sender), AddVsock(v) => ApiRequest::VmAddVsock(v, response_sender), AddUserDevice(v) => ApiRequest::VmAddUserDevice(v, response_sender), RemoveDevice(v) => ApiRequest::VmRemoveDevice(v, response_sender), @@ -634,6 +644,14 @@ pub fn vm_add_net( vm_action(api_evt, api_sender, VmAction::AddNet(data)) } +pub fn vm_add_vdpa( + api_evt: EventFd, + api_sender: Sender, + data: Arc, +) -> ApiResult> { + vm_action(api_evt, api_sender, VmAction::AddVdpa(data)) +} + pub fn vm_add_vsock( api_evt: EventFd, api_sender: Sender, diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml index 8531bdcf3..533e610ca 100644 --- a/vmm/src/api/openapi/cloud-hypervisor.yaml +++ b/vmm/src/api/openapi/cloud-hypervisor.yaml @@ -325,7 +325,28 @@ paths: description: The new device was successfully (cold) added to the VM instance. 500: description: The new device could not be added to the VM instance. - + + /vm.add-vdpa: + put: + summary: Add a new vDPA device to the VM + requestBody: + description: The details of the new vDPA device + content: + application/json: + schema: + $ref: '#/components/schemas/VdpaConfig' + required: true + responses: + 200: + description: The new vDPA device was successfully added to the VM instance. + content: + application/json: + schema: + $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new vDPA device was successfully (cold) added to the VM instance. + 500: + description: The new vDPA device could not be added to the VM instance. /vm.snapshot: put: diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index 9a776d908..41fb1b5b9 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -3977,6 +3977,11 @@ impl DeviceManager { self.hotplug_virtio_pci_device(device) } + pub fn add_vdpa(&mut self, vdpa_cfg: &mut VdpaConfig) -> DeviceManagerResult { + let device = self.make_vdpa_device(vdpa_cfg)?; + self.hotplug_virtio_pci_device(device) + } + pub fn add_vsock(&mut self, vsock_cfg: &mut VsockConfig) -> DeviceManagerResult { let device = self.make_virtio_vsock_device(vsock_cfg)?; self.hotplug_virtio_pci_device(device) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index a783c069c..36e5f5748 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -18,7 +18,7 @@ use crate::api::{ }; use crate::config::{ add_to_config, DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, - UserDeviceConfig, VmConfig, VsockConfig, + UserDeviceConfig, VdpaConfig, VmConfig, VsockConfig, }; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] use crate::migration::get_vm_snapshot; @@ -925,6 +925,32 @@ impl Vmm { } } + fn vm_add_vdpa(&mut self, vdpa_cfg: VdpaConfig) -> result::Result>, VmError> { + self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; + + { + // Validate the configuration change in a cloned configuration + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap().clone(); + add_to_config(&mut config.vdpa, vdpa_cfg.clone()); + config.validate().map_err(VmError::ConfigValidation)?; + } + + if let Some(ref mut vm) = self.vm { + let info = vm.add_vdpa(vdpa_cfg).map_err(|e| { + error!("Error when adding new vDPA device to the VM: {:?}", e); + e + })?; + serde_json::to_vec(&info) + .map(Some) + .map_err(VmError::SerializeJson) + } else { + // Update VmConfig by adding the new device. + let mut config = self.vm_config.as_ref().unwrap().lock().unwrap(); + add_to_config(&mut config.vdpa, vdpa_cfg); + Ok(None) + } + } + fn vm_add_vsock(&mut self, vsock_cfg: VsockConfig) -> result::Result>, VmError> { self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?; @@ -1742,6 +1768,13 @@ impl Vmm { .map(ApiResponsePayload::VmAction); sender.send(response).map_err(Error::ApiResponseSend)?; } + ApiRequest::VmAddVdpa(add_vdpa_data, sender) => { + let response = self + .vm_add_vdpa(add_vdpa_data.as_ref().clone()) + .map_err(ApiError::VmAddVdpa) + .map(ApiResponsePayload::VmAction); + sender.send(response).map_err(Error::ApiResponseSend)?; + } ApiRequest::VmAddVsock(add_vsock_data, sender) => { let response = self .vm_add_vsock(add_vsock_data.as_ref().clone()) @@ -2199,6 +2232,54 @@ mod unit_tests { ); } + #[test] + fn test_vmm_vm_cold_add_vdpa() { + let mut vmm = create_dummy_vmm(); + let vdpa_config = VdpaConfig::parse("path=/dev/vhost-vdpa,num_queues=2").unwrap(); + + assert!(matches!( + vmm.vm_add_vdpa(vdpa_config.clone()), + Err(VmError::VmNotCreated) + )); + + let _ = vmm.vm_create(create_dummy_vm_config()); + assert!(vmm + .vm_config + .as_ref() + .unwrap() + .lock() + .unwrap() + .vdpa + .is_none()); + + let result = vmm.vm_add_vdpa(vdpa_config.clone()); + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + assert_eq!( + vmm.vm_config + .as_ref() + .unwrap() + .lock() + .unwrap() + .vdpa + .clone() + .unwrap() + .len(), + 1 + ); + assert_eq!( + vmm.vm_config + .as_ref() + .unwrap() + .lock() + .unwrap() + .vdpa + .clone() + .unwrap()[0], + vdpa_config + ); + } + #[test] fn test_vmm_vm_cold_add_vsock() { let mut vmm = create_dummy_vmm(); diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 2e352d28f..5b57b6949 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -15,7 +15,7 @@ use crate::config::NumaConfig; use crate::config::{ add_to_config, DeviceConfig, DiskConfig, FsConfig, HotplugMethod, NetConfig, PmemConfig, - UserDeviceConfig, ValidationError, VmConfig, VsockConfig, + UserDeviceConfig, ValidationError, VdpaConfig, VmConfig, VsockConfig, }; use crate::cpu; use crate::device_manager::{self, Console, DeviceManager, DeviceManagerError, PtyPair}; @@ -1506,6 +1506,11 @@ impl Vm { pmem.retain(|dev| dev.id.as_ref() != Some(&id)); } + // Remove if vDPA device + if let Some(vdpa) = config.vdpa.as_mut() { + vdpa.retain(|dev| dev.id.as_ref() != Some(&id)); + } + // Remove if vsock device if let Some(vsock) = config.vsock.as_ref() { if vsock.id.as_ref() == Some(&id) { @@ -1617,6 +1622,30 @@ impl Vm { Ok(pci_device_info) } + pub fn add_vdpa(&mut self, mut vdpa_cfg: VdpaConfig) -> Result { + let pci_device_info = self + .device_manager + .lock() + .unwrap() + .add_vdpa(&mut vdpa_cfg) + .map_err(Error::DeviceManager)?; + + // Update VmConfig by adding the new device. This is important to + // ensure the device would be created in case of a reboot. + { + let mut config = self.config.lock().unwrap(); + add_to_config(&mut config.vdpa, vdpa_cfg); + } + + self.device_manager + .lock() + .unwrap() + .notify_hotplug(AcpiNotificationFlags::PCI_DEVICES_CHANGED) + .map_err(Error::DeviceManager)?; + + Ok(pci_device_info) + } + pub fn add_vsock(&mut self, mut vsock_cfg: VsockConfig) -> Result { let pci_device_info = self .device_manager