From 6cbdb9aa47ed3c4395516eb5988c1a4fa5586ed3 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Mon, 9 Mar 2020 11:49:15 +0100 Subject: [PATCH] vmm: api: Introduce new "remove-device" HTTP endpoint This commit introduces the new command "remove-device" that will let a user hot-unplug a VFIO PCI device from an already running VM. Signed-off-by: Sebastien Boeuf --- docs/api.md | 27 ++++++------ vmm/src/api/http.rs | 3 +- vmm/src/api/http_endpoint.rs | 53 +++++++++++++++++++++-- vmm/src/api/mod.rs | 29 +++++++++++++ vmm/src/api/openapi/cloud-hypervisor.yaml | 22 ++++++++++ vmm/src/device_manager.rs | 16 +++++++ vmm/src/lib.rs | 20 +++++++++ vmm/src/vm.rs | 22 ++++++++++ 8 files changed, 175 insertions(+), 17 deletions(-) diff --git a/docs/api.md b/docs/api.md index abb421e54..39ce40f4f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -78,19 +78,20 @@ Shut the VMM down | `/vmm.shutdown` | N/A | N/A #### Virtual Machine (VM) Actions -Action | Endpoint | Request Body | Response Body | Prerequisites ----------------------------------|------------------|------------------------|-------------------|--------------------------- -Create the VM | `/vm.create` | `/schemas/VmConfig` | N/A | The VM is not created yet -Delete the VM | `/vm.delete` | N/A | N/A | The VM is created but not booted -Boot the VM | `/vm.boot` | N/A | N/A | The VM is created -Shut the VM down | `/vm.shutdown` | N/A | N/A | The VM is booted -Reboot the VM | `/vm.reboot` | N/A | N/A | The VM is booted -Pause the VM | `/vm.pause` | N/A | N/A | The VM is booted -Resume the VM | `/vm.resume` | N/A | N/A | The VM is paused -Add/remove CPUs to/from the VM | `/vm.resize` | `/schemas/VmResize` | N/A | The VM is booted -Remove memory from the VM | `/vm.resize` | `/schemas/VmResize` | N/A | The VM is booted -Dump the VM information | `/vm.info` | N/A | `/schemas/VmInfo` | The VM is created -Add VFIO PCI device to the VM | `/vm.add-device` | `/schemas/VmAddDevice` | N/A | The VM is booted +Action | Endpoint | Request Body | Response Body | Prerequisites +-----------------------------------|---------------------|---------------------------|-------------------|--------------------------- +Create the VM | `/vm.create` | `/schemas/VmConfig` | N/A | The VM is not created yet +Delete the VM | `/vm.delete` | N/A | N/A | The VM is created but not booted +Boot the VM | `/vm.boot` | N/A | N/A | The VM is created +Shut the VM down | `/vm.shutdown` | N/A | N/A | The VM is booted +Reboot the VM | `/vm.reboot` | N/A | N/A | The VM is booted +Pause the VM | `/vm.pause` | N/A | N/A | The VM is booted +Resume the VM | `/vm.resume` | N/A | N/A | The VM is paused +Add/remove CPUs to/from the VM | `/vm.resize` | `/schemas/VmResize` | N/A | The VM is booted +Remove memory from the VM | `/vm.resize` | `/schemas/VmResize` | N/A | The VM is booted +Dump the VM information | `/vm.info` | N/A | `/schemas/VmInfo` | The VM is created +Add VFIO PCI device to the VM | `/vm.add-device` | `/schemas/VmAddDevice` | N/A | The VM is booted +Remove VFIO PCI device from the VM | `/vm.remove-device` | `/schemas/VmRemoveDevice` | N/A | The VM is booted ### REST API Examples diff --git a/vmm/src/api/http.rs b/vmm/src/api/http.rs index d59e7b776..ecdd47959 100644 --- a/vmm/src/api/http.rs +++ b/vmm/src/api/http.rs @@ -4,7 +4,7 @@ // use crate::api::http_endpoint::{ - VmActionHandler, VmAddDevice, VmCreate, VmInfo, VmResize, VmmPing, VmmShutdown, + VmActionHandler, VmAddDevice, VmCreate, VmInfo, VmRemoveDevice, VmResize, VmmPing, VmmShutdown, }; use crate::api::{ApiRequest, VmAction}; use crate::{Error, Result}; @@ -63,6 +63,7 @@ lazy_static! { r.routes.insert(endpoint!("/vmm.ping"), Box::new(VmmPing {})); r.routes.insert(endpoint!("/vm.resize"), Box::new(VmResize {})); r.routes.insert(endpoint!("/vm.add-device"), Box::new(VmAddDevice {})); + r.routes.insert(endpoint!("/vm.remove-device"), Box::new(VmRemoveDevice {})); r }; diff --git a/vmm/src/api/http_endpoint.rs b/vmm/src/api/http_endpoint.rs index ec79a08fb..7c0b46b91 100644 --- a/vmm/src/api/http_endpoint.rs +++ b/vmm/src/api/http_endpoint.rs @@ -5,9 +5,9 @@ use crate::api::http::EndpointHandler; use crate::api::{ - vm_add_device, vm_boot, vm_create, vm_delete, vm_info, vm_pause, vm_reboot, vm_resize, - vm_resume, vm_shutdown, vmm_ping, vmm_shutdown, ApiError, ApiRequest, ApiResult, VmAction, - VmAddDeviceData, VmConfig, VmResizeData, + vm_add_device, vm_boot, vm_create, vm_delete, vm_info, vm_pause, vm_reboot, vm_remove_device, + vm_resize, vm_resume, vm_shutdown, vmm_ping, vmm_shutdown, ApiError, ApiRequest, ApiResult, + VmAction, VmAddDeviceData, VmConfig, VmRemoveDeviceData, VmResizeData, }; use micro_http::{Body, Method, Request, Response, StatusCode, Version}; use serde_json::Error as SerdeError; @@ -51,6 +51,9 @@ pub enum HttpError { /// Could not add a device to a VM VmAddDevice(ApiError), + /// Could not remove a device from a VM + VmRemoveDevice(ApiError), + /// Could not shut the VMM down VmmShutdown(ApiError), @@ -305,3 +308,47 @@ impl EndpointHandler for VmAddDevice { } } } + +// /api/v1/vm.remove-device handler +pub struct VmRemoveDevice {} + +impl EndpointHandler for VmRemoveDevice { + fn handle_request( + &self, + req: &Request, + api_notifier: EventFd, + api_sender: Sender, + ) -> Response { + match req.method() { + Method::Put => { + match &req.body { + Some(body) => { + // Deserialize into a VmRemoveDeviceData + let vm_remove_device_data: VmRemoveDeviceData = + match serde_json::from_slice(body.raw()) + .map_err(HttpError::SerdeJsonDeserialize) + { + Ok(config) => config, + Err(e) => return error_response(e, StatusCode::BadRequest), + }; + + // Call vm_remove_device() + match vm_remove_device( + api_notifier, + api_sender, + Arc::new(vm_remove_device_data), + ) + .map_err(HttpError::VmRemoveDevice) + { + Ok(_) => Response::new(Version::Http11, StatusCode::NoContent), + Err(e) => error_response(e, StatusCode::InternalServerError), + } + } + + None => Response::new(Version::Http11, StatusCode::BadRequest), + } + } + _ => Response::new(Version::Http11, StatusCode::BadRequest), + } + } +} diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 9e2a0cbba..6c5e3e6f0 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -102,6 +102,9 @@ pub enum ApiError { /// The device could not be added to the VM. VmAddDevice(VmError), + + /// The device could not be removed from the VM. + VmRemoveDevice(VmError), } pub type ApiResult = std::result::Result; @@ -127,6 +130,11 @@ pub struct VmAddDeviceData { pub path: String, } +#[derive(Clone, Deserialize, Serialize)] +pub struct VmRemoveDeviceData { + pub id: String, +} + pub enum ApiResponsePayload { /// No data is sent on the channel. Empty, @@ -192,6 +200,9 @@ pub enum ApiRequest { /// Add a device to the VM. VmAddDevice(Arc, Sender), + + /// Remove a device from the VM. + VmRemoveDevice(Arc, Sender), } pub fn vm_create( @@ -362,3 +373,21 @@ pub fn vm_add_device( Ok(()) } + +pub fn vm_remove_device( + api_evt: EventFd, + api_sender: Sender, + data: Arc, +) -> ApiResult<()> { + let (response_sender, response_receiver) = channel(); + + // Send the VM remove-device request. + api_sender + .send(ApiRequest::VmRemoveDevice(data, response_sender)) + .map_err(ApiError::RequestSend)?; + api_evt.write(1).map_err(ApiError::EventFdWrite)?; + + response_receiver.recv().map_err(ApiError::ResponseRecv)??; + + Ok(()) +} diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml index fdbc08867..fc3e568c5 100644 --- a/vmm/src/api/openapi/cloud-hypervisor.yaml +++ b/vmm/src/api/openapi/cloud-hypervisor.yaml @@ -155,6 +155,22 @@ paths: 404: description: The new device could not be added to the VM instance. + /vm.remove-device: + put: + summary: Remove a device from the VM + requestBody: + description: The identifier of the device + content: + application/json: + schema: + $ref: '#/components/schemas/VmRemoveDevice' + required: true + responses: + 204: + description: The device was successfully removed from the VM instance. + 404: + description: The device could not be removed from the VM instance. + components: schemas: @@ -449,3 +465,9 @@ components: properties: path: type: string + + VmRemoveDevice: + type: object + properties: + id: + type: string diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index a9a1648dd..2ec0575f7 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -236,6 +236,10 @@ pub enum DeviceManagerError { /// Failed removing a bus device from the MMIO bus. RemoveDeviceFromMmioBus(devices::BusError), + + /// Failed to find VFIO device corresponding to the given identifier. + #[cfg(feature = "pci_support")] + UnknownVfioDeviceId(String), } pub type DeviceManagerResult = result::Result; @@ -1791,6 +1795,18 @@ impl DeviceManager { Ok(device_cfg) } + #[cfg(feature = "pci_support")] + pub fn remove_device(&mut self, id: String) -> DeviceManagerResult<()> { + if let Some(pci_device_bdf) = self.pci_id_list.get(&id) { + // Update the PCID bitmap + self.pci_devices_down |= 1 << (*pci_device_bdf >> 3); + + Ok(()) + } else { + Err(DeviceManagerError::UnknownVfioDeviceId(id)) + } + } + #[cfg(feature = "pci_support")] pub fn eject_device(&mut self, device_id: u8) -> DeviceManagerResult<()> { // Retrieve the PCI bus. diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 02bdc0e59..0b2fc73b6 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -385,6 +385,19 @@ impl Vmm { } } + fn vm_remove_device(&mut self, id: String) -> result::Result<(), VmError> { + if let Some(ref mut vm) = self.vm { + if let Err(e) = vm.remove_device(id) { + error!("Error when adding new device to the VM: {:?}", e); + Err(e) + } else { + Ok(()) + } + } else { + Err(VmError::VmNotRunning) + } + } + fn control_loop(&mut self, api_receiver: Arc>) -> Result<()> { const EPOLL_EVENTS_LEN: usize = 100; @@ -547,6 +560,13 @@ impl Vmm { .map(|_| ApiResponsePayload::Empty); sender.send(response).map_err(Error::ApiResponseSend)?; } + ApiRequest::VmRemoveDevice(remove_device_data, sender) => { + let response = self + .vm_remove_device(remove_device_data.id.clone()) + .map_err(ApiError::VmRemoveDevice) + .map(|_| ApiResponsePayload::Empty); + sender.send(response).map_err(Error::ApiResponseSend)?; + } } } } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 7d7daea59..a4898df18 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -577,6 +577,28 @@ impl Vm { } } + pub fn remove_device(&mut self, _id: String) -> Result<()> { + if cfg!(feature = "pci_support") { + #[cfg(feature = "pci_support")] + { + self.device_manager + .lock() + .unwrap() + .remove_device(_id) + .map_err(Error::DeviceManager)?; + + self.device_manager + .lock() + .unwrap() + .notify_hotplug(HotPlugNotificationFlags::PCI_DEVICES_CHANGED) + .map_err(Error::DeviceManager)?; + } + Ok(()) + } else { + Err(Error::NoPciSupport) + } + } + fn os_signal_handler(signals: Signals, console_input_clone: Arc, on_tty: bool) { for signal in signals.forever() { match signal {