vmm: Implement VM pause

In order to pause a VM, we signal all the vCPU threads to get them out
of vmx non-root. Once out, the vCPU thread will check for a an atomic
pause boolean. If it's set to true, then the thread will park until
being resumed.

Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>

vmm/src/api/http.rs

@@ -58,6 +58,7 @@ lazy_static! {
         r.routes.insert(endpoint!("/vm.boot"), Box::new(VmActionHandler::new(VmAction::Boot)));
         r.routes.insert(endpoint!("/vm.delete"), Box::new(VmActionHandler::new(VmAction::Delete)));
         r.routes.insert(endpoint!("/vm.info"), Box::new(VmInfo {}));
+        r.routes.insert(endpoint!("/vm.pause"), Box::new(VmActionHandler::new(VmAction::Pause)));
         r.routes.insert(endpoint!("/vm.shutdown"), Box::new(VmActionHandler::new(VmAction::Shutdown)));
         r.routes.insert(endpoint!("/vm.reboot"), Box::new(VmActionHandler::new(VmAction::Reboot)));
         r.routes.insert(endpoint!("/vmm.shutdown"), Box::new(VmmShutdown {}));

vmm/src/api/http_endpoint.rs

@@ -5,8 +5,8 @@
 
 use crate::api::http::EndpointHandler;
 use crate::api::{
-    vm_boot, vm_create, vm_delete, vm_info, vm_reboot, vm_shutdown, vmm_shutdown, ApiError,
-    ApiRequest, ApiResult, VmAction, VmConfig,
+    vm_boot, vm_create, vm_delete, vm_info, vm_pause, vm_reboot, vm_shutdown, vmm_shutdown,
+    ApiError, ApiRequest, ApiResult, VmAction, VmConfig,
 };
 use micro_http::{Body, Method, Request, Response, StatusCode, Version};
 use serde_json::Error as SerdeError;
@@ -29,6 +29,9 @@ pub enum HttpError {
     /// Could not get the VM information
     VmInfo(ApiError),
 
+    /// Could not pause the VM
+    VmPause(ApiError),
+
     /// Could not shut a VM down
     VmShutdown(ApiError),
 
@@ -103,6 +106,7 @@ impl VmActionHandler {
             VmAction::Delete => vm_delete,
             VmAction::Shutdown => vm_shutdown,
             VmAction::Reboot => vm_reboot,
+            VmAction::Pause => vm_pause,
         });
 
         VmActionHandler { action_fn }
@@ -122,6 +126,7 @@ impl EndpointHandler for VmActionHandler {
                     ApiError::VmBoot(_) => HttpError::VmBoot(e),
                     ApiError::VmShutdown(_) => HttpError::VmShutdown(e),
                     ApiError::VmReboot(_) => HttpError::VmReboot(e),
+                    ApiError::VmPause(_) => HttpError::VmPause(e),
                     _ => HttpError::VmAction(e),
                 }) {
                     Ok(_) => Response::new(Version::Http11, StatusCode::OK),

vmm/src/api/mod.rs

@@ -76,6 +76,9 @@ pub enum ApiError {
     /// The VM config is missing.
     VmMissingConfig,
 
+    /// The VM could not be paused.
+    VmPause(VmError),
+
     /// The VM is not booted.
     VmNotBooted,
 
@@ -132,6 +135,9 @@ pub enum ApiRequest {
     /// Request the VM information.
     VmInfo(Sender<ApiResponse>),
 
+    /// Pause a VM.
+    VmPause(Sender<ApiResponse>),
+
     /// Shut the previously booted virtual machine down.
     /// If the VM was not previously booted or created, the VMM API server
     /// will send a VmShutdown error back.
@@ -181,6 +187,9 @@ pub enum VmAction {
 
     /// Reboot a VM
     Reboot,
+
+    /// Pause a VM
+    Pause,
 }
 
 fn vm_action(api_evt: EventFd, api_sender: Sender<ApiRequest>, action: VmAction) -> ApiResult<()> {
@@ -191,6 +200,7 @@ fn vm_action(api_evt: EventFd, api_sender: Sender<ApiRequest>, action: VmAction)
         VmAction::Delete => ApiRequest::VmDelete(response_sender),
         VmAction::Shutdown => ApiRequest::VmShutdown(response_sender),
         VmAction::Reboot => ApiRequest::VmReboot(response_sender),
+        VmAction::Pause => ApiRequest::VmPause(response_sender),
     };
 
     // Send the VM request.
@@ -218,6 +228,10 @@ pub fn vm_reboot(api_evt: EventFd, api_sender: Sender<ApiRequest>) -> ApiResult<
     vm_action(api_evt, api_sender, VmAction::Reboot)
 }
 
+pub fn vm_pause(api_evt: EventFd, api_sender: Sender<ApiRequest>) -> ApiResult<()> {
+    vm_action(api_evt, api_sender, VmAction::Pause)
+}
+
 pub fn vm_info(api_evt: EventFd, api_sender: Sender<ApiRequest>) -> ApiResult<VmInfo> {
     let (response_sender, response_receiver) = channel();
 

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -80,6 +80,21 @@ paths:
           description: The VM instance could not boot because it is not created yet
           content: {}
 
+  /vm.pause:
+    put:
+      summary: Pause a previously booted VM instance.
+      operationId: pauseVM
+      responses:
+        201:
+          description: The VM instance successfully paused.
+          content: {}
+        404:
+          description: The VM instance could not pause because it is not created yet
+          content: {}
+        405:
+          description: The VM instance could not pause because it is not booted.
+          content: {}
+
   /vm.shutdown:
     put:
       summary: Shut the VM instance down.

vmm/src/lib.rs

@@ -227,6 +227,14 @@ impl Vmm {
         }
     }
 
+    fn vm_pause(&mut self) -> result::Result<(), VmError> {
+        if let Some(ref mut vm) = self.vm {
+            vm.pause()
+        } else {
+            Err(VmError::VmNotBooted)
+        }
+    }
+
     fn vm_shutdown(&mut self) -> result::Result<(), VmError> {
         if let Some(ref mut vm) = self.vm.take() {
             vm.shutdown()
@@ -414,6 +422,14 @@ impl Vmm {
 
                                     sender.send(response).map_err(Error::ApiResponseSend)?;
                                 }
+                                ApiRequest::VmPause(sender) => {
+                                    let response = self
+                                        .vm_pause()
+                                        .map_err(ApiError::VmPause)
+                                        .map(|_| ApiResponsePayload::Empty);
+
+                                    sender.send(response).map_err(Error::ApiResponseSend)?;
+                                }
                                 ApiRequest::VmmShutdown(sender) => {
                                     let response = self
                                         .vmm_shutdown()

vmm/src/vm.rs

@@ -441,6 +441,7 @@ pub enum VmState {
     Created,
     Booted,
     Shutdown,
+    Paused,
 }
 
 pub struct Vm {
@@ -454,6 +455,7 @@ pub struct Vm {
     on_tty: bool,
     creation_ts: std::time::Instant,
     vcpus_kill_signalled: Arc<AtomicBool>,
+    vcpus_pause_signalled: Arc<AtomicBool>,
     // Reboot (reset) control
     reset_evt: EventFd,
     signals: Option<Signals>,
@@ -685,6 +687,7 @@ impl Vm {
             on_tty,
             creation_ts,
             vcpus_kill_signalled: Arc::new(AtomicBool::new(false)),
+            vcpus_pause_signalled: Arc::new(AtomicBool::new(false)),
             reset_evt,
             signals: None,
             state: RwLock::new(VmState::Created),
@@ -808,6 +811,26 @@ impl Vm {
         Ok(())
     }
 
+    pub fn pause(&mut self) -> Result<()> {
+        // Tell the vCPUs to pause themselves next time they exit
+        self.vcpus_pause_signalled.store(true, Ordering::SeqCst);
+
+        // Signal to the spawned threads (vCPUs and console signal handler). For the vCPU threads
+        // this will interrupt the KVM_RUN ioctl() allowing the loop to check the boolean set
+        // above. The signal handler thread will ignore this signal
+        for thread in self.threads.iter() {
+            let signum = validate_signal_num(VCPU_RTSIG_OFFSET, true).unwrap();
+            unsafe {
+                libc::pthread_kill(thread.as_pthread_t(), signum);
+            }
+        }
+
+        let mut state = self.state.try_write().map_err(|_| Error::PoisonedState)?;
+        *state = VmState::Paused;
+
+        Ok(())
+    }
+
     fn os_signal_handler(signals: Signals, console_input_clone: Arc<Console>) {
         for signal in signals.forever() {
             if signal == SIGWINCH {
@@ -838,6 +861,7 @@ impl Vm {
 
             let reset_evt = self.reset_evt.try_clone().unwrap();
             let vcpu_kill_signalled = self.vcpus_kill_signalled.clone();
+            let vcpu_pause_signalled = self.vcpus_pause_signalled.clone();
             self.threads.push(
                 thread::Builder::new()
                     .name(format!("vcpu{}", vcpu.id))
@@ -876,6 +900,17 @@ impl Vm {
                             if vcpu_kill_signalled.load(Ordering::SeqCst) {
                                 break;
                             }
+
+                            // If we are being told to pause, we park the thread
+                            // until the pause boolean is toggled.
+                            // The resume operation is responsible for toggling
+                            // the boolean and unpark the thread.
+                            // We enter a loop because park() could spuriously
+                            // return. We will then park() again unless the
+                            // pause boolean has been toggled.
+                            while vcpu_pause_signalled.load(Ordering::SeqCst) {
+                                thread::park();
+                            }
                         }
                     })
                     .map_err(Error::VcpuSpawn)?,