vmm: Shutdown VMM if API thread panics

See: #3031

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

vmm/src/api/http.rs

@@ -14,6 +14,7 @@ use std::collections::HashMap;
 use std::fs::File;
 use std::os::unix::io::{IntoRawFd, RawFd};
 use std::os::unix::net::UnixListener;
+use std::panic::AssertUnwindSafe;
 use std::path::PathBuf;
 use std::sync::mpsc::Sender;
 use std::sync::Arc;
@@ -267,6 +268,7 @@ fn start_http_thread(
     api_notifier: EventFd,
     api_sender: Sender<ApiRequest>,
     seccomp_action: &SeccompAction,
+    exit_evt: EventFd,
 ) -> Result<thread::JoinHandle<Result<()>>> {
     // Retrieve seccomp filter for API thread
     let api_seccomp_filter =
@@ -277,32 +279,44 @@ fn start_http_thread(
         .spawn(move || {
             // Apply seccomp filter for API thread.
             if !api_seccomp_filter.is_empty() {
-                apply_filter(&api_seccomp_filter).map_err(Error::ApplySeccompFilter)?;
+                apply_filter(&api_seccomp_filter)
+                    .map_err(Error::ApplySeccompFilter)
+                    .map_err(|e| {
+                        error!("Error applying seccomp filter: {:?}", e);
+                        exit_evt.write(1).ok();
+                        e
+                    })?;
             }
 
-            server.start_server().unwrap();
+            std::panic::catch_unwind(AssertUnwindSafe(move || {
-            loop {
+                server.start_server().unwrap();
-                match server.requests() {
+                loop {
-                    Ok(request_vec) => {
+                    match server.requests() {
-                        for server_request in request_vec {
+                        Ok(request_vec) => {
-                            server
+                            for server_request in request_vec {
-                                .respond(server_request.process(|request| {
+                                if let Err(e) = server.respond(server_request.process(|request| {
                                     handle_http_request(request, &api_notifier, &api_sender)
-                                }))
+                                })) {
-                                .or_else(|e| {
                                     error!("HTTP server error on response: {}", e);
-                                    Ok(())
+                                }
-                                })?;
+                            }
+                        }
+                        Err(e) => {
+                            error!(
+                                "HTTP server error on retrieving incoming request. Error: {}",
+                                e
+                            );
                         }
                     }
-                    Err(e) => {
-                        error!(
-                            "HTTP server error on retrieving incoming request. Error: {}",
-                            e
-                        );
-                    }
                 }
-            }
+            }))
+            .map_err(|_| {
+                error!("http-server thread panicked");
+                exit_evt.write(1).ok()
+            })
+            .ok();
+
+            Ok(())
         })
         .map_err(Error::HttpThreadSpawn)
 }
@@ -312,13 +326,14 @@ pub fn start_http_path_thread(
     api_notifier: EventFd,
     api_sender: Sender<ApiRequest>,
     seccomp_action: &SeccompAction,
+    exit_evt: EventFd,
 ) -> Result<thread::JoinHandle<Result<()>>> {
     std::fs::remove_file(path).unwrap_or_default();
     let socket_path = PathBuf::from(path);
     let socket_fd = UnixListener::bind(socket_path).map_err(Error::CreateApiServerSocket)?;
     let server =
         HttpServer::new_from_fd(socket_fd.into_raw_fd()).map_err(Error::CreateApiServer)?;
-    start_http_thread(server, api_notifier, api_sender, seccomp_action)
+    start_http_thread(server, api_notifier, api_sender, seccomp_action, exit_evt)
 }
 
 pub fn start_http_fd_thread(
@@ -326,7 +341,8 @@ pub fn start_http_fd_thread(
     api_notifier: EventFd,
     api_sender: Sender<ApiRequest>,
     seccomp_action: &SeccompAction,
+    exit_evt: EventFd,
 ) -> Result<thread::JoinHandle<Result<()>>> {
     let server = HttpServer::new_from_fd(fd).map_err(Error::CreateApiServer)?;
-    start_http_thread(server, api_notifier, api_sender, seccomp_action)
+    start_http_thread(server, api_notifier, api_sender, seccomp_action, exit_evt)
 }

vmm/src/lib.rs

@@ -262,30 +262,47 @@ pub fn start_vmm_thread(
         get_seccomp_filter(seccomp_action, Thread::Vmm).map_err(Error::CreateSeccompFilter)?;
 
     let vmm_seccomp_action = seccomp_action.clone();
-    let thread = thread::Builder::new()
+    let exit_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?;
-        .name("vmm".to_string())
+    let thread = {
-        .spawn(move || {
+        let exit_evt = exit_evt.try_clone().map_err(Error::EventFdClone)?;
-            // Apply seccomp filter for VMM thread.
+        thread::Builder::new()
-            if !vmm_seccomp_filter.is_empty() {
+            .name("vmm".to_string())
-                apply_filter(&vmm_seccomp_filter).map_err(Error::ApplySeccompFilter)?;
+            .spawn(move || {
-            }
+                // Apply seccomp filter for VMM thread.
+                if !vmm_seccomp_filter.is_empty() {
+                    apply_filter(&vmm_seccomp_filter).map_err(Error::ApplySeccompFilter)?;
+                }
 
-            let mut vmm = Vmm::new(
+                let mut vmm = Vmm::new(
-                vmm_version.to_string(),
+                    vmm_version.to_string(),
-                api_event,
+                    api_event,
-                vmm_seccomp_action,
+                    vmm_seccomp_action,
-                hypervisor,
+                    hypervisor,
-            )?;
+                    exit_evt,
+                )?;
 
-            vmm.control_loop(Arc::new(api_receiver))
+                vmm.control_loop(Arc::new(api_receiver))
-        })
+            })
-        .map_err(Error::VmmThreadSpawn)?;
+            .map_err(Error::VmmThreadSpawn)?
+    };
 
     // The VMM thread is started, we can start serving HTTP requests
     if let Some(http_path) = http_path {
-        api::start_http_path_thread(http_path, http_api_event, api_sender, seccomp_action)?;
+        api::start_http_path_thread(
+            http_path,
+            http_api_event,
+            api_sender,
+            seccomp_action,
+            exit_evt,
+        )?;
     } else if let Some(http_fd) = http_fd {
-        api::start_http_fd_thread(http_fd, http_api_event, api_sender, seccomp_action)?;
+        api::start_http_fd_thread(
+            http_fd,
+            http_api_event,
+            api_sender,
+            seccomp_action,
+            exit_evt,
+        )?;
     }
     Ok(thread)
 }
@@ -316,9 +333,9 @@ impl Vmm {
         api_evt: EventFd,
         seccomp_action: SeccompAction,
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
+        exit_evt: EventFd,
     ) -> Result<Self> {
         let mut epoll = EpollContext::new().map_err(Error::Epoll)?;
-        let exit_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?;
         let reset_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?;
         let activate_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?;