vm-virtio: Expand vhost-user handler to be reused from virtio-fs

In order to factorize the code between vhost-user-net and virtio-fs one
step further, this patch extends the vhost-user handler implementation
to support slave requests.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

vm-virtio/src/vhost_user/fs.rs

@@ -3,17 +3,16 @@
 
 use super::vu_common_ctrl::setup_vhost_user;
 use super::{Error, Result};
+use crate::vhost_user::handler::{VhostUserEpollConfig, VhostUserEpollHandler};
 use crate::{
     ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType, VirtioInterrupt,
-    VirtioInterruptType, VirtioSharedMemoryList, VIRTIO_F_VERSION_1,
+    VirtioSharedMemoryList, VIRTIO_F_VERSION_1,
 };
-use epoll;
 use libc::{self, EFD_NONBLOCK};
 use std::cmp;
 use std::io;
 use std::io::Write;
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::io::RawFd;
-use std::result;
 use std::sync::{Arc, Mutex, RwLock};
 use std::thread;
 use vhost_rs::vhost_user::message::{
@@ -110,115 +109,6 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
     }
 }
 
-struct FsEpollHandler<S: VhostUserMasterReqHandler> {
-    vu_call_evt_queue_list: Vec<(EventFd, Queue)>,
-    interrupt_cb: Arc<VirtioInterrupt>,
-    kill_evt: EventFd,
-    slave_req_handler: Option<MasterReqHandler<S>>,
-}
-
-impl<S: VhostUserMasterReqHandler> FsEpollHandler<S> {
-    fn run(&mut self) -> result::Result<(), Error> {
-        // Create the epoll file descriptor
-        let epoll_fd = epoll::create(true).map_err(Error::EpollCreateFd)?;
-
-        for (evt_index, vu_call_evt_queue) in self.vu_call_evt_queue_list.iter().enumerate() {
-            // Add events
-            epoll::ctl(
-                epoll_fd,
-                epoll::ControlOptions::EPOLL_CTL_ADD,
-                vu_call_evt_queue.0.as_raw_fd(),
-                epoll::Event::new(epoll::Events::EPOLLIN, evt_index as u64),
-            )
-            .map_err(Error::EpollCtl)?;
-        }
-
-        let kill_evt_index = self.vu_call_evt_queue_list.len();
-        epoll::ctl(
-            epoll_fd,
-            epoll::ControlOptions::EPOLL_CTL_ADD,
-            self.kill_evt.as_raw_fd(),
-            epoll::Event::new(epoll::Events::EPOLLIN, kill_evt_index as u64),
-        )
-        .map_err(Error::EpollCtl)?;
-
-        let slave_evt_index = if let Some(self_req_handler) = &self.slave_req_handler {
-            let index = kill_evt_index + 1;
-            epoll::ctl(
-                epoll_fd,
-                epoll::ControlOptions::EPOLL_CTL_ADD,
-                self_req_handler.as_raw_fd(),
-                epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
-            )
-            .map_err(Error::EpollCtl)?;
-
-            Some(index)
-        } else {
-            None
-        };
-
-        const EPOLL_EVENTS_LEN: usize = 100;
-        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
-
-        'epoll: loop {
-            let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
-                Ok(res) => res,
-                Err(e) => {
-                    if e.kind() == io::ErrorKind::Interrupted {
-                        // It's well defined from the epoll_wait() syscall
-                        // documentation that the epoll loop can be interrupted
-                        // before any of the requested events occurred or the
-                        // timeout expired. In both those cases, epoll_wait()
-                        // returns an error of type EINTR, but this should not
-                        // be considered as a regular error. Instead it is more
-                        // appropriate to retry, by calling into epoll_wait().
-                        continue;
-                    }
-                    return Err(Error::EpollWait(e));
-                }
-            };
-
-            for event in events.iter().take(num_events) {
-                let ev_type = event.data as usize;
-
-                match ev_type {
-                    x if (x < kill_evt_index) => {
-                        if let Err(e) = self.vu_call_evt_queue_list[x].0.read() {
-                            error!("Failed to get queue event: {:?}", e);
-                            break 'epoll;
-                        } else if let Err(e) = (self.interrupt_cb)(
-                            &VirtioInterruptType::Queue,
-                            Some(&self.vu_call_evt_queue_list[x].1),
-                        ) {
-                            error!(
-                                "Failed to signal used queue: {:?}",
-                                Error::FailedSignalingUsedQueue(e)
-                            );
-                            break 'epoll;
-                        }
-                    }
-                    x if (x == kill_evt_index) => {
-                        debug!("KILL_EVENT received, stopping epoll loop");
-                        break 'epoll;
-                    }
-                    x if (slave_evt_index.is_some() && slave_evt_index.unwrap() == x) => {
-                        if let Some(slave_req_handler) = self.slave_req_handler.as_mut() {
-                            slave_req_handler
-                                .handle_request()
-                                .map_err(Error::VhostUserSlaveRequest)?;
-                        }
-                    }
-                    _ => {
-                        error!("Unknown event for virtio-fs");
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }
-}
-
 pub struct Fs {
     vu: Master,
     queue_sizes: Vec<u16>,
@@ -448,19 +338,23 @@ impl VirtioDevice for Fs {
             None
         };
 
-        let mut handler = FsEpollHandler {
+        let mut handler = VhostUserEpollHandler::new(VhostUserEpollConfig {
-            vu_call_evt_queue_list,
+            vu_interrupt_list: vu_call_evt_queue_list,
             interrupt_cb,
             kill_evt,
             slave_req_handler,
-        };
+        });
 
         let worker_result = thread::Builder::new()
             .name("virtio_fs".to_string())
-            .spawn(move || handler.run());
+            .spawn(move || {
+                if let Err(e) = handler.run() {
+                    error!("net worker thread exited with error {:?}!", e);
+                }
+            });
 
         if let Err(e) = worker_result {
-            error!("failed to spawn virtio_blk worker: {}", e);
+            error!("failed to spawn virtio-fs worker: {}", e);
             return Err(ActivateError::BadActivate);
         }
 

vm-virtio/src/vhost_user/handler.rs

@@ -16,6 +16,7 @@ use crate::VirtioInterrupt;
 use std::io;
 use std::os::unix::io::AsRawFd;
 use std::sync::Arc;
+use vhost_rs::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler};
 
 /// Collection of common parameters required by vhost-user devices while
 /// call Epoll handler.
@@ -24,17 +25,18 @@ use std::sync::Arc;
 /// * `interrupt_cb` interrupt for virtqueue change.
 /// * `kill_evt` - EventFd used to kill the vhost-user device.
 /// * `vu_interrupt_list` - virtqueue and EventFd to signal when buffer used.
-pub struct VhostUserEpollConfig {
+pub struct VhostUserEpollConfig<S: VhostUserMasterReqHandler> {
     pub interrupt_cb: Arc<VirtioInterrupt>,
     pub kill_evt: EventFd,
     pub vu_interrupt_list: Vec<(EventFd, Queue)>,
+    pub slave_req_handler: Option<MasterReqHandler<S>>,
 }
 
-pub struct VhostUserEpollHandler {
+pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> {
-    pub vu_epoll_cfg: VhostUserEpollConfig,
+    vu_epoll_cfg: VhostUserEpollConfig<S>,
 }
 
-impl VhostUserEpollHandler {
+impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
     /// Construct a new event handler for vhost-user based devices.
     ///
     /// # Arguments
@@ -42,21 +44,22 @@ impl VhostUserEpollHandler {
     ///
     /// # Return
     /// * `VhostUserEpollHandler` - epoll handler for vhost-user based devices
-    pub fn new(vu_epoll_cfg: VhostUserEpollConfig) -> VhostUserEpollHandler {
+    pub fn new(vu_epoll_cfg: VhostUserEpollConfig<S>) -> VhostUserEpollHandler<S> {
         VhostUserEpollHandler { vu_epoll_cfg }
     }
 
     fn signal_used_queue(&self, queue: &Queue) -> Result<()> {
         (self.vu_epoll_cfg.interrupt_cb)(&VirtioInterruptType::Queue, Some(queue))
-            .map_err(Error::FailedSignalingUsedQueue)?;
+            .map_err(Error::FailedSignalingUsedQueue)
-        Ok(())
     }
 
     pub fn run(&mut self) -> Result<()> {
+        // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(Error::EpollCreateFd)?;
 
         for (index, vhost_user_interrupt) in self.vu_epoll_cfg.vu_interrupt_list.iter().enumerate()
         {
+            // Add events
             epoll::ctl(
                 epoll_fd,
                 epoll::ControlOptions::EPOLL_CTL_ADD,
@@ -76,6 +79,21 @@ impl VhostUserEpollHandler {
         )
         .map_err(Error::EpollCtl)?;
 
+        let slave_evt_index = if let Some(self_req_handler) = &self.vu_epoll_cfg.slave_req_handler {
+            let index = kill_evt_index + 1;
+            epoll::ctl(
+                epoll_fd,
+                epoll::ControlOptions::EPOLL_CTL_ADD,
+                self_req_handler.as_raw_fd(),
+                epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
+            )
+            .map_err(Error::EpollCtl)?;
+
+            Some(index)
+        } else {
+            None
+        };
+
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); kill_evt_index + 1];
 
         'poll: loop {
@@ -101,21 +119,32 @@ impl VhostUserEpollHandler {
 
                 match ev_type {
                     x if x < kill_evt_index => {
-                        let vhost_user_interrupt = &self.vu_epoll_cfg.vu_interrupt_list[x].0;
+                        self.vu_epoll_cfg.vu_interrupt_list[x]
-                        vhost_user_interrupt
+                            .0
                             .read()
                             .map_err(Error::FailedReadingQueue)?;
-                        let result =
+                        if let Err(e) =
-                            self.signal_used_queue(&self.vu_epoll_cfg.vu_interrupt_list[x].1);
+                            self.signal_used_queue(&self.vu_epoll_cfg.vu_interrupt_list[x].1)
-                        if let Err(_e) = result {
+                        {
-                            error!("failed to signal used queue");
+                            error!("Failed to signal used queue: {:?}", e);
+                            break 'poll;
                         }
                     }
                     x if kill_evt_index == x => {
+                        debug!("KILL_EVENT received, stopping epoll loop");
                         break 'poll;
                     }
+                    x if (slave_evt_index.is_some() && slave_evt_index.unwrap() == x) => {
+                        if let Some(slave_req_handler) =
+                            self.vu_epoll_cfg.slave_req_handler.as_mut()
+                        {
+                            slave_req_handler
+                                .handle_request()
+                                .map_err(Error::VhostUserSlaveRequest)?;
+                        }
+                    }
                     _ => {
-                        error!("Unknown event for vhost-user-net");
+                        error!("Unknown event for vhost-user");
                     }
                 }
             }

vm-virtio/src/vhost_user/net.rs

@@ -20,11 +20,14 @@ use super::handler::*;
 use super::vu_common_ctrl::*;
 use super::{Error, Result};
 use vhost_rs::vhost_user::message::VhostUserVirtioFeatures;
-use vhost_rs::vhost_user::{Master, VhostUserMaster};
+use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
 use vhost_rs::VhostBackend;
 use virtio_bindings::virtio_net;
 use virtio_bindings::virtio_ring;
 
+struct SlaveReqHandler {}
+impl VhostUserMasterReqHandler for SlaveReqHandler {}
+
 pub struct Net {
     vhost_user_net: Master,
     kill_evt: EventFd,
@@ -200,23 +203,22 @@ impl VirtioDevice for Net {
         )
         .map_err(ActivateError::VhostUserNetSetup)?;
 
-        let vu_epoll_cfg = VhostUserEpollConfig {
+        let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
             interrupt_cb,
             kill_evt: handler_kill_evt,
             vu_interrupt_list,
-        };
+            slave_req_handler: None,
+        });
 
-        let _handler_result = thread::Builder::new()
+        let handler_result = thread::Builder::new()
             .name("vhost_user_net".to_string())
             .spawn(move || {
-                let mut handler = VhostUserEpollHandler::new(vu_epoll_cfg);
+                if let Err(e) = handler.run() {
-                let result = handler.run();
+                    error!("net worker thread exited with error {:?}!", e);
-                if let Err(_e) = result {
-                    error!("net worker thread exited with error {:?}!", _e);
                 }
             });
-        if let Err(_e) = _handler_result {
+        if let Err(e) = handler_result {
-            error!("vhost-user net thread create failed with error {:?}", _e);
+            error!("vhost-user net thread create failed with error {:?}", e);
         }
         Ok(())
     }