vmm, virtio-devices: Restore vhost-user devices in a dedicated way

We cannot let vhost-user devices connect to the backend when the Block,
Fs or Net object is being created during a restore/migration. The reason
is we can't have two VMs (source and destination) connected to the same
backend at the same time. That's why we must delay the connection with
the vhost-user backend until the restoration is performed.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

virtio-devices/src/vhost_user/blk.rs

@@ -43,6 +43,8 @@ pub struct State {
     pub avail_features: u64,
     pub acked_features: u64,
     pub config: VirtioBlockConfig,
+    pub acked_protocol_features: u64,
+    pub vu_num_queues: usize,
 }
 
 impl VersionMapped for State {}
@@ -65,9 +67,33 @@ pub struct Blk {
 
 impl Blk {
     /// Create a new vhost-user-blk device
-    pub fn new(id: String, vu_cfg: VhostUserConfig) -> Result<Blk> {
+    pub fn new(id: String, vu_cfg: VhostUserConfig, restoring: bool) -> Result<Blk> {
         let num_queues = vu_cfg.num_queues;
 
+        if restoring {
+            // We need 'queue_sizes' to report a number of queues that will be
+            // enough to handle all the potential queues. VirtioPciDevice::new()
+            // will create the actual queues based on this information.
+            return Ok(Blk {
+                id,
+                common: VirtioCommon {
+                    device_type: VirtioDeviceType::Block as u32,
+                    queue_sizes: vec![vu_cfg.queue_size; num_queues],
+                    paused_sync: Some(Arc::new(Barrier::new(2))),
+                    min_queues: DEFAULT_QUEUE_NUMBER as u16,
+                    ..Default::default()
+                },
+                vu: None,
+                config: VirtioBlockConfig::default(),
+                guest_memory: None,
+                acked_protocol_features: 0,
+                socket_path: vu_cfg.socket,
+                epoll_thread: None,
+                vu_num_queues: num_queues,
+                migration_started: false,
+            });
+        }
+
         let mut vu =
             VhostUserHandle::connect_vhost_user(false, &vu_cfg.socket, num_queues as u64, false)?;
 
@@ -157,6 +183,8 @@ impl Blk {
             avail_features: self.common.avail_features,
             acked_features: self.common.acked_features,
             config: self.config,
+            acked_protocol_features: self.acked_protocol_features,
+            vu_num_queues: self.vu_num_queues,
         }
     }
 
@@ -164,6 +192,31 @@ impl Blk {
         self.common.avail_features = state.avail_features;
         self.common.acked_features = state.acked_features;
         self.config = state.config;
+        self.acked_protocol_features = state.acked_protocol_features;
+        self.vu_num_queues = state.vu_num_queues;
+
+        let mut vu = match VhostUserHandle::connect_vhost_user(
+            false,
+            &self.socket_path,
+            self.vu_num_queues as u64,
+            false,
+        ) {
+            Ok(r) => r,
+            Err(e) => {
+                error!("Failed connecting vhost-user backend: {:?}", e);
+                return;
+            }
+        };
+
+        if let Err(e) = vu.set_protocol_features_vhost_user(
+            self.common.acked_features,
+            self.acked_protocol_features,
+        ) {
+            error!("Failed setting up vhost-user backend: {:?}", e);
+            return;
+        }
+
+        self.vu = Some(Arc::new(Mutex::new(vu)));
     }
 }
 

virtio-devices/src/vhost_user/fs.rs

@@ -45,6 +45,9 @@ pub struct State {
     pub avail_features: u64,
     pub acked_features: u64,
     pub config: VirtioFsConfig,
+    pub acked_protocol_features: u64,
+    pub vu_num_queues: usize,
+    pub slave_req_support: bool,
 }
 
 impl VersionMapped for State {}
@@ -315,6 +318,7 @@ pub struct Fs {
 
 impl Fs {
     /// Create a new virtio-fs device.
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         id: String,
         path: &str,
@@ -323,12 +327,40 @@ impl Fs {
         queue_size: u16,
         cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
         seccomp_action: SeccompAction,
+        restoring: bool,
     ) -> Result<Fs> {
         let mut slave_req_support = false;
 
         // Calculate the actual number of queues needed.
         let num_queues = NUM_QUEUE_OFFSET + req_num_queues;
 
+        if restoring {
+            // We need 'queue_sizes' to report a number of queues that will be
+            // enough to handle all the potential queues. VirtioPciDevice::new()
+            // will create the actual queues based on this information.
+            return Ok(Fs {
+                id,
+                common: VirtioCommon {
+                    device_type: VirtioDeviceType::Fs as u32,
+                    queue_sizes: vec![queue_size; num_queues],
+                    paused_sync: Some(Arc::new(Barrier::new(2))),
+                    min_queues: DEFAULT_QUEUE_NUMBER as u16,
+                    ..Default::default()
+                },
+                vu: None,
+                config: VirtioFsConfig::default(),
+                cache,
+                slave_req_support,
+                seccomp_action,
+                guest_memory: None,
+                acked_protocol_features: 0,
+                socket_path: path.to_string(),
+                epoll_thread: None,
+                vu_num_queues: num_queues,
+                migration_started: false,
+            });
+        }
+
         // Connect to the vhost-user socket.
         let mut vu = VhostUserHandle::connect_vhost_user(false, path, num_queues as u64, false)?;
 
@@ -408,6 +440,9 @@ impl Fs {
             avail_features: self.common.avail_features,
             acked_features: self.common.acked_features,
             config: self.config,
+            acked_protocol_features: self.acked_protocol_features,
+            vu_num_queues: self.vu_num_queues,
+            slave_req_support: self.slave_req_support,
         }
     }
 
@@ -415,6 +450,32 @@ impl Fs {
         self.common.avail_features = state.avail_features;
         self.common.acked_features = state.acked_features;
         self.config = state.config;
+        self.acked_protocol_features = state.acked_protocol_features;
+        self.vu_num_queues = state.vu_num_queues;
+        self.slave_req_support = state.slave_req_support;
+
+        let mut vu = match VhostUserHandle::connect_vhost_user(
+            false,
+            &self.socket_path,
+            self.vu_num_queues as u64,
+            false,
+        ) {
+            Ok(r) => r,
+            Err(e) => {
+                error!("Failed connecting vhost-user backend: {:?}", e);
+                return;
+            }
+        };
+
+        if let Err(e) = vu.set_protocol_features_vhost_user(
+            self.common.acked_features,
+            self.acked_protocol_features,
+        ) {
+            error!("Failed setting up vhost-user backend: {:?}", e);
+            return;
+        }
+
+        self.vu = Some(Arc::new(Mutex::new(vu)));
     }
 }
 

virtio-devices/src/vhost_user/net.rs

@@ -44,6 +44,8 @@ pub struct State {
     pub avail_features: u64,
     pub acked_features: u64,
     pub config: VirtioNetConfig,
+    pub acked_protocol_features: u64,
+    pub vu_num_queues: usize,
 }
 
 impl VersionMapped for State {}
@@ -127,9 +129,38 @@ impl Net {
         vu_cfg: VhostUserConfig,
         server: bool,
         seccomp_action: SeccompAction,
+        restoring: bool,
     ) -> Result<Net> {
         let mut num_queues = vu_cfg.num_queues;
 
+        if restoring {
+            // We need 'queue_sizes' to report a number of queues that will be
+            // enough to handle all the potential queues. Including the control
+            // queue (with +1) will guarantee that. VirtioPciDevice::new() will
+            // create the actual queues based on this information.
+            return Ok(Net {
+                id,
+                common: VirtioCommon {
+                    device_type: VirtioDeviceType::Net as u32,
+                    queue_sizes: vec![vu_cfg.queue_size; num_queues + 1],
+                    paused_sync: Some(Arc::new(Barrier::new(2))),
+                    min_queues: DEFAULT_QUEUE_NUMBER as u16,
+                    ..Default::default()
+                },
+                vu: None,
+                config: VirtioNetConfig::default(),
+                guest_memory: None,
+                acked_protocol_features: 0,
+                socket_path: vu_cfg.socket,
+                server,
+                ctrl_queue_epoll_thread: None,
+                epoll_thread: None,
+                seccomp_action,
+                vu_num_queues: num_queues,
+                migration_started: false,
+            });
+        }
+
         // Filling device and vring features VMM supports.
         let mut avail_features = 1 << VIRTIO_NET_F_CSUM
             | 1 << VIRTIO_NET_F_GUEST_CSUM
@@ -218,6 +249,8 @@ impl Net {
             avail_features: self.common.avail_features,
             acked_features: self.common.acked_features,
             config: self.config,
+            acked_protocol_features: self.acked_protocol_features,
+            vu_num_queues: self.vu_num_queues,
         }
     }
 
@@ -225,6 +258,31 @@ impl Net {
         self.common.avail_features = state.avail_features;
         self.common.acked_features = state.acked_features;
         self.config = state.config;
+        self.acked_protocol_features = state.acked_protocol_features;
+        self.vu_num_queues = state.vu_num_queues;
+
+        let mut vu = match VhostUserHandle::connect_vhost_user(
+            self.server,
+            &self.socket_path,
+            self.vu_num_queues as u64,
+            false,
+        ) {
+            Ok(r) => r,
+            Err(e) => {
+                error!("Failed connecting vhost-user backend: {:?}", e);
+                return;
+            }
+        };
+
+        if let Err(e) = vu.set_protocol_features_vhost_user(
+            self.common.acked_features,
+            self.acked_protocol_features,
+        ) {
+            error!("Failed setting up vhost-user backend: {:?}", e);
+            return;
+        }
+
+        self.vu = Some(Arc::new(Mutex::new(vu)));
     }
 }
 

vmm/src/device_manager.rs

@@ -1864,7 +1864,7 @@ impl DeviceManager {
                 queue_size: disk_cfg.queue_size,
             };
             let vhost_user_block_device = Arc::new(Mutex::new(
-                match virtio_devices::vhost_user::Blk::new(id.clone(), vu_cfg) {
+                match virtio_devices::vhost_user::Blk::new(id.clone(), vu_cfg, self.restoring) {
                     Ok(vub_device) => vub_device,
                     Err(e) => {
                         return Err(DeviceManagerError::CreateVhostUserBlk(e));
@@ -2021,6 +2021,7 @@ impl DeviceManager {
                     vu_cfg,
                     server,
                     self.seccomp_action.clone(),
+                    self.restoring,
                 ) {
                     Ok(vun_device) => vun_device,
                     Err(e) => {
@@ -2288,6 +2289,7 @@ impl DeviceManager {
                     fs_cfg.queue_size,
                     cache,
                     self.seccomp_action.clone(),
+                    self.restoring,
                 )
                 .map_err(DeviceManagerError::CreateVirtioFs)?,
             ));