vmm, virtio-devices: Restore vhost-user devices in a dedicated way

We cannot let vhost-user devices connect to the backend when the Block,
Fs or Net object is being created during a restore/migration. The reason
is we can't have two VMs (source and destination) connected to the same
backend at the same time. That's why we must delay the connection with
the vhost-user backend until the restoration is performed.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
This commit is contained in:
Sebastien Boeuf 2021-08-09 14:02:30 +02:00 committed by Bo Chen
parent a636411522
commit 4735cb8563
4 changed files with 176 additions and 2 deletions

View File

@ -43,6 +43,8 @@ pub struct State {
pub avail_features: u64,
pub acked_features: u64,
pub config: VirtioBlockConfig,
pub acked_protocol_features: u64,
pub vu_num_queues: usize,
}
impl VersionMapped for State {}
@ -65,9 +67,33 @@ pub struct Blk {
impl Blk {
/// Create a new vhost-user-blk device
pub fn new(id: String, vu_cfg: VhostUserConfig) -> Result<Blk> {
pub fn new(id: String, vu_cfg: VhostUserConfig, restoring: bool) -> Result<Blk> {
let num_queues = vu_cfg.num_queues;
if restoring {
// We need 'queue_sizes' to report a number of queues that will be
// enough to handle all the potential queues. VirtioPciDevice::new()
// will create the actual queues based on this information.
return Ok(Blk {
id,
common: VirtioCommon {
device_type: VirtioDeviceType::Block as u32,
queue_sizes: vec![vu_cfg.queue_size; num_queues],
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: DEFAULT_QUEUE_NUMBER as u16,
..Default::default()
},
vu: None,
config: VirtioBlockConfig::default(),
guest_memory: None,
acked_protocol_features: 0,
socket_path: vu_cfg.socket,
epoll_thread: None,
vu_num_queues: num_queues,
migration_started: false,
});
}
let mut vu =
VhostUserHandle::connect_vhost_user(false, &vu_cfg.socket, num_queues as u64, false)?;
@ -157,6 +183,8 @@ impl Blk {
avail_features: self.common.avail_features,
acked_features: self.common.acked_features,
config: self.config,
acked_protocol_features: self.acked_protocol_features,
vu_num_queues: self.vu_num_queues,
}
}
@ -164,6 +192,31 @@ impl Blk {
self.common.avail_features = state.avail_features;
self.common.acked_features = state.acked_features;
self.config = state.config;
self.acked_protocol_features = state.acked_protocol_features;
self.vu_num_queues = state.vu_num_queues;
let mut vu = match VhostUserHandle::connect_vhost_user(
false,
&self.socket_path,
self.vu_num_queues as u64,
false,
) {
Ok(r) => r,
Err(e) => {
error!("Failed connecting vhost-user backend: {:?}", e);
return;
}
};
if let Err(e) = vu.set_protocol_features_vhost_user(
self.common.acked_features,
self.acked_protocol_features,
) {
error!("Failed setting up vhost-user backend: {:?}", e);
return;
}
self.vu = Some(Arc::new(Mutex::new(vu)));
}
}

View File

@ -45,6 +45,9 @@ pub struct State {
pub avail_features: u64,
pub acked_features: u64,
pub config: VirtioFsConfig,
pub acked_protocol_features: u64,
pub vu_num_queues: usize,
pub slave_req_support: bool,
}
impl VersionMapped for State {}
@ -315,6 +318,7 @@ pub struct Fs {
impl Fs {
/// Create a new virtio-fs device.
#[allow(clippy::too_many_arguments)]
pub fn new(
id: String,
path: &str,
@ -323,12 +327,40 @@ impl Fs {
queue_size: u16,
cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
seccomp_action: SeccompAction,
restoring: bool,
) -> Result<Fs> {
let mut slave_req_support = false;
// Calculate the actual number of queues needed.
let num_queues = NUM_QUEUE_OFFSET + req_num_queues;
if restoring {
// We need 'queue_sizes' to report a number of queues that will be
// enough to handle all the potential queues. VirtioPciDevice::new()
// will create the actual queues based on this information.
return Ok(Fs {
id,
common: VirtioCommon {
device_type: VirtioDeviceType::Fs as u32,
queue_sizes: vec![queue_size; num_queues],
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: DEFAULT_QUEUE_NUMBER as u16,
..Default::default()
},
vu: None,
config: VirtioFsConfig::default(),
cache,
slave_req_support,
seccomp_action,
guest_memory: None,
acked_protocol_features: 0,
socket_path: path.to_string(),
epoll_thread: None,
vu_num_queues: num_queues,
migration_started: false,
});
}
// Connect to the vhost-user socket.
let mut vu = VhostUserHandle::connect_vhost_user(false, path, num_queues as u64, false)?;
@ -408,6 +440,9 @@ impl Fs {
avail_features: self.common.avail_features,
acked_features: self.common.acked_features,
config: self.config,
acked_protocol_features: self.acked_protocol_features,
vu_num_queues: self.vu_num_queues,
slave_req_support: self.slave_req_support,
}
}
@ -415,6 +450,32 @@ impl Fs {
self.common.avail_features = state.avail_features;
self.common.acked_features = state.acked_features;
self.config = state.config;
self.acked_protocol_features = state.acked_protocol_features;
self.vu_num_queues = state.vu_num_queues;
self.slave_req_support = state.slave_req_support;
let mut vu = match VhostUserHandle::connect_vhost_user(
false,
&self.socket_path,
self.vu_num_queues as u64,
false,
) {
Ok(r) => r,
Err(e) => {
error!("Failed connecting vhost-user backend: {:?}", e);
return;
}
};
if let Err(e) = vu.set_protocol_features_vhost_user(
self.common.acked_features,
self.acked_protocol_features,
) {
error!("Failed setting up vhost-user backend: {:?}", e);
return;
}
self.vu = Some(Arc::new(Mutex::new(vu)));
}
}

View File

@ -44,6 +44,8 @@ pub struct State {
pub avail_features: u64,
pub acked_features: u64,
pub config: VirtioNetConfig,
pub acked_protocol_features: u64,
pub vu_num_queues: usize,
}
impl VersionMapped for State {}
@ -127,9 +129,38 @@ impl Net {
vu_cfg: VhostUserConfig,
server: bool,
seccomp_action: SeccompAction,
restoring: bool,
) -> Result<Net> {
let mut num_queues = vu_cfg.num_queues;
if restoring {
// We need 'queue_sizes' to report a number of queues that will be
// enough to handle all the potential queues. Including the control
// queue (with +1) will guarantee that. VirtioPciDevice::new() will
// create the actual queues based on this information.
return Ok(Net {
id,
common: VirtioCommon {
device_type: VirtioDeviceType::Net as u32,
queue_sizes: vec![vu_cfg.queue_size; num_queues + 1],
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: DEFAULT_QUEUE_NUMBER as u16,
..Default::default()
},
vu: None,
config: VirtioNetConfig::default(),
guest_memory: None,
acked_protocol_features: 0,
socket_path: vu_cfg.socket,
server,
ctrl_queue_epoll_thread: None,
epoll_thread: None,
seccomp_action,
vu_num_queues: num_queues,
migration_started: false,
});
}
// Filling device and vring features VMM supports.
let mut avail_features = 1 << VIRTIO_NET_F_CSUM
| 1 << VIRTIO_NET_F_GUEST_CSUM
@ -218,6 +249,8 @@ impl Net {
avail_features: self.common.avail_features,
acked_features: self.common.acked_features,
config: self.config,
acked_protocol_features: self.acked_protocol_features,
vu_num_queues: self.vu_num_queues,
}
}
@ -225,6 +258,31 @@ impl Net {
self.common.avail_features = state.avail_features;
self.common.acked_features = state.acked_features;
self.config = state.config;
self.acked_protocol_features = state.acked_protocol_features;
self.vu_num_queues = state.vu_num_queues;
let mut vu = match VhostUserHandle::connect_vhost_user(
self.server,
&self.socket_path,
self.vu_num_queues as u64,
false,
) {
Ok(r) => r,
Err(e) => {
error!("Failed connecting vhost-user backend: {:?}", e);
return;
}
};
if let Err(e) = vu.set_protocol_features_vhost_user(
self.common.acked_features,
self.acked_protocol_features,
) {
error!("Failed setting up vhost-user backend: {:?}", e);
return;
}
self.vu = Some(Arc::new(Mutex::new(vu)));
}
}

View File

@ -1864,7 +1864,7 @@ impl DeviceManager {
queue_size: disk_cfg.queue_size,
};
let vhost_user_block_device = Arc::new(Mutex::new(
match virtio_devices::vhost_user::Blk::new(id.clone(), vu_cfg) {
match virtio_devices::vhost_user::Blk::new(id.clone(), vu_cfg, self.restoring) {
Ok(vub_device) => vub_device,
Err(e) => {
return Err(DeviceManagerError::CreateVhostUserBlk(e));
@ -2021,6 +2021,7 @@ impl DeviceManager {
vu_cfg,
server,
self.seccomp_action.clone(),
self.restoring,
) {
Ok(vun_device) => vun_device,
Err(e) => {
@ -2288,6 +2289,7 @@ impl DeviceManager {
fs_cfg.queue_size,
cache,
self.seccomp_action.clone(),
self.restoring,
)
.map_err(DeviceManagerError::CreateVirtioFs)?,
));