External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2024-07-07 10:15:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

virtio-devices, vmm: Always restore virtio devices in paused state

Browse Source 
Following the new restore design, it is not appropriate to set every
virtio device threads into a paused state after they've been started.

This is why we remove the line of code pausing the devices only after
they've been restored, and replace it with a small patch in every virtio
device implementation. When a virtio device is created as part of a
restored VM, the associated "paused" boolean is set to true. This
ensures the corresponding thread will be directly parked when being
started, avoiding the thread to be in a different state than the one it
was on the source VM during the snapshot.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
This commit is contained in:
Sebastien Boeuf 2022-11-30 17:10:04 +01:00
parent 8f1e03fcf4
commit b62a40efae
15 changed files with 400 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
virtio-devices/src/balloon.rs
View File

@ -365,9 +365,14 @@ impl Balloon {
) -> io::Result<Self> {
let mut queue_sizes = vec![QUEUE_SIZE; MIN_NUM_QUEUES];
let (avail_features, acked_features, config) = if let Some(state) = state {
let (avail_features, acked_features, config, paused) = if let Some(state) = state {
info!("Restoring virtio-balloon {}", id);
(state.avail_features, state.acked_features, state.config)
(
state.avail_features,
state.acked_features,
state.config,
true,
)
} else {
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
if deflate_on_oom {
@ -382,7 +387,7 @@ impl Balloon {
..Default::default()
};
(avail_features, 0, config)
(avail_features, 0, config, false)
};
if free_page_reporting {
@ -397,6 +402,7 @@ impl Balloon {
paused_sync: Some(Arc::new(Barrier::new(2))),
queue_sizes,
min_queues: MIN_NUM_QUEUES as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

7
virtio-devices/src/block.rs
View File

@ -442,13 +442,15 @@ impl Block {
exit_evt: EventFd,
state: Option<BlockState>,
) -> io::Result<Self> {
let (disk_nsectors, avail_features, acked_features, config) = if let Some(state) = state {
let (disk_nsectors, avail_features, acked_features, config, paused) =
if let Some(state) = state {
info!("Restoring virtio-block {}", id);
(
state.disk_nsectors,
state.avail_features,
state.acked_features,
state.config,
true,
)
} else {
let disk_size = disk_image.size().map_err(|e| {
@ -512,7 +514,7 @@ impl Block {
config.num_queues = num_queues as u16;
}
(disk_nsectors, avail_features, 0, config)
(disk_nsectors, avail_features, 0, config, false)
};
Ok(Block {
@ -523,6 +525,7 @@ impl Block {
paused_sync: Some(Arc::new(Barrier::new(num_queues + 1))),
queue_sizes: vec![queue_size; num_queues],
min_queues: 1,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

6
virtio-devices/src/console.rs
View File

@ -628,13 +628,15 @@ impl Console {
exit_evt: EventFd,
state: Option<ConsoleState>,
) -> io::Result<(Console, Arc<ConsoleResizer>)> {
let (avail_features, acked_features, config, in_buffer) = if let Some(state) = state {
let (avail_features, acked_features, config, in_buffer, paused) = if let Some(state) = state
{
info!("Restoring virtio-console {}", id);
(
state.avail_features,
state.acked_features,
state.config,
state.in_buffer.into(),
true,
)
} else {
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_CONSOLE_F_SIZE;
@ -647,6 +649,7 @@ impl Console {
0,
VirtioConsoleConfig::default(),
VecDeque::new(),
false,
)
};
@ -670,6 +673,7 @@ impl Console {
acked_features,
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: NUM_QUEUES as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

7
virtio-devices/src/iommu.rs
View File

@ -879,7 +879,8 @@ impl Iommu {
msi_iova_space: (u64, u64),
state: Option<IommuState>,
) -> io::Result<(Self, Arc<IommuMapping>)> {
let (avail_features, acked_features, endpoints, domains) = if let Some(state) = state {
let (avail_features, acked_features, endpoints, domains, paused) =
if let Some(state) = state {
info!("Restoring virtio-iommu {}", id);
(
state.avail_features,
@ -898,6 +899,7 @@ impl Iommu {
)
})
.collect(),
true,
)
} else {
let avail_features = 1u64 << VIRTIO_F_VERSION_1
@ -905,7 +907,7 @@ impl Iommu {
| 1u64 << VIRTIO_IOMMU_F_PROBE
| 1u64 << VIRTIO_IOMMU_F_BYPASS_CONFIG;
(avail_features, 0, BTreeMap::new(), BTreeMap::new())
(avail_features, 0, BTreeMap::new(), BTreeMap::new(), false)
};
let config = VirtioIommuConfig {
@ -930,6 +932,7 @@ impl Iommu {
acked_features,
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: NUM_QUEUES as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
config,

12
virtio-devices/src/mem.rs
View File

@ -736,10 +736,15 @@ impl Mem {
));
}
let (avail_features, acked_features, config) = if let Some(state) = state {
let (avail_features, acked_features, config, paused) = if let Some(state) = state {
info!("Restoring virtio-mem {}", id);
*(blocks_state.lock().unwrap()) = state.blocks_state.clone();
(state.avail_features, state.acked_features, state.config)
(
state.avail_features,
state.acked_features,
state.config,
true,
)
} else {
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
@ -779,7 +784,7 @@ impl Mem {
)
})?;
(avail_features, 0, config)
(avail_features, 0, config, false)
};
let host_fd = region
@ -794,6 +799,7 @@ impl Mem {
paused_sync: Some(Arc::new(Barrier::new(2))),
queue_sizes: QUEUE_SIZES.to_vec(),
min_queues: 1,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

13
virtio-devices/src/net.rs
View File

@ -451,13 +451,15 @@ impl Net {
let mtu = taps[0].mtu().map_err(Error::TapError)? as u16;
let (avail_features, acked_features, config, queue_sizes) = if let Some(state) = state {
let (avail_features, acked_features, config, queue_sizes, paused) =
if let Some(state) = state {
info!("Restoring virtio-net {}", id);
(
state.avail_features,
state.acked_features,
state.config,
state.queue_size,
true,
)
} else {
let mut avail_features = 1 << VIRTIO_NET_F_CSUM
@ -500,7 +502,13 @@ impl Net {
);
}
(avail_features, 0, config, vec![queue_size; queue_num])
(
avail_features,
0,
config,
vec![queue_size; queue_num],
false,
)
};
Ok(Net {
@ -511,6 +519,7 @@ impl Net {
queue_sizes,
paused_sync: Some(Arc::new(Barrier::new((num_queues / 2) + 1))),
min_queues: 2,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

12
virtio-devices/src/pmem.rs
View File

@ -301,9 +301,14 @@ impl Pmem {
exit_evt: EventFd,
state: Option<PmemState>,
) -> io::Result<Pmem> {
let (avail_features, acked_features, config) = if let Some(state) = state {
let (avail_features, acked_features, config, paused) = if let Some(state) = state {
info!("Restoring virtio-pmem {}", id);
(state.avail_features, state.acked_features, state.config)
(
state.avail_features,
state.acked_features,
state.config,
true,
)
} else {
let config = VirtioPmemConfig {
start: addr.raw_value().to_le(),
@ -315,7 +320,7 @@ impl Pmem {
if iommu {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
(avail_features, 0, config)
(avail_features, 0, config, false)
};
Ok(Pmem {
@ -326,6 +331,7 @@ impl Pmem {
avail_features,
acked_features,
min_queues: 1,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

7
virtio-devices/src/rng.rs
View File

@ -178,9 +178,9 @@ impl Rng {
) -> io::Result<Rng> {
let random_file = File::open(path)?;
let (avail_features, acked_features) = if let Some(state) = state {
let (avail_features, acked_features, paused) = if let Some(state) = state {
info!("Restoring virtio-rng {}", id);
(state.avail_features, state.acked_features)
(state.avail_features, state.acked_features, true)
} else {
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
@ -188,7 +188,7 @@ impl Rng {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
(avail_features, 0)
(avail_features, 0, false)
};
Ok(Rng {
@ -199,6 +199,7 @@ impl Rng {
avail_features,
acked_features,
min_queues: 1,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

9
virtio-devices/src/vdpa.rs
View File

@ -12,7 +12,10 @@ use anyhow::anyhow;
use std::{
collections::BTreeMap,
io, result,
sync::{atomic::Ordering, Arc, Mutex},
sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex,
},
};
use thiserror::Error;
use versionize::{VersionMap, Versionize, VersionizeResult};
@ -134,6 +137,7 @@ impl Vdpa {
queue_sizes,
iova_range,
backend_features,
paused,
) = if let Some(state) = state {
info!("Restoring vDPA {}", id);
@ -152,6 +156,7 @@ impl Vdpa {
last: state.iova_range_last,
},
state.backend_features,
true,
)
} else {
let device_type = vhost.get_device_id().map_err(Error::GetDeviceId)?;
@ -175,6 +180,7 @@ impl Vdpa {
vec![queue_size; num_queues as usize],
iova_range,
backend_features,
false,
)
};
@ -185,6 +191,7 @@ impl Vdpa {
avail_features,
acked_features,
min_queues: num_queues,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

20
virtio-devices/src/vhost_user/blk.rs
View File

@ -13,6 +13,7 @@ use block_util::VirtioBlockConfig;
use seccompiler::SeccompAction;
use std::mem;
use std::result;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Barrier, Mutex};
use std::thread;
use std::vec::Vec;
@ -79,8 +80,14 @@ impl Blk {
let mut vu =
VhostUserHandle::connect_vhost_user(false, &vu_cfg.socket, num_queues as u64, false)?;
let (avail_features, acked_features, acked_protocol_features, vu_num_queues, config) =
if let Some(state) = state {
let (
avail_features,
acked_features,
acked_protocol_features,
vu_num_queues,
config,
paused,
) = if let Some(state) = state {
info!("Restoring vhost-user-block {}", id);
vu.set_protocol_features_vhost_user(
@ -94,6 +101,7 @@ impl Blk {
state.acked_protocol_features,
state.vu_num_queues,
state.config,
true,
)
} else {
// Filling device and vring features VMM supports.
@ -127,8 +135,7 @@ impl Blk {
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
vu.socket_handle()
.get_queue_num()
.map_err(Error::VhostUserGetQueueMaxNum)?
as usize
.map_err(Error::VhostUserGetQueueMaxNum)? as usize
} else {
DEFAULT_QUEUE_NUMBER
};
@ -151,8 +158,7 @@ impl Blk {
)
.map_err(Error::VhostUserGetConfig)?;
let mut config = VirtioBlockConfig::default();
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice())
{
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
config = *backend_config;
config.num_queues = num_queues as u16;
}
@ -167,6 +173,7 @@ impl Blk {
acked_protocol_features,
num_queues,
config,
false,
)
};
@ -178,6 +185,7 @@ impl Blk {
acked_features,
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: DEFAULT_QUEUE_NUMBER as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
vu_common: VhostUserCommon {

5
virtio-devices/src/vhost_user/fs.rs
View File

@ -16,6 +16,7 @@ use seccompiler::SeccompAction;
use std::io;
use std::os::unix::io::AsRawFd;
use std::result;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Barrier, Mutex};
use std::thread;
use versionize::{VersionMap, Versionize, VersionizeResult};
@ -333,6 +334,7 @@ impl Fs {
vu_num_queues,
config,
slave_req_support,
paused,
) = if let Some(state) = state {
info!("Restoring vhost-user-fs {}", id);
@ -348,6 +350,7 @@ impl Fs {
state.vu_num_queues,
state.config,
state.slave_req_support,
true,
)
} else {
// Filling device and vring features VMM supports.
@ -407,6 +410,7 @@ impl Fs {
num_queues,
config,
slave_req_support,
false,
)
};
@ -418,6 +422,7 @@ impl Fs {
queue_sizes: vec![queue_size; num_queues],
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: 1,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
vu_common: VhostUserCommon {

17
virtio-devices/src/vhost_user/net.rs
View File

@ -13,6 +13,7 @@ use crate::{GuestMemoryMmap, GuestRegionMmap};
use net_util::{build_net_config_space, CtrlQueue, MacAddr, VirtioNetConfig};
use seccompiler::SeccompAction;
use std::result;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Barrier, Mutex};
use std::thread;
use std::vec::Vec;
@ -83,8 +84,14 @@ impl Net {
let mut vu =
VhostUserHandle::connect_vhost_user(server, &vu_cfg.socket, num_queues as u64, false)?;
let (avail_features, acked_features, acked_protocol_features, vu_num_queues, config) =
if let Some(state) = state {
let (
avail_features,
acked_features,
acked_protocol_features,
vu_num_queues,
config,
paused,
) = if let Some(state) = state {
info!("Restoring vhost-user-net {}", id);
// The backend acknowledged features must not contain
@ -109,6 +116,7 @@ impl Net {
state.acked_protocol_features,
state.vu_num_queues,
state.config,
true,
)
} else {
// Filling device and vring features VMM supports.
@ -148,8 +156,7 @@ impl Net {
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
vu.socket_handle()
.get_queue_num()
.map_err(Error::VhostUserGetQueueMaxNum)?
as usize
.map_err(Error::VhostUserGetQueueMaxNum)? as usize
} else {
DEFAULT_QUEUE_NUMBER
};
@ -181,6 +188,7 @@ impl Net {
acked_protocol_features,
vu_num_queues,
config,
false,
)
};
@ -193,6 +201,7 @@ impl Net {
acked_features,
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: DEFAULT_QUEUE_NUMBER as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
vu_common: VhostUserCommon {

7
virtio-devices/src/vsock/device.rs
View File

@ -348,16 +348,16 @@ where
exit_evt: EventFd,
state: Option<VsockState>,
) -> io::Result<Vsock<B>> {
let (avail_features, acked_features) = if let Some(state) = state {
let (avail_features, acked_features, paused) = if let Some(state) = state {
info!("Restoring virtio-vsock {}", id);
(state.avail_features, state.acked_features)
(state.avail_features, state.acked_features, true)
} else {
let mut avail_features = 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_F_IN_ORDER;
if iommu {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
(avail_features, 0)
(avail_features, 0, false)
};
Ok(Vsock {
@ -368,6 +368,7 @@ where
paused_sync: Some(Arc::new(Barrier::new(2))),
queue_sizes: QUEUE_SIZES.to_vec(),
min_queues: NUM_QUEUES as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

7
virtio-devices/src/watchdog.rs
View File

@ -216,7 +216,7 @@ impl Watchdog {
state: Option<WatchdogState>,
) -> io::Result<Watchdog> {
let mut last_ping_time = None;
let (avail_features, acked_features) = if let Some(state) = state {
let (avail_features, acked_features, paused) = if let Some(state) = state {
info!("Restoring virtio-watchdog {}", id);
// When restoring enable the watchdog if it was previously enabled.
@ -226,9 +226,9 @@ impl Watchdog {
last_ping_time = Some(Instant::now());
}
(state.avail_features, state.acked_features)
(state.avail_features, state.acked_features, true)
} else {
(1u64 << VIRTIO_F_VERSION_1, 0)
(1u64 << VIRTIO_F_VERSION_1, 0, false)
};
let timer_fd = timerfd_create().map_err(|e| {
@ -246,6 +246,7 @@ impl Watchdog {
avail_features,
acked_features,
min_queues: 1,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
id,

1
vmm/src/device_manager.rs
View File

@ -4135,7 +4135,6 @@ impl DeviceManager {
if let Some(migratable) = &node.migratable {
info!("Restoring {} from DeviceManager", node.id);
if let Some(snapshot) = snapshot.snapshots.get(&node.id) {
migratable.lock().unwrap().pause()?;
migratable.lock().unwrap().restore(*snapshot.clone())?;
} else {
return Err(MigratableError::Restore(anyhow!(