From e382dc66572a7a9eaa6e95739907080267da47b8 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Mon, 22 Jun 2020 16:00:02 +0200 Subject: [PATCH] vmm, vm-virtio: Restore DeviceManager's devices in a paused state The same way the VM and the vCPUs are restored in a paused state, all devices associated with the device manager must be restored in the same paused state. Signed-off-by: Sebastien Boeuf --- vm-virtio/src/block.rs | 8 ++++++++ vm-virtio/src/console.rs | 8 ++++++++ vm-virtio/src/iommu.rs | 8 ++++++++ vm-virtio/src/mem.rs | 8 ++++++++ vm-virtio/src/net.rs | 8 ++++++++ vm-virtio/src/net_util.rs | 9 +++++++++ vm-virtio/src/pmem.rs | 8 ++++++++ vm-virtio/src/rng.rs | 8 ++++++++ vm-virtio/src/vhost_user/handler.rs | 8 ++++++++ vm-virtio/src/vsock/device.rs | 8 ++++++++ vmm/src/device_manager.rs | 1 + 11 files changed, 82 insertions(+) diff --git a/vm-virtio/src/block.rs b/vm-virtio/src/block.rs index 970d784f3..4af880897 100644 --- a/vm-virtio/src/block.rs +++ b/vm-virtio/src/block.rs @@ -722,6 +722,14 @@ impl BlockEpollHandler { const EPOLL_EVENTS_LEN: usize = 100; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/console.rs b/vm-virtio/src/console.rs index f09e45bc4..9086915f8 100644 --- a/vm-virtio/src/console.rs +++ b/vm-virtio/src/console.rs @@ -236,6 +236,14 @@ impl ConsoleEpollHandler { const EPOLL_EVENTS_LEN: usize = 100; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/iommu.rs b/vm-virtio/src/iommu.rs index 895fb7910..1ec1b7c89 100644 --- a/vm-virtio/src/iommu.rs +++ b/vm-virtio/src/iommu.rs @@ -687,6 +687,14 @@ impl IommuEpollHandler { const EPOLL_EVENTS_LEN: usize = 100; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/mem.rs b/vm-virtio/src/mem.rs index 33791ff2f..248978caf 100644 --- a/vm-virtio/src/mem.rs +++ b/vm-virtio/src/mem.rs @@ -634,6 +634,14 @@ impl MemEpollHandler { const EPOLL_EVENTS_LEN: usize = 100; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/net.rs b/vm-virtio/src/net.rs index a8a650789..4277ec9a6 100644 --- a/vm-virtio/src/net.rs +++ b/vm-virtio/src/net.rs @@ -323,6 +323,14 @@ impl NetEpollHandler { let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); NET_EVENTS_COUNT]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/net_util.rs b/vm-virtio/src/net_util.rs index 66b6af2ad..61e1d51c6 100644 --- a/vm-virtio/src/net_util.rs +++ b/vm-virtio/src/net_util.rs @@ -16,6 +16,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::thread; use virtio_bindings::bindings::virtio_net::*; use vm_memory::{ ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryError, @@ -278,6 +279,14 @@ impl NetCtrlEpollHandler { let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); CTRL_EVENT_COUNT]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/pmem.rs b/vm-virtio/src/pmem.rs index 2789fb210..ebb474eba 100644 --- a/vm-virtio/src/pmem.rs +++ b/vm-virtio/src/pmem.rs @@ -278,6 +278,14 @@ impl PmemEpollHandler { const EPOLL_EVENTS_LEN: usize = 100; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/rng.rs b/vm-virtio/src/rng.rs index c4d1f8717..295e06b86 100644 --- a/vm-virtio/src/rng.rs +++ b/vm-virtio/src/rng.rs @@ -121,6 +121,14 @@ impl RngEpollHandler { const EPOLL_EVENTS_LEN: usize = 100; let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/vhost_user/handler.rs b/vm-virtio/src/vhost_user/handler.rs index 9e9dce453..8197c6938 100644 --- a/vm-virtio/src/vhost_user/handler.rs +++ b/vm-virtio/src/vhost_user/handler.rs @@ -117,6 +117,14 @@ impl VhostUserEpollHandler { let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); index + 1]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'poll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vm-virtio/src/vsock/device.rs b/vm-virtio/src/vsock/device.rs index 3b3f77f1d..6dd01e392 100644 --- a/vm-virtio/src/vsock/device.rs +++ b/vm-virtio/src/vsock/device.rs @@ -253,6 +253,14 @@ where let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EVENTS_LEN]; + // Before jumping into the epoll loop, check if the device is expected + // to be in a paused state. This is helpful for the restore code path + // as the device thread should not start processing anything before the + // device has been resumed. + while paused.load(Ordering::SeqCst) { + thread::park(); + } + 'epoll: loop { let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) { Ok(res) => res, diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index 4a3f779cf..7e1843815 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -3477,6 +3477,7 @@ impl Snapshottable for DeviceManager { if let Some(migratable) = &node.migratable { debug!("Restoring {} from DeviceManager", node.id); if let Some(snapshot) = snapshot.snapshots.get(&node.id) { + migratable.lock().unwrap().pause()?; migratable.lock().unwrap().restore(*snapshot.clone())?; } else { return Err(MigratableError::Restore(anyhow!(