diff --git a/vm-virtio/src/device.rs b/vm-virtio/src/device.rs index 0585dcd86..aa930d571 100644 --- a/vm-virtio/src/device.rs +++ b/vm-virtio/src/device.rs @@ -133,7 +133,41 @@ macro_rules! virtio_pausable_inner { Ok(()) } - } + }; + ($ctrl_q:expr) => { + fn pause(&mut self) -> result::Result<(), MigratableError> { + debug!( + "Pausing virtio-{}", + VirtioDeviceType::from(self.device_type()) + ); + self.paused.store(true, Ordering::SeqCst); + if let Some(pause_evt) = &self.pause_evt { + pause_evt + .write(1) + .map_err(|e| MigratableError::Pause(e.into()))?; + } + + Ok(()) + } + + fn resume(&mut self) -> result::Result<(), MigratableError> { + debug!( + "Resuming virtio-{}", + VirtioDeviceType::from(self.device_type()) + ); + self.paused.store(false, Ordering::SeqCst); + + if let Some(epoll_thread) = &self.epoll_thread { + epoll_thread.thread().unpark(); + } + + if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread { + ctrl_queue_epoll_thread.thread().unpark(); + } + + Ok(()) + } + }; } #[macro_export] @@ -143,4 +177,9 @@ macro_rules! virtio_pausable { virtio_pausable_inner!(); } }; + ($name:ident, $ctrl_q:expr) => { + impl Pausable for $name { + virtio_pausable_inner!($ctrl_q); + } + }; } diff --git a/vm-virtio/src/lib.rs b/vm-virtio/src/lib.rs index 593f34e7b..dc1e1c1a4 100755 --- a/vm-virtio/src/lib.rs +++ b/vm-virtio/src/lib.rs @@ -62,7 +62,7 @@ const VIRTIO_F_IOMMU_PLATFORM: u32 = 33; const VIRTIO_F_IN_ORDER: u32 = 35; // Types taken from linux/virtio_ids.h -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq, Eq)] #[allow(dead_code)] #[allow(non_camel_case_types)] #[repr(C)] diff --git a/vm-virtio/src/net.rs b/vm-virtio/src/net.rs index 4e7d53279..3ed7029d7 100644 --- a/vm-virtio/src/net.rs +++ b/vm-virtio/src/net.rs @@ -6,8 +6,9 @@ // found in the THIRD-PARTY file. use super::net_util::{ - build_net_config_space, open_tap, RxVirtio, TxVirtio, KILL_EVENT, NET_EVENTS_COUNT, - PAUSE_EVENT, RX_QUEUE_EVENT, RX_TAP_EVENT, TX_QUEUE_EVENT, + build_net_config_space, open_tap, register_listener, unregister_listener, CtrlVirtio, + NetCtrlEpollHandler, RxVirtio, TxVirtio, KILL_EVENT, NET_EVENTS_COUNT, PAUSE_EVENT, + RX_QUEUE_EVENT, RX_TAP_EVENT, TX_QUEUE_EVENT, }; use super::Error as DeviceError; use super::{ @@ -35,7 +36,7 @@ use vm_memory::GuestMemoryMmap; use vmm_sys_util::eventfd::EventFd; const QUEUE_SIZE: u16 = 256; -const NUM_QUEUES: usize = 2; +const NUM_QUEUES: usize = 3; const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES]; #[derive(Debug)] @@ -292,34 +293,6 @@ impl NetEpollHandler { } } -pub fn register_listener( - epoll_fd: RawFd, - fd: RawFd, - ev_type: epoll::Events, - data: u64, -) -> result::Result<(), io::Error> { - epoll::ctl( - epoll_fd, - epoll::ControlOptions::EPOLL_CTL_ADD, - fd, - epoll::Event::new(ev_type, data), - ) -} - -pub fn unregister_listener( - epoll_fd: RawFd, - fd: RawFd, - ev_type: epoll::Events, - data: u64, -) -> result::Result<(), io::Error> { - epoll::ctl( - epoll_fd, - epoll::ControlOptions::EPOLL_CTL_DEL, - fd, - epoll::Event::new(ev_type, data), - ) -} - pub struct Net { kill_evt: Option, pause_evt: Option, @@ -332,6 +305,7 @@ pub struct Net { queue_evts: Option>, interrupt_cb: Option>, epoll_thread: Option>>, + ctrl_queue_epoll_thread: Option>>, paused: Arc, } @@ -350,6 +324,8 @@ impl Net { avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM; } + avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ; + let config_space; if let Some(mac) = guest_mac { config_space = build_net_config_space(mac, &mut avail_features); @@ -367,6 +343,7 @@ impl Net { queue_evts: None, interrupt_cb: None, epoll_thread: None, + ctrl_queue_epoll_thread: None, paused: Arc::new(AtomicBool::new(false)), }) } @@ -508,6 +485,30 @@ impl VirtioDevice for Net { } self.queue_evts = Some(tmp_queue_evts); + let queue_num = queues.len(); + if (self.acked_features & 1 << VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0 { + let cvq_queue = queues.remove(queue_num - 1); + let cvq_queue_evt = queue_evts.remove(queue_num - 1); + + let mut ctrl_handler = NetCtrlEpollHandler { + mem: mem.clone(), + kill_evt: kill_evt.try_clone().unwrap(), + pause_evt: pause_evt.try_clone().unwrap(), + ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt), + epoll_fd: 0, + }; + + let paused = self.paused.clone(); + thread::Builder::new() + .name("virtio_net".to_string()) + .spawn(move || ctrl_handler.run_ctrl(paused)) + .map(|thread| self.ctrl_queue_epoll_thread = Some(thread)) + .map_err(|e| { + error!("failed to clone queue EventFd: {}", e); + ActivateError::BadActivate + })?; + } + let mut queues_v = Vec::new(); let mut queue_evts_v = Vec::new(); queues_v.push(queues.remove(0)); @@ -515,7 +516,7 @@ impl VirtioDevice for Net { queue_evts_v.push(queue_evts.remove(0)); queue_evts_v.push(queue_evts.remove(0)); let mut handler = NetEpollHandler { - mem, + mem: mem.clone(), tap, rx: RxVirtio::new(), tx: TxVirtio::new(), @@ -560,6 +561,6 @@ impl VirtioDevice for Net { } } -virtio_pausable!(Net); +virtio_pausable!(Net, true); impl Snapshotable for Net {} impl Migratable for Net {} diff --git a/vm-virtio/src/vhost_user/net.rs b/vm-virtio/src/vhost_user/net.rs index 75ad8ba28..fd64e707d 100644 --- a/vm-virtio/src/vhost_user/net.rs +++ b/vm-virtio/src/vhost_user/net.rs @@ -1,7 +1,8 @@ // Copyright 2019 Intel Corporation. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use super::super::net_util::build_net_config_space; +use super::super::net_util::{build_net_config_space, CtrlVirtio, NetCtrlEpollHandler}; +use super::super::Error as CtrlError; use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType}; use super::handler::*; use super::vu_common_ctrl::*; @@ -43,6 +44,7 @@ pub struct Net { queue_evts: Option>, interrupt_cb: Option>, epoll_thread: Option>>, + ctrl_queue_epoll_thread: Option>>, paused: Arc, } @@ -99,6 +101,9 @@ impl Net { return Err(Error::VhostUserProtocolNotSupport); } + avail_features |= 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ; + let queue_num = vu_cfg.num_queues + 1; + let config_space = build_net_config_space(mac_addr, &mut avail_features); // Send set_vring_base here, since it could tell backends, like OVS + DPDK, @@ -117,10 +122,11 @@ impl Net { acked_features, backend_features, config_space, - queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues], + queue_sizes: vec![vu_cfg.queue_size; queue_num], queue_evts: None, interrupt_cb: None, epoll_thread: None, + ctrl_queue_epoll_thread: None, paused: Arc::new(AtomicBool::new(false)), }) } @@ -204,9 +210,18 @@ impl VirtioDevice for Net { &mut self, mem: Arc>, interrupt_cb: Arc, - queues: Vec, - queue_evts: Vec, + mut queues: Vec, + mut queue_evts: Vec, ) -> ActivateResult { + if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() { + error!( + "Cannot perform activate. Expected {} queue(s), got {}", + self.queue_sizes.len(), + queues.len() + ); + return Err(ActivateError::BadActivate); + } + let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK) .and_then(|e| Ok((e.try_clone()?, e))) .map_err(|e| { @@ -238,6 +253,32 @@ impl VirtioDevice for Net { } self.queue_evts = Some(tmp_queue_evts); + let queue_num = queue_evts.len(); + + if (self.acked_features & 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0 + { + let cvq_queue = queues.remove(queue_num - 1); + let cvq_queue_evt = queue_evts.remove(queue_num - 1); + + let mut ctrl_handler = NetCtrlEpollHandler { + mem: mem.clone(), + kill_evt: kill_evt.try_clone().unwrap(), + pause_evt: pause_evt.try_clone().unwrap(), + ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt), + epoll_fd: 0, + }; + + let paused = self.paused.clone(); + thread::Builder::new() + .name("virtio_net".to_string()) + .spawn(move || ctrl_handler.run_ctrl(paused)) + .map(|thread| self.ctrl_queue_epoll_thread = Some(thread)) + .map_err(|e| { + error!("failed to clone queue EventFd: {}", e); + ActivateError::BadActivate + })?; + } + let vu_interrupt_list = setup_vhost_user( &mut self.vhost_user_net, mem.load().as_ref(), @@ -292,6 +333,6 @@ impl VirtioDevice for Net { } } -virtio_pausable!(Net); +virtio_pausable!(Net, true); impl Snapshotable for Net {} impl Migratable for Net {}