External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2024-07-07 10:15:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

vm-virtio: Implement control queue support for net devices

Browse Source 
While feature VIRTIO_NET_F_CTRL_VQ is negotiated, control queue
will exits besides the Tx/Rx virtqueues, an epoll handler should
be started to monitor and handle the control queue event.

Signed-off-by: Cathy Zhang <cathy.zhang@intel.com>
This commit is contained in:
Cathy Zhang 2020-01-15 17:32:05 +08:00 committed by Sebastien Boeuf
parent d38787c578
commit 709f7fe607
4 changed files with 121 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
vm-virtio/src/device.rs
View File

@ -133,7 +133,41 @@ macro_rules! virtio_pausable_inner {
Ok(())
}
}
};
($ctrl_q:expr) => {
fn pause(&mut self) -> result::Result<(), MigratableError> {
debug!(
"Pausing virtio-{}",
VirtioDeviceType::from(self.device_type())
);
self.paused.store(true, Ordering::SeqCst);
if let Some(pause_evt) = &self.pause_evt {
pause_evt
.write(1)
.map_err(|e| MigratableError::Pause(e.into()))?;
}
Ok(())
}
fn resume(&mut self) -> result::Result<(), MigratableError> {
debug!(
"Resuming virtio-{}",
VirtioDeviceType::from(self.device_type())
);
self.paused.store(false, Ordering::SeqCst);
if let Some(epoll_thread) = &self.epoll_thread {
epoll_thread.thread().unpark();
}
if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread {
ctrl_queue_epoll_thread.thread().unpark();
}
Ok(())
}
};
}
#[macro_export]
@ -143,4 +177,9 @@ macro_rules! virtio_pausable {
virtio_pausable_inner!();
}
};
($name:ident, $ctrl_q:expr) => {
impl Pausable for $name {
virtio_pausable_inner!($ctrl_q);
}
};
}

2
vm-virtio/src/lib.rs
View File

@ -62,7 +62,7 @@ const VIRTIO_F_IOMMU_PLATFORM: u32 = 33;
const VIRTIO_F_IN_ORDER: u32 = 35;
// Types taken from linux/virtio_ids.h
#[derive(Copy, Clone)]
#[derive(Copy, Clone, PartialEq, Eq)]
#[allow(dead_code)]
#[allow(non_camel_case_types)]
#[repr(C)]

67
vm-virtio/src/net.rs
View File

@ -6,8 +6,9 @@
// found in the THIRD-PARTY file.
use super::net_util::{
build_net_config_space, open_tap, RxVirtio, TxVirtio, KILL_EVENT, NET_EVENTS_COUNT,
PAUSE_EVENT, RX_QUEUE_EVENT, RX_TAP_EVENT, TX_QUEUE_EVENT,
build_net_config_space, open_tap, register_listener, unregister_listener, CtrlVirtio,
NetCtrlEpollHandler, RxVirtio, TxVirtio, KILL_EVENT, NET_EVENTS_COUNT, PAUSE_EVENT,
RX_QUEUE_EVENT, RX_TAP_EVENT, TX_QUEUE_EVENT,
};
use super::Error as DeviceError;
use super::{
@ -35,7 +36,7 @@ use vm_memory::GuestMemoryMmap;
use vmm_sys_util::eventfd::EventFd;
const QUEUE_SIZE: u16 = 256;
const NUM_QUEUES: usize = 2;
const NUM_QUEUES: usize = 3;
const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES];
#[derive(Debug)]
@ -292,34 +293,6 @@ impl NetEpollHandler {
}
}
pub fn register_listener(
epoll_fd: RawFd,
fd: RawFd,
ev_type: epoll::Events,
data: u64,
) -> result::Result<(), io::Error> {
epoll::ctl(
epoll_fd,
epoll::ControlOptions::EPOLL_CTL_ADD,
fd,
epoll::Event::new(ev_type, data),
)
}
pub fn unregister_listener(
epoll_fd: RawFd,
fd: RawFd,
ev_type: epoll::Events,
data: u64,
) -> result::Result<(), io::Error> {
epoll::ctl(
epoll_fd,
epoll::ControlOptions::EPOLL_CTL_DEL,
fd,
epoll::Event::new(ev_type, data),
)
}
pub struct Net {
kill_evt: Option<EventFd>,
pause_evt: Option<EventFd>,
@ -332,6 +305,7 @@ pub struct Net {
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<VirtioInterrupt>>,
epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
paused: Arc<AtomicBool>,
}
@ -350,6 +324,8 @@ impl Net {
avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
}
avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ;
let config_space;
if let Some(mac) = guest_mac {
config_space = build_net_config_space(mac, &mut avail_features);
@ -367,6 +343,7 @@ impl Net {
queue_evts: None,
interrupt_cb: None,
epoll_thread: None,
ctrl_queue_epoll_thread: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
@ -508,6 +485,30 @@ impl VirtioDevice for Net {
}
self.queue_evts = Some(tmp_queue_evts);
let queue_num = queues.len();
if (self.acked_features & 1 << VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0 {
let cvq_queue = queues.remove(queue_num - 1);
let cvq_queue_evt = queue_evts.remove(queue_num - 1);
let mut ctrl_handler = NetCtrlEpollHandler {
mem: mem.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
epoll_fd: 0,
};
let paused = self.paused.clone();
thread::Builder::new()
.name("virtio_net".to_string())
.spawn(move || ctrl_handler.run_ctrl(paused))
.map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
}
let mut queues_v = Vec::new();
let mut queue_evts_v = Vec::new();
queues_v.push(queues.remove(0));
@ -515,7 +516,7 @@ impl VirtioDevice for Net {
queue_evts_v.push(queue_evts.remove(0));
queue_evts_v.push(queue_evts.remove(0));
let mut handler = NetEpollHandler {
mem,
mem: mem.clone(),
tap,
rx: RxVirtio::new(),
tx: TxVirtio::new(),
@ -560,6 +561,6 @@ impl VirtioDevice for Net {
}
}
virtio_pausable!(Net);
virtio_pausable!(Net, true);
impl Snapshotable for Net {}
impl Migratable for Net {}

51
vm-virtio/src/vhost_user/net.rs
View File

@ -1,7 +1,8 @@
// Copyright 2019 Intel Corporation. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use super::super::net_util::build_net_config_space;
use super::super::net_util::{build_net_config_space, CtrlVirtio, NetCtrlEpollHandler};
use super::super::Error as CtrlError;
use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
use super::handler::*;
use super::vu_common_ctrl::*;
@ -43,6 +44,7 @@ pub struct Net {
queue_evts: Option<Vec<EventFd>>,
interrupt_cb: Option<Arc<VirtioInterrupt>>,
epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), CtrlError>>>,
paused: Arc<AtomicBool>,
}
@ -99,6 +101,9 @@ impl Net {
return Err(Error::VhostUserProtocolNotSupport);
}
avail_features |= 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ;
let queue_num = vu_cfg.num_queues + 1;
let config_space = build_net_config_space(mac_addr, &mut avail_features);
// Send set_vring_base here, since it could tell backends, like OVS + DPDK,
@ -117,10 +122,11 @@ impl Net {
acked_features,
backend_features,
config_space,
queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
queue_sizes: vec![vu_cfg.queue_size; queue_num],
queue_evts: None,
interrupt_cb: None,
epoll_thread: None,
ctrl_queue_epoll_thread: None,
paused: Arc::new(AtomicBool::new(false)),
})
}
@ -204,9 +210,18 @@ impl VirtioDevice for Net {
&mut self,
mem: Arc<ArcSwap<GuestMemoryMmap>>,
interrupt_cb: Arc<VirtioInterrupt>,
queues: Vec<Queue>,
queue_evts: Vec<EventFd>,
mut queues: Vec<Queue>,
mut queue_evts: Vec<EventFd>,
) -> ActivateResult {
if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
error!(
"Cannot perform activate. Expected {} queue(s), got {}",
self.queue_sizes.len(),
queues.len()
);
return Err(ActivateError::BadActivate);
}
let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(|e| {
@ -238,6 +253,32 @@ impl VirtioDevice for Net {
}
self.queue_evts = Some(tmp_queue_evts);
let queue_num = queue_evts.len();
if (self.acked_features & 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0
{
let cvq_queue = queues.remove(queue_num - 1);
let cvq_queue_evt = queue_evts.remove(queue_num - 1);
let mut ctrl_handler = NetCtrlEpollHandler {
mem: mem.clone(),
kill_evt: kill_evt.try_clone().unwrap(),
pause_evt: pause_evt.try_clone().unwrap(),
ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
epoll_fd: 0,
};
let paused = self.paused.clone();
thread::Builder::new()
.name("virtio_net".to_string())
.spawn(move || ctrl_handler.run_ctrl(paused))
.map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
.map_err(|e| {
error!("failed to clone queue EventFd: {}", e);
ActivateError::BadActivate
})?;
}
let vu_interrupt_list = setup_vhost_user(
&mut self.vhost_user_net,
mem.load().as_ref(),
@ -292,6 +333,6 @@ impl VirtioDevice for Net {
}
}
virtio_pausable!(Net);
virtio_pausable!(Net, true);
impl Snapshotable for Net {}
impl Migratable for Net {}