2019-08-28 09:50:48 +00:00
|
|
|
// Copyright 2019 Intel Corporation. All Rights Reserved.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
2020-07-02 12:25:19 +00:00
|
|
|
use super::super::{Descriptor, Queue};
|
2020-05-27 14:19:54 +00:00
|
|
|
use super::{Error, Result};
|
2021-02-23 13:57:10 +00:00
|
|
|
use crate::{get_host_address_range, VirtioInterrupt, VirtioInterruptType};
|
2021-06-02 19:08:04 +00:00
|
|
|
use crate::{GuestMemoryMmap, GuestRegionMmap};
|
2019-12-12 13:08:34 +00:00
|
|
|
use std::convert::TryInto;
|
2019-08-28 09:50:48 +00:00
|
|
|
use std::os::unix::io::AsRawFd;
|
2021-06-01 14:48:58 +00:00
|
|
|
use std::os::unix::net::UnixListener;
|
2020-01-14 07:18:35 +00:00
|
|
|
use std::sync::Arc;
|
2021-06-01 15:45:00 +00:00
|
|
|
use std::thread::sleep;
|
|
|
|
use std::time::{Duration, Instant};
|
2019-08-28 09:50:48 +00:00
|
|
|
use std::vec::Vec;
|
2021-05-19 21:58:02 +00:00
|
|
|
use vhost::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
|
2021-06-09 15:27:14 +00:00
|
|
|
use vhost::vhost_user::{Master, MasterReqHandler, VhostUserMaster, VhostUserMasterReqHandler};
|
2021-02-25 09:46:27 +00:00
|
|
|
use vhost::{VhostBackend, VhostUserMemoryRegionInfo, VringConfigData};
|
2021-06-02 19:08:04 +00:00
|
|
|
use vm_memory::{Address, Error as MmapError, GuestMemory, GuestMemoryRegion};
|
2019-08-28 09:50:48 +00:00
|
|
|
use vmm_sys_util::eventfd::EventFd;
|
|
|
|
|
2019-09-23 17:42:52 +00:00
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
pub struct VhostUserConfig {
|
2020-06-04 19:19:24 +00:00
|
|
|
pub socket: String,
|
2019-08-28 09:50:48 +00:00
|
|
|
pub num_queues: usize,
|
|
|
|
pub queue_size: u16,
|
|
|
|
}
|
|
|
|
|
2020-03-23 12:21:39 +00:00
|
|
|
pub fn update_mem_table(vu: &mut Master, mem: &GuestMemoryMmap) -> Result<()> {
|
2019-08-28 09:50:48 +00:00
|
|
|
let mut regions: Vec<VhostUserMemoryRegionInfo> = Vec::new();
|
2021-05-24 19:23:25 +00:00
|
|
|
for region in mem.iter() {
|
2019-08-28 09:50:48 +00:00
|
|
|
let (mmap_handle, mmap_offset) = match region.file_offset() {
|
|
|
|
Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()),
|
2021-05-24 19:23:25 +00:00
|
|
|
None => return Err(Error::VhostUserMemoryRegion(MmapError::NoMemoryRegion)),
|
2019-08-28 09:50:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
let vhost_user_net_reg = VhostUserMemoryRegionInfo {
|
|
|
|
guest_phys_addr: region.start_addr().raw_value(),
|
|
|
|
memory_size: region.len() as u64,
|
|
|
|
userspace_addr: region.as_ptr() as u64,
|
|
|
|
mmap_offset,
|
|
|
|
mmap_handle,
|
|
|
|
};
|
|
|
|
|
|
|
|
regions.push(vhost_user_net_reg);
|
2021-05-24 19:23:25 +00:00
|
|
|
}
|
2019-08-28 09:50:48 +00:00
|
|
|
|
|
|
|
vu.set_mem_table(regions.as_slice())
|
|
|
|
.map_err(Error::VhostUserSetMemTable)?;
|
|
|
|
|
2020-03-23 12:21:39 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2021-03-05 13:32:57 +00:00
|
|
|
pub fn add_memory_region(vu: &mut Master, region: &Arc<GuestRegionMmap>) -> Result<()> {
|
|
|
|
let (mmap_handle, mmap_offset) = match region.file_offset() {
|
|
|
|
Some(file_offset) => (file_offset.file().as_raw_fd(), file_offset.start()),
|
|
|
|
None => return Err(Error::MissingRegionFd),
|
|
|
|
};
|
|
|
|
|
|
|
|
let region = VhostUserMemoryRegionInfo {
|
|
|
|
guest_phys_addr: region.start_addr().raw_value(),
|
|
|
|
memory_size: region.len() as u64,
|
|
|
|
userspace_addr: region.as_ptr() as u64,
|
|
|
|
mmap_offset,
|
|
|
|
mmap_handle,
|
|
|
|
};
|
|
|
|
|
|
|
|
vu.add_mem_region(®ion)
|
|
|
|
.map_err(Error::VhostUserAddMemReg)
|
|
|
|
}
|
|
|
|
|
2021-05-19 21:58:02 +00:00
|
|
|
pub fn negotiate_features_vhost_user(
|
|
|
|
vu: &mut Master,
|
|
|
|
avail_features: u64,
|
|
|
|
avail_protocol_features: VhostUserProtocolFeatures,
|
|
|
|
) -> Result<(u64, u64)> {
|
|
|
|
// Set vhost-user owner.
|
|
|
|
vu.set_owner().map_err(Error::VhostUserSetOwner)?;
|
|
|
|
|
|
|
|
// Get features from backend, do negotiation to get a feature collection which
|
|
|
|
// both VMM and backend support.
|
|
|
|
let backend_features = vu.get_features().map_err(Error::VhostUserGetFeatures)?;
|
|
|
|
let acked_features = avail_features & backend_features;
|
|
|
|
|
|
|
|
let acked_protocol_features =
|
|
|
|
if acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
|
|
|
|
let backend_protocol_features = vu
|
|
|
|
.get_protocol_features()
|
|
|
|
.map_err(Error::VhostUserGetProtocolFeatures)?;
|
|
|
|
|
|
|
|
let acked_protocol_features = avail_protocol_features & backend_protocol_features;
|
|
|
|
|
|
|
|
vu.set_protocol_features(acked_protocol_features)
|
|
|
|
.map_err(Error::VhostUserSetProtocolFeatures)?;
|
|
|
|
|
|
|
|
acked_protocol_features.bits()
|
|
|
|
} else {
|
|
|
|
0
|
|
|
|
};
|
|
|
|
|
|
|
|
Ok((acked_features, acked_protocol_features))
|
|
|
|
}
|
|
|
|
|
2021-06-09 15:27:14 +00:00
|
|
|
pub fn setup_vhost_user<S: VhostUserMasterReqHandler>(
|
2020-03-23 12:21:39 +00:00
|
|
|
vu: &mut Master,
|
|
|
|
mem: &GuestMemoryMmap,
|
|
|
|
queues: Vec<Queue>,
|
|
|
|
queue_evts: Vec<EventFd>,
|
|
|
|
virtio_interrupt: &Arc<dyn VirtioInterrupt>,
|
2021-05-20 21:43:40 +00:00
|
|
|
acked_features: u64,
|
2021-06-09 15:27:14 +00:00
|
|
|
slave_req_handler: &Option<MasterReqHandler<S>>,
|
2021-05-12 15:13:08 +00:00
|
|
|
) -> Result<()> {
|
2021-05-20 21:43:40 +00:00
|
|
|
vu.set_features(acked_features)
|
|
|
|
.map_err(Error::VhostUserSetFeatures)?;
|
|
|
|
|
2020-03-23 12:21:39 +00:00
|
|
|
// Let's first provide the memory table to the backend.
|
|
|
|
update_mem_table(vu, mem)?;
|
|
|
|
|
2019-08-28 09:50:48 +00:00
|
|
|
for (queue_index, queue) in queues.into_iter().enumerate() {
|
2019-12-12 13:08:34 +00:00
|
|
|
let actual_size: usize = queue.actual_size().try_into().unwrap();
|
|
|
|
|
2019-09-20 16:57:00 +00:00
|
|
|
vu.set_vring_num(queue_index, queue.actual_size())
|
2019-08-28 09:50:48 +00:00
|
|
|
.map_err(Error::VhostUserSetVringNum)?;
|
|
|
|
|
|
|
|
let config_data = VringConfigData {
|
|
|
|
queue_max_size: queue.get_max_size(),
|
|
|
|
queue_size: queue.actual_size(),
|
|
|
|
flags: 0u32,
|
2019-12-12 13:08:34 +00:00
|
|
|
desc_table_addr: get_host_address_range(
|
2020-06-03 09:56:12 +00:00
|
|
|
mem,
|
2019-12-12 13:08:34 +00:00
|
|
|
queue.desc_table,
|
|
|
|
actual_size * std::mem::size_of::<Descriptor>(),
|
|
|
|
)
|
2020-11-20 11:26:11 +00:00
|
|
|
.ok_or(Error::DescriptorTableAddress)? as u64,
|
2019-12-12 13:08:34 +00:00
|
|
|
// The used ring is {flags: u16; idx: u16; virtq_used_elem [{id: u16, len: u16}; actual_size]},
|
|
|
|
// i.e. 4 + (4 + 4) * actual_size.
|
2020-06-03 09:56:12 +00:00
|
|
|
used_ring_addr: get_host_address_range(mem, queue.used_ring, 4 + actual_size * 8)
|
2020-11-20 11:26:11 +00:00
|
|
|
.ok_or(Error::UsedAddress)? as u64,
|
2019-12-12 13:08:34 +00:00
|
|
|
// The used ring is {flags: u16; idx: u16; elem [u16; actual_size]},
|
|
|
|
// i.e. 4 + (2) * actual_size.
|
2020-06-03 09:56:12 +00:00
|
|
|
avail_ring_addr: get_host_address_range(mem, queue.avail_ring, 4 + actual_size * 2)
|
2020-11-20 11:26:11 +00:00
|
|
|
.ok_or(Error::AvailAddress)? as u64,
|
2019-08-28 09:50:48 +00:00
|
|
|
log_addr: None,
|
|
|
|
};
|
|
|
|
|
|
|
|
vu.set_vring_addr(queue_index, &config_data)
|
|
|
|
.map_err(Error::VhostUserSetVringAddr)?;
|
2021-06-03 04:10:03 +00:00
|
|
|
vu.set_vring_base(
|
|
|
|
queue_index,
|
|
|
|
queue
|
|
|
|
.avail_index_from_memory(mem)
|
|
|
|
.map_err(Error::GetAvailableIndex)?,
|
|
|
|
)
|
|
|
|
.map_err(Error::VhostUserSetVringBase)?;
|
2019-08-28 09:50:48 +00:00
|
|
|
|
2020-01-14 07:18:35 +00:00
|
|
|
if let Some(eventfd) = virtio_interrupt.notifier(&VirtioInterruptType::Queue, Some(&queue))
|
|
|
|
{
|
|
|
|
vu.set_vring_call(queue_index, &eventfd)
|
|
|
|
.map_err(Error::VhostUserSetVringCall)?;
|
|
|
|
}
|
2019-08-28 09:50:48 +00:00
|
|
|
|
|
|
|
vu.set_vring_kick(queue_index, &queue_evts[queue_index])
|
|
|
|
.map_err(Error::VhostUserSetVringKick)?;
|
2019-08-30 19:01:00 +00:00
|
|
|
|
|
|
|
vu.set_vring_enable(queue_index, true)
|
|
|
|
.map_err(Error::VhostUserSetVringEnable)?;
|
2019-08-28 09:50:48 +00:00
|
|
|
}
|
|
|
|
|
2021-06-09 15:27:14 +00:00
|
|
|
if let Some(slave_req_handler) = slave_req_handler {
|
|
|
|
vu.set_slave_request_fd(slave_req_handler.get_tx_raw_fd())
|
|
|
|
.map_err(Error::VhostUserSetSlaveRequestFd)
|
|
|
|
} else {
|
|
|
|
Ok(())
|
|
|
|
}
|
2019-08-28 09:50:48 +00:00
|
|
|
}
|
|
|
|
|
2019-09-19 16:14:55 +00:00
|
|
|
pub fn reset_vhost_user(vu: &mut Master, num_queues: usize) -> Result<()> {
|
|
|
|
for queue_index in 0..num_queues {
|
|
|
|
// Disable the vrings.
|
|
|
|
vu.set_vring_enable(queue_index, false)
|
|
|
|
.map_err(Error::VhostUserSetVringEnable)?;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reset the owner.
|
|
|
|
vu.reset_owner().map_err(Error::VhostUserResetOwner)
|
|
|
|
}
|
2021-06-01 10:06:23 +00:00
|
|
|
|
2021-06-09 15:27:14 +00:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
|
|
pub fn reinitialize_vhost_user<S: VhostUserMasterReqHandler>(
|
2021-06-01 10:06:23 +00:00
|
|
|
vu: &mut Master,
|
|
|
|
mem: &GuestMemoryMmap,
|
|
|
|
queues: Vec<Queue>,
|
|
|
|
queue_evts: Vec<EventFd>,
|
|
|
|
virtio_interrupt: &Arc<dyn VirtioInterrupt>,
|
|
|
|
acked_features: u64,
|
|
|
|
acked_protocol_features: u64,
|
2021-06-09 15:27:14 +00:00
|
|
|
slave_req_handler: &Option<MasterReqHandler<S>>,
|
2021-06-01 10:06:23 +00:00
|
|
|
) -> Result<()> {
|
|
|
|
vu.set_owner().map_err(Error::VhostUserSetOwner)?;
|
|
|
|
vu.get_features().map_err(Error::VhostUserGetFeatures)?;
|
|
|
|
|
|
|
|
if acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
|
|
|
|
if let Some(acked_protocol_features) =
|
|
|
|
VhostUserProtocolFeatures::from_bits(acked_protocol_features)
|
|
|
|
{
|
|
|
|
vu.set_protocol_features(acked_protocol_features)
|
|
|
|
.map_err(Error::VhostUserSetProtocolFeatures)?;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
setup_vhost_user(
|
|
|
|
vu,
|
|
|
|
mem,
|
|
|
|
queues,
|
|
|
|
queue_evts,
|
|
|
|
virtio_interrupt,
|
|
|
|
acked_features,
|
2021-06-09 15:27:14 +00:00
|
|
|
slave_req_handler,
|
2021-06-01 10:06:23 +00:00
|
|
|
)
|
|
|
|
}
|
2021-06-01 14:48:58 +00:00
|
|
|
|
|
|
|
pub fn connect_vhost_user(
|
|
|
|
server: bool,
|
|
|
|
socket_path: &str,
|
|
|
|
num_queues: u64,
|
|
|
|
unlink_socket: bool,
|
|
|
|
) -> Result<Master> {
|
2021-06-01 15:45:00 +00:00
|
|
|
if server {
|
2021-06-01 14:48:58 +00:00
|
|
|
if unlink_socket {
|
|
|
|
std::fs::remove_file(socket_path).map_err(Error::RemoveSocketPath)?;
|
|
|
|
}
|
|
|
|
|
|
|
|
info!("Binding vhost-user listener...");
|
|
|
|
let listener = UnixListener::bind(socket_path).map_err(Error::BindSocket)?;
|
|
|
|
info!("Waiting for incoming vhost-user connection...");
|
|
|
|
let (stream, _) = listener.accept().map_err(Error::AcceptConnection)?;
|
|
|
|
|
2021-06-01 15:45:00 +00:00
|
|
|
Ok(Master::from_stream(stream, num_queues))
|
2021-06-01 14:48:58 +00:00
|
|
|
} else {
|
2021-06-01 15:45:00 +00:00
|
|
|
let now = Instant::now();
|
|
|
|
|
|
|
|
// Retry connecting for a full minute
|
|
|
|
let err = loop {
|
|
|
|
let err = match Master::connect(socket_path, num_queues) {
|
|
|
|
Ok(m) => return Ok(m),
|
|
|
|
Err(e) => e,
|
|
|
|
};
|
|
|
|
sleep(Duration::from_millis(100));
|
|
|
|
|
2021-06-02 14:57:40 +00:00
|
|
|
if now.elapsed().as_secs() >= 60 {
|
2021-06-01 15:45:00 +00:00
|
|
|
break err;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
error!(
|
|
|
|
"Failed connecting the backend after trying for 1 minute: {:?}",
|
|
|
|
err
|
|
|
|
);
|
|
|
|
Err(Error::VhostUserConnect)
|
|
|
|
}
|
2021-06-01 14:48:58 +00:00
|
|
|
}
|