2019-08-28 09:50:48 +00:00
|
|
|
// Copyright 2019 Intel Corporation. All Rights Reserved.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
2021-05-21 16:26:00 +00:00
|
|
|
use crate::{
|
2021-06-02 19:08:04 +00:00
|
|
|
EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, Queue, VirtioInterrupt,
|
2021-06-01 15:16:09 +00:00
|
|
|
EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, VIRTIO_F_NOTIFICATION_DATA,
|
|
|
|
VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, VIRTIO_F_RING_INDIRECT_DESC,
|
2021-07-07 09:12:26 +00:00
|
|
|
VIRTIO_F_VERSION_1,
|
2021-05-21 16:26:00 +00:00
|
|
|
};
|
2019-08-28 09:50:48 +00:00
|
|
|
use std::io;
|
2021-06-01 15:16:09 +00:00
|
|
|
use std::ops::Deref;
|
|
|
|
use std::os::unix::io::AsRawFd;
|
|
|
|
use std::sync::{atomic::AtomicBool, Arc, Barrier, Mutex};
|
2021-06-10 07:37:30 +00:00
|
|
|
use vhost::vhost_user::message::{VhostUserInflight, VhostUserVirtioFeatures};
|
2021-06-09 14:33:34 +00:00
|
|
|
use vhost::vhost_user::{Master, MasterReqHandler, VhostUserMasterReqHandler};
|
2021-02-25 09:46:27 +00:00
|
|
|
use vhost::Error as VhostError;
|
2021-06-02 19:08:04 +00:00
|
|
|
use vm_memory::{Error as MmapError, GuestAddressSpace, GuestMemoryAtomic};
|
2021-06-03 04:10:03 +00:00
|
|
|
use vm_virtio::Error as VirtioError;
|
2021-06-01 15:16:09 +00:00
|
|
|
use vmm_sys_util::eventfd::EventFd;
|
|
|
|
use vu_common_ctrl::{connect_vhost_user, reinitialize_vhost_user};
|
2019-08-28 09:50:48 +00:00
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
pub mod blk;
|
2019-08-30 16:16:32 +00:00
|
|
|
pub mod fs;
|
2019-08-28 09:50:48 +00:00
|
|
|
pub mod net;
|
|
|
|
pub mod vu_common_ctrl;
|
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
pub use self::blk::Blk;
|
2019-08-30 16:16:32 +00:00
|
|
|
pub use self::fs::*;
|
2019-08-28 09:50:48 +00:00
|
|
|
pub use self::net::Net;
|
|
|
|
pub use self::vu_common_ctrl::VhostUserConfig;
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum Error {
|
2021-05-04 08:33:35 +00:00
|
|
|
/// Failed accepting connection.
|
|
|
|
AcceptConnection(io::Error),
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Invalid available address.
|
|
|
|
AvailAddress,
|
|
|
|
/// Queue number is not correct
|
|
|
|
BadQueueNum,
|
2021-05-04 08:33:35 +00:00
|
|
|
/// Failed binding vhost-user socket.
|
|
|
|
BindSocket(io::Error),
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Creating kill eventfd failed.
|
|
|
|
CreateKillEventFd(io::Error),
|
|
|
|
/// Cloning kill eventfd failed.
|
|
|
|
CloneKillEventFd(io::Error),
|
|
|
|
/// Invalid descriptor table address.
|
|
|
|
DescriptorTableAddress,
|
|
|
|
/// Signal used queue failed.
|
|
|
|
FailedSignalingUsedQueue(io::Error),
|
|
|
|
/// Failed to read vhost eventfd.
|
|
|
|
MemoryRegions(MmapError),
|
2021-06-01 14:48:58 +00:00
|
|
|
/// Failed removing socket path
|
|
|
|
RemoveSocketPath(io::Error),
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Failed to create master.
|
|
|
|
VhostUserCreateMaster(VhostError),
|
|
|
|
/// Failed to open vhost device.
|
|
|
|
VhostUserOpen(VhostError),
|
2019-08-30 17:40:33 +00:00
|
|
|
/// Connection to socket failed.
|
2021-06-01 15:45:00 +00:00
|
|
|
VhostUserConnect,
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Get features failed.
|
|
|
|
VhostUserGetFeatures(VhostError),
|
2020-02-03 06:24:41 +00:00
|
|
|
/// Get queue max number failed.
|
|
|
|
VhostUserGetQueueMaxNum(VhostError),
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Get protocol features failed.
|
|
|
|
VhostUserGetProtocolFeatures(VhostError),
|
|
|
|
/// Vhost-user Backend not support vhost-user protocol.
|
|
|
|
VhostUserProtocolNotSupport,
|
|
|
|
/// Set owner failed.
|
|
|
|
VhostUserSetOwner(VhostError),
|
2019-09-19 16:14:55 +00:00
|
|
|
/// Reset owner failed.
|
|
|
|
VhostUserResetOwner(VhostError),
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Set features failed.
|
|
|
|
VhostUserSetFeatures(VhostError),
|
|
|
|
/// Set protocol features failed.
|
|
|
|
VhostUserSetProtocolFeatures(VhostError),
|
|
|
|
/// Set mem table failed.
|
|
|
|
VhostUserSetMemTable(VhostError),
|
|
|
|
/// Set vring num failed.
|
|
|
|
VhostUserSetVringNum(VhostError),
|
|
|
|
/// Set vring addr failed.
|
|
|
|
VhostUserSetVringAddr(VhostError),
|
|
|
|
/// Set vring base failed.
|
|
|
|
VhostUserSetVringBase(VhostError),
|
|
|
|
/// Set vring call failed.
|
|
|
|
VhostUserSetVringCall(VhostError),
|
|
|
|
/// Set vring kick failed.
|
|
|
|
VhostUserSetVringKick(VhostError),
|
|
|
|
/// Set vring enable failed.
|
|
|
|
VhostUserSetVringEnable(VhostError),
|
|
|
|
/// Failed to create vhost eventfd.
|
|
|
|
VhostIrqCreate(io::Error),
|
|
|
|
/// Failed to read vhost eventfd.
|
|
|
|
VhostIrqRead(io::Error),
|
|
|
|
/// Failed to read vhost eventfd.
|
|
|
|
VhostUserMemoryRegion(MmapError),
|
2019-08-30 17:40:33 +00:00
|
|
|
/// Failed to create the master request handler from slave.
|
2021-02-25 09:46:27 +00:00
|
|
|
MasterReqHandlerCreation(vhost::vhost_user::Error),
|
2019-08-30 17:40:33 +00:00
|
|
|
/// Set slave request fd failed.
|
2021-02-25 09:46:27 +00:00
|
|
|
VhostUserSetSlaveRequestFd(vhost::Error),
|
2021-03-05 13:32:57 +00:00
|
|
|
/// Add memory region failed.
|
|
|
|
VhostUserAddMemReg(VhostError),
|
2021-05-19 20:33:04 +00:00
|
|
|
/// Failed getting the configuration.
|
|
|
|
VhostUserGetConfig(VhostError),
|
|
|
|
/// Failed setting the configuration.
|
|
|
|
VhostUserSetConfig(VhostError),
|
2021-06-10 07:37:30 +00:00
|
|
|
/// Failed getting inflight shm log.
|
|
|
|
VhostUserGetInflight(VhostError),
|
|
|
|
/// Failed setting inflight shm log.
|
|
|
|
VhostUserSetInflight(VhostError),
|
2019-08-28 09:50:48 +00:00
|
|
|
/// Invalid used address.
|
|
|
|
UsedAddress,
|
2019-09-02 11:00:35 +00:00
|
|
|
/// Invalid features provided from vhost-user backend
|
|
|
|
InvalidFeatures,
|
2021-03-05 13:32:57 +00:00
|
|
|
/// Missing file descriptor for the region.
|
|
|
|
MissingRegionFd,
|
2021-05-12 15:13:08 +00:00
|
|
|
/// Missing IrqFd
|
|
|
|
MissingIrqFd,
|
2021-06-03 04:10:03 +00:00
|
|
|
/// Failed getting the available index.
|
|
|
|
GetAvailableIndex(VirtioError),
|
2019-08-28 09:50:48 +00:00
|
|
|
}
|
|
|
|
type Result<T> = std::result::Result<T, Error>;
|
2021-05-21 16:26:00 +00:00
|
|
|
|
|
|
|
pub const DEFAULT_VIRTIO_FEATURES: u64 = 1 << VIRTIO_F_RING_INDIRECT_DESC
|
|
|
|
| 1 << VIRTIO_F_RING_EVENT_IDX
|
|
|
|
| 1 << VIRTIO_F_VERSION_1
|
|
|
|
| 1 << VIRTIO_F_IN_ORDER
|
|
|
|
| 1 << VIRTIO_F_ORDER_PLATFORM
|
|
|
|
| 1 << VIRTIO_F_NOTIFICATION_DATA
|
|
|
|
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
|
2021-06-01 15:16:09 +00:00
|
|
|
|
|
|
|
const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;
|
2021-06-09 14:33:34 +00:00
|
|
|
const SLAVE_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2;
|
2021-06-01 15:16:09 +00:00
|
|
|
|
2021-06-10 07:37:30 +00:00
|
|
|
#[derive(Default)]
|
|
|
|
pub struct Inflight {
|
|
|
|
pub info: VhostUserInflight,
|
|
|
|
pub fd: Option<std::fs::File>,
|
|
|
|
}
|
|
|
|
|
2021-06-09 14:33:34 +00:00
|
|
|
pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> {
|
2021-06-01 15:16:09 +00:00
|
|
|
pub vu: Arc<Mutex<Master>>,
|
|
|
|
pub mem: GuestMemoryAtomic<GuestMemoryMmap>,
|
|
|
|
pub kill_evt: EventFd,
|
|
|
|
pub pause_evt: EventFd,
|
|
|
|
pub queues: Vec<Queue>,
|
|
|
|
pub queue_evts: Vec<EventFd>,
|
|
|
|
pub virtio_interrupt: Arc<dyn VirtioInterrupt>,
|
|
|
|
pub acked_features: u64,
|
|
|
|
pub acked_protocol_features: u64,
|
|
|
|
pub socket_path: String,
|
|
|
|
pub server: bool,
|
2021-06-09 14:33:34 +00:00
|
|
|
pub slave_req_handler: Option<MasterReqHandler<S>>,
|
2021-06-10 07:37:30 +00:00
|
|
|
pub inflight: Option<Inflight>,
|
2021-06-01 15:16:09 +00:00
|
|
|
}
|
|
|
|
|
2021-06-09 14:33:34 +00:00
|
|
|
impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
|
2021-06-01 15:16:09 +00:00
|
|
|
pub fn run(
|
|
|
|
&mut self,
|
|
|
|
paused: Arc<AtomicBool>,
|
|
|
|
paused_sync: Arc<Barrier>,
|
|
|
|
) -> std::result::Result<(), EpollHelperError> {
|
|
|
|
let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?;
|
|
|
|
helper.add_event_custom(
|
|
|
|
self.vu.lock().unwrap().as_raw_fd(),
|
|
|
|
HUP_CONNECTION_EVENT,
|
|
|
|
epoll::Events::EPOLLHUP,
|
|
|
|
)?;
|
2021-06-09 14:33:34 +00:00
|
|
|
|
|
|
|
if let Some(slave_req_handler) = &self.slave_req_handler {
|
|
|
|
helper.add_event(slave_req_handler.as_raw_fd(), SLAVE_REQ_EVENT)?;
|
|
|
|
}
|
|
|
|
|
2021-06-01 15:16:09 +00:00
|
|
|
helper.run(paused, paused_sync, self)?;
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> {
|
|
|
|
helper.del_event_custom(
|
|
|
|
self.vu.lock().unwrap().as_raw_fd(),
|
|
|
|
HUP_CONNECTION_EVENT,
|
|
|
|
epoll::Events::EPOLLHUP,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
let mut vhost_user = connect_vhost_user(
|
|
|
|
self.server,
|
|
|
|
&self.socket_path,
|
|
|
|
self.queues.len() as u64,
|
|
|
|
true,
|
|
|
|
)
|
|
|
|
.map_err(|e| {
|
|
|
|
EpollHelperError::IoError(std::io::Error::new(
|
|
|
|
std::io::ErrorKind::Other,
|
|
|
|
format!("failed connecting vhost-user backend{:?}", e),
|
|
|
|
))
|
|
|
|
})?;
|
|
|
|
|
|
|
|
// Initialize the backend
|
|
|
|
reinitialize_vhost_user(
|
|
|
|
&mut vhost_user,
|
|
|
|
self.mem.memory().deref(),
|
|
|
|
self.queues.clone(),
|
|
|
|
self.queue_evts
|
|
|
|
.iter()
|
|
|
|
.map(|q| q.try_clone().unwrap())
|
|
|
|
.collect(),
|
|
|
|
&self.virtio_interrupt,
|
|
|
|
self.acked_features,
|
|
|
|
self.acked_protocol_features,
|
2021-06-09 15:27:14 +00:00
|
|
|
&self.slave_req_handler,
|
2021-06-10 07:37:30 +00:00
|
|
|
self.inflight.as_mut(),
|
2021-06-01 15:16:09 +00:00
|
|
|
)
|
|
|
|
.map_err(|e| {
|
|
|
|
EpollHelperError::IoError(std::io::Error::new(
|
|
|
|
std::io::ErrorKind::Other,
|
|
|
|
format!("failed reconnecting vhost-user backend{:?}", e),
|
|
|
|
))
|
|
|
|
})?;
|
|
|
|
|
|
|
|
helper.add_event_custom(
|
|
|
|
vhost_user.as_raw_fd(),
|
|
|
|
HUP_CONNECTION_EVENT,
|
|
|
|
epoll::Events::EPOLLHUP,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
// Update vhost-user reference
|
|
|
|
let mut vu = self.vu.lock().unwrap();
|
|
|
|
*vu = vhost_user;
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 14:33:34 +00:00
|
|
|
impl<S: VhostUserMasterReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> {
|
2021-06-01 15:16:09 +00:00
|
|
|
fn handle_event(&mut self, helper: &mut EpollHelper, event: &epoll::Event) -> bool {
|
|
|
|
let ev_type = event.data as u16;
|
|
|
|
match ev_type {
|
|
|
|
HUP_CONNECTION_EVENT => {
|
|
|
|
if let Err(e) = self.reconnect(helper) {
|
|
|
|
error!("failed to reconnect vhost-user backend: {:?}", e);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2021-06-09 14:33:34 +00:00
|
|
|
SLAVE_REQ_EVENT => {
|
|
|
|
if let Some(slave_req_handler) = self.slave_req_handler.as_mut() {
|
|
|
|
if let Err(e) = slave_req_handler.handle_request() {
|
|
|
|
error!("Failed to handle request from vhost-user backend: {:?}", e);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-06-01 15:16:09 +00:00
|
|
|
_ => {
|
2021-06-14 08:46:44 +00:00
|
|
|
error!("Unknown event for vhost-user thread");
|
2021-06-01 15:16:09 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
false
|
|
|
|
}
|
|
|
|
}
|