2019-09-02 11:00:35 +00:00
|
|
|
// Copyright 2019 Intel Corporation. All Rights Reserved.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
2020-09-03 09:37:36 +00:00
|
|
|
use super::super::{
|
|
|
|
ActivateError, ActivateResult, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType,
|
|
|
|
};
|
2019-12-31 10:49:11 +00:00
|
|
|
use super::handler::*;
|
|
|
|
use super::vu_common_ctrl::*;
|
|
|
|
use super::{Error, Result};
|
2020-08-18 04:59:27 +00:00
|
|
|
use crate::seccomp_filters::{get_seccomp_filter, Thread};
|
2019-12-31 10:49:11 +00:00
|
|
|
use crate::VirtioInterrupt;
|
2020-07-14 09:43:33 +00:00
|
|
|
use block_util::VirtioBlockConfig;
|
2020-08-18 04:59:27 +00:00
|
|
|
use seccomp::{SeccompAction, SeccompFilter};
|
2019-12-31 10:49:11 +00:00
|
|
|
use std::mem;
|
2020-02-14 10:08:59 +00:00
|
|
|
use std::os::unix::io::AsRawFd;
|
2019-12-02 20:08:53 +00:00
|
|
|
use std::result;
|
2020-08-11 14:05:06 +00:00
|
|
|
use std::sync::{Arc, Barrier};
|
2019-09-02 11:00:35 +00:00
|
|
|
use std::thread;
|
|
|
|
use std::vec::Vec;
|
|
|
|
use vhost_rs::vhost_user::message::VhostUserConfigFlags;
|
2020-03-03 11:43:48 +00:00
|
|
|
use vhost_rs::vhost_user::message::VHOST_USER_CONFIG_OFFSET;
|
2019-09-02 11:00:35 +00:00
|
|
|
use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
|
|
|
|
use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
|
|
|
|
use vhost_rs::VhostBackend;
|
|
|
|
use virtio_bindings::bindings::virtio_blk::*;
|
2020-02-14 11:14:27 +00:00
|
|
|
use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
|
2020-02-11 16:22:40 +00:00
|
|
|
use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
|
2019-05-01 16:59:51 +00:00
|
|
|
use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
|
2019-12-31 10:49:11 +00:00
|
|
|
use vmm_sys_util::eventfd::EventFd;
|
2019-09-02 11:00:35 +00:00
|
|
|
|
|
|
|
struct SlaveReqHandler {}
|
|
|
|
impl VhostUserMasterReqHandler for SlaveReqHandler {}
|
|
|
|
|
|
|
|
pub struct Blk {
|
2020-09-03 09:37:36 +00:00
|
|
|
common: VirtioCommon,
|
2020-04-27 11:56:26 +00:00
|
|
|
id: String,
|
2019-09-02 11:00:35 +00:00
|
|
|
vhost_user_blk: Master,
|
2020-01-27 17:12:58 +00:00
|
|
|
config: VirtioBlockConfig,
|
2020-08-18 04:59:27 +00:00
|
|
|
seccomp_action: SeccompAction,
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2019-09-23 17:42:52 +00:00
|
|
|
impl Blk {
|
2019-09-02 11:00:35 +00:00
|
|
|
/// Create a new vhost-user-blk device
|
2020-08-18 04:59:27 +00:00
|
|
|
pub fn new(id: String, vu_cfg: VhostUserConfig, seccomp_action: SeccompAction) -> Result<Blk> {
|
2020-06-04 19:19:24 +00:00
|
|
|
let mut vhost_user_blk = Master::connect(&vu_cfg.socket, vu_cfg.num_queues as u64)
|
2019-09-02 11:00:35 +00:00
|
|
|
.map_err(Error::VhostUserCreateMaster)?;
|
|
|
|
|
|
|
|
// Filling device and vring features VMM supports.
|
|
|
|
let mut avail_features = 1 << VIRTIO_BLK_F_SEG_MAX
|
|
|
|
| 1 << VIRTIO_BLK_F_RO
|
|
|
|
| 1 << VIRTIO_BLK_F_BLK_SIZE
|
|
|
|
| 1 << VIRTIO_BLK_F_FLUSH
|
|
|
|
| 1 << VIRTIO_BLK_F_TOPOLOGY
|
2020-02-14 11:14:27 +00:00
|
|
|
| 1 << VIRTIO_RING_F_EVENT_IDX
|
2020-05-20 16:39:30 +00:00
|
|
|
| 1 << VIRTIO_BLK_F_CONFIG_WCE
|
2019-09-02 11:00:35 +00:00
|
|
|
| 1 << VIRTIO_F_VERSION_1
|
|
|
|
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
|
|
|
|
|
2020-02-03 06:24:41 +00:00
|
|
|
if vu_cfg.num_queues > 1 {
|
|
|
|
avail_features |= 1 << VIRTIO_BLK_F_MQ;
|
|
|
|
}
|
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
// Set vhost-user owner.
|
|
|
|
vhost_user_blk
|
|
|
|
.set_owner()
|
|
|
|
.map_err(Error::VhostUserSetOwner)?;
|
|
|
|
|
|
|
|
// Get features from backend, do negotiation to get a feature collection which
|
|
|
|
// both VMM and backend support.
|
|
|
|
let backend_features = vhost_user_blk
|
|
|
|
.get_features()
|
|
|
|
.map_err(Error::VhostUserGetFeatures)?;
|
|
|
|
avail_features &= backend_features;
|
|
|
|
// Set features back is required by the vhost crate mechanism, since the
|
|
|
|
// later vhost call will check if features is filled in master before execution.
|
|
|
|
vhost_user_blk
|
|
|
|
.set_features(avail_features)
|
|
|
|
.map_err(Error::VhostUserSetFeatures)?;
|
|
|
|
|
|
|
|
// Identify if protocol features are supported by the slave.
|
|
|
|
let mut acked_features = 0;
|
|
|
|
if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
|
|
|
|
acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
|
|
|
|
|
|
|
|
let mut protocol_features = vhost_user_blk
|
|
|
|
.get_protocol_features()
|
|
|
|
.map_err(Error::VhostUserGetProtocolFeatures)?;
|
|
|
|
protocol_features |= VhostUserProtocolFeatures::MQ;
|
|
|
|
protocol_features &= !VhostUserProtocolFeatures::INFLIGHT_SHMFD;
|
|
|
|
vhost_user_blk
|
|
|
|
.set_protocol_features(protocol_features)
|
|
|
|
.map_err(Error::VhostUserSetProtocolFeatures)?;
|
|
|
|
}
|
2020-02-03 06:24:41 +00:00
|
|
|
// Get the max queues number from backend, and the queue number set
|
|
|
|
// should be less than this max queue number.
|
|
|
|
let max_queues_num = vhost_user_blk
|
|
|
|
.get_queue_num()
|
|
|
|
.map_err(Error::VhostUserGetQueueMaxNum)?;
|
|
|
|
|
|
|
|
if vu_cfg.num_queues > max_queues_num as usize {
|
|
|
|
error!("vhost-user-blk has queue number: {} larger than the max queue number: {} backend allowed\n",
|
|
|
|
vu_cfg.num_queues, max_queues_num);
|
|
|
|
return Err(Error::BadQueueNum);
|
|
|
|
}
|
2020-01-27 17:12:58 +00:00
|
|
|
let config_len = mem::size_of::<VirtioBlockConfig>();
|
2019-09-11 03:22:48 +00:00
|
|
|
let config_space: Vec<u8> = vec![0u8; config_len as usize];
|
2020-01-27 17:12:58 +00:00
|
|
|
let (_, config_space) = vhost_user_blk
|
2019-09-11 03:22:48 +00:00
|
|
|
.get_config(
|
2020-03-03 11:43:48 +00:00
|
|
|
VHOST_USER_CONFIG_OFFSET,
|
2019-09-11 03:22:48 +00:00
|
|
|
config_len as u32,
|
|
|
|
VhostUserConfigFlags::WRITABLE,
|
|
|
|
config_space.as_slice(),
|
|
|
|
)
|
|
|
|
.unwrap();
|
2020-01-27 17:12:58 +00:00
|
|
|
let mut config = VirtioBlockConfig::default();
|
|
|
|
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
|
|
|
|
config = *backend_config;
|
|
|
|
// Only set num_queues value(u16).
|
|
|
|
config.num_queues = vu_cfg.num_queues as u16;
|
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2020-02-03 06:24:41 +00:00
|
|
|
// Send set_vring_base here, since it could tell backends, like SPDK,
|
|
|
|
// how many virt queues to be handled, which backend required to know
|
|
|
|
// at early stage.
|
|
|
|
for i in 0..vu_cfg.num_queues {
|
|
|
|
vhost_user_blk
|
|
|
|
.set_vring_base(i, 0)
|
|
|
|
.map_err(Error::VhostUserSetVringBase)?;
|
|
|
|
}
|
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
Ok(Blk {
|
2020-09-03 09:37:36 +00:00
|
|
|
common: VirtioCommon {
|
2020-09-04 08:37:37 +00:00
|
|
|
device_type: VirtioDeviceType::TYPE_BLOCK as u32,
|
|
|
|
queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
|
2020-09-03 09:37:36 +00:00
|
|
|
avail_features,
|
|
|
|
acked_features,
|
2020-09-04 08:37:37 +00:00
|
|
|
paused_sync: Some(Arc::new(Barrier::new(vu_cfg.num_queues + 1))),
|
2021-01-19 06:11:07 +00:00
|
|
|
min_queues: 1,
|
2020-09-03 15:56:32 +00:00
|
|
|
..Default::default()
|
2020-09-03 09:37:36 +00:00
|
|
|
},
|
2020-04-27 11:56:26 +00:00
|
|
|
id,
|
2019-09-02 11:00:35 +00:00
|
|
|
vhost_user_blk,
|
2020-01-27 17:12:58 +00:00
|
|
|
config,
|
2020-08-18 04:59:27 +00:00
|
|
|
seccomp_action,
|
2019-09-02 11:00:35 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Drop for Blk {
|
|
|
|
fn drop(&mut self) {
|
2020-09-04 08:37:37 +00:00
|
|
|
if let Some(kill_evt) = self.common.kill_evt.take() {
|
2019-09-19 16:14:55 +00:00
|
|
|
if let Err(e) = kill_evt.write(1) {
|
|
|
|
error!("failed to kill vhost-user-blk: {:?}", e);
|
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl VirtioDevice for Blk {
|
|
|
|
fn device_type(&self) -> u32 {
|
2020-09-04 08:37:37 +00:00
|
|
|
self.common.device_type
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn queue_max_sizes(&self) -> &[u16] {
|
2020-09-04 08:37:37 +00:00
|
|
|
&self.common.queue_sizes
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2020-01-23 10:14:38 +00:00
|
|
|
fn features(&self) -> u64 {
|
2020-09-03 09:37:36 +00:00
|
|
|
self.common.avail_features
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2020-01-23 10:14:38 +00:00
|
|
|
fn ack_features(&mut self, value: u64) {
|
2020-09-03 09:37:36 +00:00
|
|
|
self.common.ack_features(value)
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2020-07-16 09:34:51 +00:00
|
|
|
fn read_config(&self, offset: u64, data: &mut [u8]) {
|
|
|
|
self.read_config_from_slice(self.config.as_slice(), offset, data);
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn write_config(&mut self, offset: u64, data: &[u8]) {
|
2020-07-16 10:00:48 +00:00
|
|
|
// The "writeback" field is the only mutable field
|
|
|
|
let writeback_offset =
|
|
|
|
(&self.config.writeback as *const _ as u64) - (&self.config as *const _ as u64);
|
|
|
|
if offset != writeback_offset || data.len() != std::mem::size_of_val(&self.config.writeback)
|
|
|
|
{
|
|
|
|
error!(
|
|
|
|
"Attempt to write to read-only field: offset {:x} length {}",
|
|
|
|
offset,
|
|
|
|
data.len()
|
|
|
|
);
|
2019-09-02 11:00:35 +00:00
|
|
|
return;
|
|
|
|
}
|
2020-07-16 10:00:48 +00:00
|
|
|
|
|
|
|
self.config.writeback = data[0];
|
2019-09-05 07:39:57 +00:00
|
|
|
self.vhost_user_blk
|
|
|
|
.set_config(offset as u32, VhostUserConfigFlags::WRITABLE, data)
|
|
|
|
.expect("Failed to set config");
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn activate(
|
|
|
|
&mut self,
|
2020-02-11 16:22:40 +00:00
|
|
|
mem: GuestMemoryAtomic<GuestMemoryMmap>,
|
2020-01-13 17:52:19 +00:00
|
|
|
interrupt_cb: Arc<dyn VirtioInterrupt>,
|
2019-09-02 11:00:35 +00:00
|
|
|
queues: Vec<Queue>,
|
|
|
|
queue_evts: Vec<EventFd>,
|
|
|
|
) -> ActivateResult {
|
2020-09-04 08:37:37 +00:00
|
|
|
self.common.activate(&queues, &queue_evts, &interrupt_cb)?;
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2020-02-03 06:24:41 +00:00
|
|
|
let mut vu_interrupt_list = setup_vhost_user(
|
2019-09-02 11:00:35 +00:00
|
|
|
&mut self.vhost_user_blk,
|
2020-02-11 16:22:40 +00:00
|
|
|
&mem.memory(),
|
2019-09-02 11:00:35 +00:00
|
|
|
queues,
|
|
|
|
queue_evts,
|
2020-01-14 07:18:35 +00:00
|
|
|
&interrupt_cb,
|
2020-09-03 09:37:36 +00:00
|
|
|
self.common.acked_features,
|
2019-09-02 11:00:35 +00:00
|
|
|
)
|
|
|
|
.map_err(ActivateError::VhostUserBlkSetup)?;
|
|
|
|
|
2020-01-27 12:56:05 +00:00
|
|
|
let mut epoll_threads = Vec::new();
|
2021-01-13 13:10:36 +00:00
|
|
|
for i in 0..vu_interrupt_list.len() {
|
2020-02-03 06:24:41 +00:00
|
|
|
let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(1);
|
|
|
|
interrupt_list_sub.push(vu_interrupt_list.remove(0));
|
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
let kill_evt = self
|
|
|
|
.common
|
|
|
|
.kill_evt
|
|
|
|
.as_ref()
|
|
|
|
.unwrap()
|
|
|
|
.try_clone()
|
|
|
|
.map_err(|e| {
|
|
|
|
error!("failed to clone kill_evt eventfd: {}", e);
|
|
|
|
ActivateError::BadActivate
|
|
|
|
})?;
|
|
|
|
let pause_evt = self
|
|
|
|
.common
|
|
|
|
.pause_evt
|
|
|
|
.as_ref()
|
|
|
|
.unwrap()
|
|
|
|
.try_clone()
|
|
|
|
.map_err(|e| {
|
|
|
|
error!("failed to clone pause_evt eventfd: {}", e);
|
|
|
|
ActivateError::BadActivate
|
|
|
|
})?;
|
|
|
|
|
2020-02-03 06:24:41 +00:00
|
|
|
let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
|
|
|
|
interrupt_cb: interrupt_cb.clone(),
|
2020-09-04 08:37:37 +00:00
|
|
|
kill_evt,
|
|
|
|
pause_evt,
|
2020-02-03 06:24:41 +00:00
|
|
|
vu_interrupt_list: interrupt_list_sub,
|
|
|
|
slave_req_handler: None,
|
|
|
|
});
|
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
let paused = self.common.paused.clone();
|
|
|
|
let paused_sync = self.common.paused_sync.clone();
|
2020-08-18 04:59:27 +00:00
|
|
|
let virtio_vhost_blk_seccomp_filter =
|
|
|
|
get_seccomp_filter(&self.seccomp_action, Thread::VirtioVhostBlk)
|
|
|
|
.map_err(ActivateError::CreateSeccompFilter)?;
|
2020-02-03 06:24:41 +00:00
|
|
|
thread::Builder::new()
|
2021-01-13 13:10:36 +00:00
|
|
|
.name(format!("{}_q{}", self.id.clone(), i))
|
2020-08-17 19:45:17 +00:00
|
|
|
.spawn(move || {
|
2020-08-18 04:59:27 +00:00
|
|
|
if let Err(e) = SeccompFilter::apply(virtio_vhost_blk_seccomp_filter) {
|
|
|
|
error!("Error applying seccomp filter: {:?}", e);
|
2020-09-04 08:37:37 +00:00
|
|
|
} else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
|
2020-08-17 19:45:17 +00:00
|
|
|
error!("Error running worker: {:?}", e);
|
|
|
|
}
|
|
|
|
})
|
2020-02-03 06:24:41 +00:00
|
|
|
.map(|thread| epoll_threads.push(thread))
|
|
|
|
.map_err(|e| {
|
|
|
|
error!("failed to clone virtio epoll thread: {}", e);
|
|
|
|
ActivateError::BadActivate
|
|
|
|
})?;
|
|
|
|
}
|
2020-09-04 08:37:37 +00:00
|
|
|
self.common.epoll_threads = Some(epoll_threads);
|
2020-01-27 12:56:05 +00:00
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
2019-09-19 16:14:55 +00:00
|
|
|
|
2021-01-18 12:38:08 +00:00
|
|
|
fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
|
2019-12-02 20:08:53 +00:00
|
|
|
// We first must resume the virtio thread if it was paused.
|
2020-09-04 08:37:37 +00:00
|
|
|
if self.common.pause_evt.take().is_some() {
|
|
|
|
self.common.resume().ok()?;
|
2019-12-02 20:08:53 +00:00
|
|
|
}
|
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
if let Err(e) = reset_vhost_user(&mut self.vhost_user_blk, self.common.queue_sizes.len()) {
|
2019-09-19 16:14:55 +00:00
|
|
|
error!("Failed to reset vhost-user daemon: {:?}", e);
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
if let Some(kill_evt) = self.common.kill_evt.take() {
|
2019-09-19 16:14:55 +00:00
|
|
|
// Ignore the result because there is nothing we can do about it.
|
|
|
|
let _ = kill_evt.write(1);
|
|
|
|
}
|
|
|
|
|
2021-01-18 12:38:08 +00:00
|
|
|
// Return the interrupt
|
|
|
|
Some(self.common.interrupt_cb.take().unwrap())
|
2019-09-19 16:14:55 +00:00
|
|
|
}
|
2020-02-14 10:08:59 +00:00
|
|
|
|
|
|
|
fn shutdown(&mut self) {
|
|
|
|
let _ = unsafe { libc::close(self.vhost_user_blk.as_raw_fd()) };
|
|
|
|
}
|
2020-03-23 12:47:55 +00:00
|
|
|
|
|
|
|
fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
|
|
|
|
update_mem_table(&mut self.vhost_user_blk, mem).map_err(crate::Error::VhostUserUpdateMemory)
|
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
2019-12-02 20:08:53 +00:00
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
impl Pausable for Blk {
|
|
|
|
fn pause(&mut self) -> result::Result<(), MigratableError> {
|
|
|
|
self.common.pause()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn resume(&mut self) -> result::Result<(), MigratableError> {
|
|
|
|
self.common.resume()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-27 11:56:26 +00:00
|
|
|
impl Snapshottable for Blk {
|
|
|
|
fn id(&self) -> String {
|
|
|
|
self.id.clone()
|
|
|
|
}
|
|
|
|
}
|
2019-05-01 16:59:51 +00:00
|
|
|
impl Transportable for Blk {}
|
2019-12-02 20:08:53 +00:00
|
|
|
impl Migratable for Blk {}
|