2019-09-02 11:00:35 +00:00
|
|
|
// Copyright 2019 Intel Corporation. All Rights Reserved.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
2020-09-03 09:37:36 +00:00
|
|
|
use super::super::{
|
|
|
|
ActivateError, ActivateResult, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType,
|
|
|
|
};
|
2021-05-21 16:26:00 +00:00
|
|
|
use super::vu_common_ctrl::{
|
|
|
|
add_memory_region, negotiate_features_vhost_user, reset_vhost_user, setup_vhost_user,
|
|
|
|
update_mem_table, VhostUserConfig,
|
|
|
|
};
|
|
|
|
use super::{Error, Result, DEFAULT_VIRTIO_FEATURES};
|
2019-12-31 10:49:11 +00:00
|
|
|
use crate::VirtioInterrupt;
|
2021-06-02 19:08:04 +00:00
|
|
|
use crate::{GuestMemoryMmap, GuestRegionMmap};
|
2020-07-14 09:43:33 +00:00
|
|
|
use block_util::VirtioBlockConfig;
|
2019-12-31 10:49:11 +00:00
|
|
|
use std::mem;
|
2021-03-05 13:32:57 +00:00
|
|
|
use std::ops::Deref;
|
2020-02-14 10:08:59 +00:00
|
|
|
use std::os::unix::io::AsRawFd;
|
2019-12-02 20:08:53 +00:00
|
|
|
use std::result;
|
2020-08-11 14:05:06 +00:00
|
|
|
use std::sync::{Arc, Barrier};
|
2019-09-02 11:00:35 +00:00
|
|
|
use std::vec::Vec;
|
2021-02-25 09:46:27 +00:00
|
|
|
use vhost::vhost_user::message::VhostUserConfigFlags;
|
|
|
|
use vhost::vhost_user::message::VHOST_USER_CONFIG_OFFSET;
|
|
|
|
use vhost::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
|
|
|
|
use vhost::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
|
|
|
|
use vhost::VhostBackend;
|
2021-05-21 16:26:00 +00:00
|
|
|
use virtio_bindings::bindings::virtio_blk::{
|
|
|
|
VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_FLUSH,
|
|
|
|
VIRTIO_BLK_F_GEOMETRY, VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_RO, VIRTIO_BLK_F_SEG_MAX,
|
|
|
|
VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_WRITE_ZEROES,
|
|
|
|
};
|
2021-06-02 19:08:04 +00:00
|
|
|
use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic};
|
2019-05-01 16:59:51 +00:00
|
|
|
use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
|
2019-12-31 10:49:11 +00:00
|
|
|
use vmm_sys_util::eventfd::EventFd;
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2021-05-19 20:33:04 +00:00
|
|
|
const DEFAULT_QUEUE_NUMBER: usize = 1;
|
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
struct SlaveReqHandler {}
|
|
|
|
impl VhostUserMasterReqHandler for SlaveReqHandler {}
|
|
|
|
|
|
|
|
pub struct Blk {
|
2020-09-03 09:37:36 +00:00
|
|
|
common: VirtioCommon,
|
2020-04-27 11:56:26 +00:00
|
|
|
id: String,
|
2019-09-02 11:00:35 +00:00
|
|
|
vhost_user_blk: Master,
|
2020-01-27 17:12:58 +00:00
|
|
|
config: VirtioBlockConfig,
|
2021-03-05 13:32:57 +00:00
|
|
|
guest_memory: Option<GuestMemoryAtomic<GuestMemoryMmap>>,
|
|
|
|
acked_protocol_features: u64,
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2019-09-23 17:42:52 +00:00
|
|
|
impl Blk {
|
2019-09-02 11:00:35 +00:00
|
|
|
/// Create a new vhost-user-blk device
|
2021-05-19 20:33:04 +00:00
|
|
|
pub fn new(id: String, vu_cfg: VhostUserConfig) -> Result<Blk> {
|
|
|
|
let num_queues = vu_cfg.num_queues;
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2021-05-19 21:58:02 +00:00
|
|
|
let mut vhost_user_blk = Master::connect(&vu_cfg.socket, num_queues as u64)
|
|
|
|
.map_err(Error::VhostUserCreateMaster)?;
|
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
// Filling device and vring features VMM supports.
|
2021-05-21 16:26:00 +00:00
|
|
|
let mut avail_features = 1 << VIRTIO_BLK_F_SIZE_MAX
|
|
|
|
| 1 << VIRTIO_BLK_F_SEG_MAX
|
|
|
|
| 1 << VIRTIO_BLK_F_GEOMETRY
|
2019-09-02 11:00:35 +00:00
|
|
|
| 1 << VIRTIO_BLK_F_RO
|
|
|
|
| 1 << VIRTIO_BLK_F_BLK_SIZE
|
|
|
|
| 1 << VIRTIO_BLK_F_FLUSH
|
|
|
|
| 1 << VIRTIO_BLK_F_TOPOLOGY
|
2020-05-20 16:39:30 +00:00
|
|
|
| 1 << VIRTIO_BLK_F_CONFIG_WCE
|
2021-05-21 16:26:00 +00:00
|
|
|
| 1 << VIRTIO_BLK_F_DISCARD
|
|
|
|
| 1 << VIRTIO_BLK_F_WRITE_ZEROES
|
|
|
|
| DEFAULT_VIRTIO_FEATURES;
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2021-05-19 20:33:04 +00:00
|
|
|
if num_queues > 1 {
|
2020-02-03 06:24:41 +00:00
|
|
|
avail_features |= 1 << VIRTIO_BLK_F_MQ;
|
|
|
|
}
|
|
|
|
|
2021-05-19 20:33:04 +00:00
|
|
|
let avail_protocol_features = VhostUserProtocolFeatures::CONFIG
|
|
|
|
| VhostUserProtocolFeatures::MQ
|
|
|
|
| VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS
|
|
|
|
| VhostUserProtocolFeatures::REPLY_ACK;
|
|
|
|
|
2021-05-19 21:58:02 +00:00
|
|
|
let (acked_features, acked_protocol_features) = negotiate_features_vhost_user(
|
|
|
|
&mut vhost_user_blk,
|
|
|
|
avail_features,
|
|
|
|
avail_protocol_features,
|
|
|
|
)?;
|
2021-05-19 20:33:04 +00:00
|
|
|
|
|
|
|
let backend_num_queues =
|
|
|
|
if acked_protocol_features & VhostUserProtocolFeatures::MQ.bits() != 0 {
|
|
|
|
vhost_user_blk
|
|
|
|
.get_queue_num()
|
|
|
|
.map_err(Error::VhostUserGetQueueMaxNum)? as usize
|
|
|
|
} else {
|
|
|
|
DEFAULT_QUEUE_NUMBER
|
|
|
|
};
|
|
|
|
|
|
|
|
if num_queues > backend_num_queues {
|
|
|
|
error!("vhost-user-blk requested too many queues ({}) since the backend only supports {}\n",
|
|
|
|
num_queues, backend_num_queues);
|
2020-02-03 06:24:41 +00:00
|
|
|
return Err(Error::BadQueueNum);
|
|
|
|
}
|
2021-05-19 20:33:04 +00:00
|
|
|
|
2020-01-27 17:12:58 +00:00
|
|
|
let config_len = mem::size_of::<VirtioBlockConfig>();
|
2019-09-11 03:22:48 +00:00
|
|
|
let config_space: Vec<u8> = vec![0u8; config_len as usize];
|
2020-01-27 17:12:58 +00:00
|
|
|
let (_, config_space) = vhost_user_blk
|
2019-09-11 03:22:48 +00:00
|
|
|
.get_config(
|
2020-03-03 11:43:48 +00:00
|
|
|
VHOST_USER_CONFIG_OFFSET,
|
2019-09-11 03:22:48 +00:00
|
|
|
config_len as u32,
|
|
|
|
VhostUserConfigFlags::WRITABLE,
|
|
|
|
config_space.as_slice(),
|
|
|
|
)
|
2021-05-19 20:33:04 +00:00
|
|
|
.map_err(Error::VhostUserGetConfig)?;
|
2020-01-27 17:12:58 +00:00
|
|
|
let mut config = VirtioBlockConfig::default();
|
|
|
|
if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
|
|
|
|
config = *backend_config;
|
2021-05-19 20:33:04 +00:00
|
|
|
config.num_queues = num_queues as u16;
|
2020-01-27 17:12:58 +00:00
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2020-02-03 06:24:41 +00:00
|
|
|
// Send set_vring_base here, since it could tell backends, like SPDK,
|
|
|
|
// how many virt queues to be handled, which backend required to know
|
|
|
|
// at early stage.
|
2021-05-19 20:33:04 +00:00
|
|
|
for i in 0..num_queues {
|
2020-02-03 06:24:41 +00:00
|
|
|
vhost_user_blk
|
|
|
|
.set_vring_base(i, 0)
|
|
|
|
.map_err(Error::VhostUserSetVringBase)?;
|
|
|
|
}
|
|
|
|
|
2019-09-02 11:00:35 +00:00
|
|
|
Ok(Blk {
|
2020-09-03 09:37:36 +00:00
|
|
|
common: VirtioCommon {
|
2021-03-25 16:54:09 +00:00
|
|
|
device_type: VirtioDeviceType::Block as u32,
|
2021-05-19 20:33:04 +00:00
|
|
|
queue_sizes: vec![vu_cfg.queue_size; num_queues],
|
|
|
|
avail_features: acked_features,
|
|
|
|
acked_features: 0,
|
|
|
|
paused_sync: Some(Arc::new(Barrier::new(1))),
|
|
|
|
min_queues: DEFAULT_QUEUE_NUMBER as u16,
|
2020-09-03 15:56:32 +00:00
|
|
|
..Default::default()
|
2020-09-03 09:37:36 +00:00
|
|
|
},
|
2020-04-27 11:56:26 +00:00
|
|
|
id,
|
2019-09-02 11:00:35 +00:00
|
|
|
vhost_user_blk,
|
2020-01-27 17:12:58 +00:00
|
|
|
config,
|
2021-03-05 13:32:57 +00:00
|
|
|
guest_memory: None,
|
|
|
|
acked_protocol_features,
|
2019-09-02 11:00:35 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Drop for Blk {
|
|
|
|
fn drop(&mut self) {
|
2020-09-04 08:37:37 +00:00
|
|
|
if let Some(kill_evt) = self.common.kill_evt.take() {
|
2019-09-19 16:14:55 +00:00
|
|
|
if let Err(e) = kill_evt.write(1) {
|
|
|
|
error!("failed to kill vhost-user-blk: {:?}", e);
|
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl VirtioDevice for Blk {
|
|
|
|
fn device_type(&self) -> u32 {
|
2020-09-04 08:37:37 +00:00
|
|
|
self.common.device_type
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn queue_max_sizes(&self) -> &[u16] {
|
2020-09-04 08:37:37 +00:00
|
|
|
&self.common.queue_sizes
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2020-01-23 10:14:38 +00:00
|
|
|
fn features(&self) -> u64 {
|
2020-09-03 09:37:36 +00:00
|
|
|
self.common.avail_features
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2020-01-23 10:14:38 +00:00
|
|
|
fn ack_features(&mut self, value: u64) {
|
2020-09-03 09:37:36 +00:00
|
|
|
self.common.ack_features(value)
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
2020-07-16 09:34:51 +00:00
|
|
|
fn read_config(&self, offset: u64, data: &mut [u8]) {
|
|
|
|
self.read_config_from_slice(self.config.as_slice(), offset, data);
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn write_config(&mut self, offset: u64, data: &[u8]) {
|
2020-07-16 10:00:48 +00:00
|
|
|
// The "writeback" field is the only mutable field
|
|
|
|
let writeback_offset =
|
|
|
|
(&self.config.writeback as *const _ as u64) - (&self.config as *const _ as u64);
|
|
|
|
if offset != writeback_offset || data.len() != std::mem::size_of_val(&self.config.writeback)
|
|
|
|
{
|
|
|
|
error!(
|
|
|
|
"Attempt to write to read-only field: offset {:x} length {}",
|
|
|
|
offset,
|
|
|
|
data.len()
|
|
|
|
);
|
2019-09-02 11:00:35 +00:00
|
|
|
return;
|
|
|
|
}
|
2020-07-16 10:00:48 +00:00
|
|
|
|
|
|
|
self.config.writeback = data[0];
|
2021-05-19 20:33:04 +00:00
|
|
|
if let Err(e) = self
|
|
|
|
.vhost_user_blk
|
2019-09-05 07:39:57 +00:00
|
|
|
.set_config(offset as u32, VhostUserConfigFlags::WRITABLE, data)
|
2021-05-19 20:33:04 +00:00
|
|
|
.map_err(Error::VhostUserSetConfig)
|
|
|
|
{
|
|
|
|
error!("Failed setting vhost-user-blk configuration: {:?}", e);
|
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn activate(
|
|
|
|
&mut self,
|
2020-02-11 16:22:40 +00:00
|
|
|
mem: GuestMemoryAtomic<GuestMemoryMmap>,
|
2020-01-13 17:52:19 +00:00
|
|
|
interrupt_cb: Arc<dyn VirtioInterrupt>,
|
2019-09-02 11:00:35 +00:00
|
|
|
queues: Vec<Queue>,
|
|
|
|
queue_evts: Vec<EventFd>,
|
|
|
|
) -> ActivateResult {
|
2020-09-04 08:37:37 +00:00
|
|
|
self.common.activate(&queues, &queue_evts, &interrupt_cb)?;
|
2019-09-02 11:00:35 +00:00
|
|
|
|
2021-03-05 13:32:57 +00:00
|
|
|
self.guest_memory = Some(mem.clone());
|
|
|
|
|
2021-05-12 15:13:08 +00:00
|
|
|
setup_vhost_user(
|
2019-09-02 11:00:35 +00:00
|
|
|
&mut self.vhost_user_blk,
|
2020-02-11 16:22:40 +00:00
|
|
|
&mem.memory(),
|
2019-09-02 11:00:35 +00:00
|
|
|
queues,
|
|
|
|
queue_evts,
|
2020-01-14 07:18:35 +00:00
|
|
|
&interrupt_cb,
|
2021-05-20 21:43:40 +00:00
|
|
|
self.common.acked_features
|
|
|
|
| (self.common.avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()),
|
2019-09-02 11:00:35 +00:00
|
|
|
)
|
|
|
|
.map_err(ActivateError::VhostUserBlkSetup)?;
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
2019-09-19 16:14:55 +00:00
|
|
|
|
2021-01-18 12:38:08 +00:00
|
|
|
fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
|
2019-12-02 20:08:53 +00:00
|
|
|
// We first must resume the virtio thread if it was paused.
|
2020-09-04 08:37:37 +00:00
|
|
|
if self.common.pause_evt.take().is_some() {
|
|
|
|
self.common.resume().ok()?;
|
2019-12-02 20:08:53 +00:00
|
|
|
}
|
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
if let Err(e) = reset_vhost_user(&mut self.vhost_user_blk, self.common.queue_sizes.len()) {
|
2019-09-19 16:14:55 +00:00
|
|
|
error!("Failed to reset vhost-user daemon: {:?}", e);
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
if let Some(kill_evt) = self.common.kill_evt.take() {
|
2019-09-19 16:14:55 +00:00
|
|
|
// Ignore the result because there is nothing we can do about it.
|
|
|
|
let _ = kill_evt.write(1);
|
|
|
|
}
|
|
|
|
|
2021-02-18 15:10:51 +00:00
|
|
|
event!("virtio-device", "reset", "id", &self.id);
|
|
|
|
|
2021-01-18 12:38:08 +00:00
|
|
|
// Return the interrupt
|
|
|
|
Some(self.common.interrupt_cb.take().unwrap())
|
2019-09-19 16:14:55 +00:00
|
|
|
}
|
2020-02-14 10:08:59 +00:00
|
|
|
|
|
|
|
fn shutdown(&mut self) {
|
|
|
|
let _ = unsafe { libc::close(self.vhost_user_blk.as_raw_fd()) };
|
|
|
|
}
|
2020-03-23 12:47:55 +00:00
|
|
|
|
2021-03-05 13:32:57 +00:00
|
|
|
fn add_memory_region(
|
|
|
|
&mut self,
|
|
|
|
region: &Arc<GuestRegionMmap>,
|
|
|
|
) -> std::result::Result<(), crate::Error> {
|
|
|
|
if self.acked_protocol_features & VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS.bits() != 0
|
|
|
|
{
|
|
|
|
add_memory_region(&mut self.vhost_user_blk, region)
|
|
|
|
.map_err(crate::Error::VhostUserAddMemoryRegion)
|
|
|
|
} else if let Some(guest_memory) = &self.guest_memory {
|
|
|
|
update_mem_table(&mut self.vhost_user_blk, guest_memory.memory().deref())
|
|
|
|
.map_err(crate::Error::VhostUserUpdateMemory)
|
|
|
|
} else {
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
2019-09-02 11:00:35 +00:00
|
|
|
}
|
2019-12-02 20:08:53 +00:00
|
|
|
|
2020-09-04 08:37:37 +00:00
|
|
|
impl Pausable for Blk {
|
|
|
|
fn pause(&mut self) -> result::Result<(), MigratableError> {
|
|
|
|
self.common.pause()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn resume(&mut self) -> result::Result<(), MigratableError> {
|
|
|
|
self.common.resume()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-27 11:56:26 +00:00
|
|
|
impl Snapshottable for Blk {
|
|
|
|
fn id(&self) -> String {
|
|
|
|
self.id.clone()
|
|
|
|
}
|
|
|
|
}
|
2019-05-01 16:59:51 +00:00
|
|
|
impl Transportable for Blk {}
|
2019-12-02 20:08:53 +00:00
|
|
|
impl Migratable for Blk {}
|