From 1dd24518957206b4e705b35fe498e38bf57a2821 Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Mon, 20 Jan 2020 16:10:21 +0000 Subject: [PATCH] vhost_user_block: Refactor vhost_user_block backend code into a new crate Extract the majority of the code that provides the vhost-user-block backend into its own crate and port the binary to use it. Signed-off-by: Rob Bradford --- Cargo.lock | 18 +++ Cargo.toml | 3 +- src/bin/vhost_user_blk.rs | 268 +--------------------------------- vhost_user_block/Cargo.toml | 18 +++ vhost_user_block/src/lib.rs | 276 ++++++++++++++++++++++++++++++++++++ 5 files changed, 321 insertions(+), 262 deletions(-) create mode 100644 vhost_user_block/Cargo.toml create mode 100644 vhost_user_block/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 04720ea8a..f03e6f39e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,6 +185,7 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "vhost_rs 0.1.0", "vhost_user_backend 0.1.0", + "vhost_user_block 0.1.0", "vhost_user_fs 0.1.0", "vhost_user_net 0.1.0", "virtio-bindings 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1026,6 +1027,23 @@ dependencies = [ "vmm-sys-util 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "vhost_user_block" +version = "0.1.0" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "epoll 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "qcow 0.1.0", + "vhost_rs 0.1.0", + "vhost_user_backend 0.1.0", + "virtio-bindings 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)", + "vm-virtio 0.1.0", + "vmm-sys-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "vhost_user_fs" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 50ebe415a..f33e489cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,8 @@ epoll = ">=4.0.1" lazy_static = "1.4.0" libc = "0.2.66" log = { version = "0.4.10", features = ["std"] } -qcow = { path = "qcow" } vhost_user_backend = { path = "vhost_user_backend"} +vhost_user_block = { path = "vhost_user_block"} vhost_user_fs = { path = "vhost_user_fs"} vhost_user_net = { path = "vhost_user_net"} virtio-bindings = "0.1.0" @@ -55,6 +55,7 @@ members = [ "vmm", "vm-virtio", "vm-device", + "vhost_user_block", "vhost_user_backend", "vhost_user_fs", "vhost_user_net", diff --git a/src/bin/vhost_user_blk.rs b/src/bin/vhost_user_blk.rs index 84d782220..a696ccfca 100644 --- a/src/bin/vhost_user_blk.rs +++ b/src/bin/vhost_user_blk.rs @@ -11,272 +11,15 @@ #[macro_use(crate_version, crate_authors)] extern crate clap; extern crate log; -extern crate vhost_rs; extern crate vhost_user_backend; -extern crate vm_virtio; +extern crate vhost_user_block; use clap::{App, Arg}; -use epoll; use log::*; -use std::fs::File; -use std::fs::OpenOptions; -use std::io::Read; -use std::io::{Seek, SeekFrom, Write}; -use std::mem; -use std::os::unix::fs::OpenOptionsExt; -use std::path::PathBuf; use std::process; -use std::slice; use std::sync::{Arc, RwLock}; -use std::vec::Vec; - -use qcow::{self, ImageType, QcowFile}; - -use vhost_rs::vhost_user::message::*; -use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker}; - -use virtio_bindings::bindings::virtio_blk::*; -use vm_memory::{Bytes, GuestMemoryError, GuestMemoryMmap}; -use vm_virtio::block::{build_disk_image_id, Request}; - -const QUEUE_SIZE: usize = 1024; -const NUM_QUEUES: usize = 1; -const SECTOR_SHIFT: u8 = 9; -const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT; -const BLK_SIZE: u32 = 512; - -trait DiskFile: Read + Seek + Write + Send + Sync {} -impl DiskFile for D {} - -pub type Result = std::result::Result; -pub type VhostUserBackendResult = std::result::Result; - -#[derive(Debug)] -pub enum Error { - /// Failed to detect image type. - DetectImageType, - /// Bad memory address. - GuestMemory(GuestMemoryError), - /// Can't open image file. - OpenImage, - /// Failed to parse direct parameter. - ParseDirectParam, - /// Failed to parse image parameter. - ParseImageParam, - /// Failed to parse sock parameter. - ParseSockParam, - /// Failed to parse readonly parameter. - ParseReadOnlyParam, -} - -struct VhostUserBlkBackend { - mem: Option, - vring_worker: Option>, - disk_image: Box, - disk_image_id: Vec, - disk_nsectors: u64, - config: virtio_blk_config, - rdonly: bool, -} - -impl VhostUserBlkBackend { - pub fn new(image_path: String, rdonly: bool, direct: bool) -> Result { - let mut options = OpenOptions::new(); - options.read(true); - options.write(!rdonly); - if direct { - options.custom_flags(libc::O_DIRECT); - } - let image: File = options.open(&image_path).unwrap(); - let mut raw_img: vm_virtio::RawFile = vm_virtio::RawFile::new(image, direct); - - let image_id = build_disk_image_id(&PathBuf::from(&image_path)); - let image_type = qcow::detect_image_type(&mut raw_img).unwrap(); - let mut image = match image_type { - ImageType::Raw => Box::new(raw_img) as Box, - ImageType::Qcow2 => Box::new(QcowFile::from(raw_img).unwrap()) as Box, - }; - - let nsectors = (image.seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE; - let mut config = virtio_blk_config::default(); - - config.capacity = nsectors; - config.blk_size = BLK_SIZE; - config.size_max = 65535; - config.seg_max = 128 - 2; - config.min_io_size = 1; - config.opt_io_size = 1; - config.num_queues = 1; - config.wce = 1; - - Ok(VhostUserBlkBackend { - mem: None, - vring_worker: None, - disk_image: image, - disk_image_id: image_id, - disk_nsectors: nsectors, - config, - rdonly, - }) - } - - pub fn process_queue(&mut self, vring: &mut Vring) -> bool { - let mut used_any = false; - let mem = match self.mem.as_ref() { - Some(m) => m, - None => return false, - }; - - while let Some(head) = vring.mut_queue().iter(mem).next() { - debug!("got an element in the queue"); - let len; - match Request::parse(&head, mem) { - Ok(request) => { - debug!("element is a valid request"); - let status = match request.execute( - &mut self.disk_image, - self.disk_nsectors, - mem, - &self.disk_image_id, - ) { - Ok(l) => { - len = l; - VIRTIO_BLK_S_OK - } - Err(e) => { - len = 1; - e.status() - } - }; - mem.write_obj(status, request.status_addr).unwrap(); - } - Err(err) => { - error!("failed to parse available descriptor chain: {:?}", err); - len = 0; - } - } - vring.mut_queue().add_used(mem, head.index, len); - used_any = true; - } - - used_any - } -} - -impl VhostUserBackend for VhostUserBlkBackend { - fn num_queues(&self) -> usize { - NUM_QUEUES - } - - fn max_queue_size(&self) -> usize { - QUEUE_SIZE - } - - fn features(&self) -> u64 { - let mut avail_features = 1 << VIRTIO_BLK_F_MQ - | 1 << VIRTIO_BLK_F_CONFIG_WCE - | 1 << VIRTIO_F_VERSION_1 - | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); - - if self.rdonly { - avail_features |= 1 << VIRTIO_BLK_F_RO; - } - avail_features - } - - fn protocol_features(&self) -> VhostUserProtocolFeatures { - VhostUserProtocolFeatures::CONFIG - } - - fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> { - self.mem = Some(mem); - Ok(()) - } - - fn handle_event( - &mut self, - device_event: u16, - evset: epoll::Events, - vrings: &[Arc>], - ) -> VhostUserBackendResult { - if evset != epoll::Events::EPOLLIN { - warn!("invalid events operation"); - return Ok(false); - } - - debug!("event received: {:?}", device_event); - - let mut vring = vrings[0].write().unwrap(); - if self.process_queue(&mut vring) { - debug!("signalling queue"); - vring.signal_used_queue().unwrap(); - } - - Ok(false) - } - - fn get_config(&self, _offset: u32, _size: u32) -> Vec { - // self.config is a statically allocated virtio_blk_config - let buf = unsafe { - slice::from_raw_parts( - &self.config as *const virtio_blk_config as *const _, - mem::size_of::(), - ) - }; - - buf.to_vec() - } -} - -pub struct VhostUserBlkBackendConfig<'a> { - pub image: &'a str, - pub sock: &'a str, - pub readonly: bool, - pub direct: bool, -} - -impl<'a> VhostUserBlkBackendConfig<'a> { - pub fn parse(backend: &'a str) -> Result { - let params_list: Vec<&str> = backend.split(',').collect(); - - let mut image: &str = ""; - let mut sock: &str = ""; - let mut readonly: bool = false; - let mut direct: bool = false; - - for param in params_list.iter() { - if param.starts_with("image=") { - image = ¶m[6..]; - } else if param.starts_with("sock=") { - sock = ¶m[5..]; - } else if param.starts_with("readonly=") { - readonly = match param[9..].parse::() { - Ok(b) => b, - Err(_) => return Err(Error::ParseReadOnlyParam), - } - } else if param.starts_with("direct=") { - direct = match param[7..].parse::() { - Ok(b) => b, - Err(_) => return Err(Error::ParseDirectParam), - } - } - } - - if image.is_empty() { - return Err(Error::ParseImageParam); - } - if sock.is_empty() { - return Err(Error::ParseSockParam); - } - - Ok(VhostUserBlkBackendConfig { - image, - sock, - readonly, - direct, - }) - } -} +use vhost_user_backend::VhostUserDaemon; +use vhost_user_block::{VhostUserBlkBackend, VhostUserBlkBackendConfig}; fn main() { let cmd_arguments = App::new("vhost-user-blk backend") @@ -328,7 +71,10 @@ fn main() { let vring_worker = blk_daemon.get_vring_worker(); - blk_backend.write().unwrap().vring_worker = Some(vring_worker); + blk_backend + .write() + .unwrap() + .set_vring_worker(Some(vring_worker)); if let Err(e) = blk_daemon.start() { println!( diff --git a/vhost_user_block/Cargo.toml b/vhost_user_block/Cargo.toml new file mode 100644 index 000000000..2b7f2ccec --- /dev/null +++ b/vhost_user_block/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "vhost_user_block" +version = "0.1.0" +authors = ["The Cloud Hypervisor Authors"] +edition = "2018" + +[dependencies] +bitflags = "1.1.0" +epoll = ">=4.0.1" +libc = "0.2.65" +log = "0.4.8" +qcow = { path = "../qcow" } +vhost_user_backend = { path = "../vhost_user_backend" } +vhost_rs = { path = "../vhost_rs" } +virtio-bindings = "0.1.0" +vm-memory = { git = "https://github.com/rust-vmm/vm-memory" } +vm-virtio = { path = "../vm-virtio" } +vmm-sys-util = ">=0.3.1" \ No newline at end of file diff --git a/vhost_user_block/src/lib.rs b/vhost_user_block/src/lib.rs new file mode 100644 index 000000000..bfd2877f7 --- /dev/null +++ b/vhost_user_block/src/lib.rs @@ -0,0 +1,276 @@ +// Copyright 2019 Red Hat, Inc. All Rights Reserved. +// +// Portions Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Portions Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// +// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause) + +extern crate log; +extern crate vhost_rs; +extern crate vhost_user_backend; +extern crate vm_virtio; + +use epoll; +use log::*; +use qcow::{self, ImageType, QcowFile}; +use std::fs::File; +use std::fs::OpenOptions; +use std::io::Read; +use std::io::{Seek, SeekFrom, Write}; +use std::mem; +use std::os::unix::fs::OpenOptionsExt; +use std::path::PathBuf; +use std::slice; +use std::sync::{Arc, RwLock}; +use std::vec::Vec; +use vhost_rs::vhost_user::message::*; +use vhost_user_backend::{VhostUserBackend, Vring, VringWorker}; +use virtio_bindings::bindings::virtio_blk::*; +use vm_memory::{Bytes, GuestMemoryError, GuestMemoryMmap}; +use vm_virtio::block::{build_disk_image_id, Request}; + +const QUEUE_SIZE: usize = 1024; +const NUM_QUEUES: usize = 1; +const SECTOR_SHIFT: u8 = 9; +const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT; +const BLK_SIZE: u32 = 512; + +trait DiskFile: Read + Seek + Write + Send + Sync {} +impl DiskFile for D {} + +pub type Result = std::result::Result; +pub type VhostUserBackendResult = std::result::Result; + +#[derive(Debug)] +pub enum Error { + /// Failed to detect image type. + DetectImageType, + /// Bad memory address. + GuestMemory(GuestMemoryError), + /// Can't open image file. + OpenImage, + /// Failed to parse direct parameter. + ParseDirectParam, + /// Failed to parse image parameter. + ParseImageParam, + /// Failed to parse sock parameter. + ParseSockParam, + /// Failed to parse readonly parameter. + ParseReadOnlyParam, +} + +pub struct VhostUserBlkBackend { + mem: Option, + vring_worker: Option>, + disk_image: Box, + disk_image_id: Vec, + disk_nsectors: u64, + config: virtio_blk_config, + rdonly: bool, +} + +impl VhostUserBlkBackend { + pub fn new(image_path: String, rdonly: bool, direct: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + options.write(!rdonly); + if direct { + options.custom_flags(libc::O_DIRECT); + } + let image: File = options.open(&image_path).unwrap(); + let mut raw_img: vm_virtio::RawFile = vm_virtio::RawFile::new(image, direct); + + let image_id = build_disk_image_id(&PathBuf::from(&image_path)); + let image_type = qcow::detect_image_type(&mut raw_img).unwrap(); + let mut image = match image_type { + ImageType::Raw => Box::new(raw_img) as Box, + ImageType::Qcow2 => Box::new(QcowFile::from(raw_img).unwrap()) as Box, + }; + + let nsectors = (image.seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE; + let mut config = virtio_blk_config::default(); + + config.capacity = nsectors; + config.blk_size = BLK_SIZE; + config.size_max = 65535; + config.seg_max = 128 - 2; + config.min_io_size = 1; + config.opt_io_size = 1; + config.num_queues = 1; + config.wce = 1; + + Ok(VhostUserBlkBackend { + mem: None, + vring_worker: None, + disk_image: image, + disk_image_id: image_id, + disk_nsectors: nsectors, + config, + rdonly, + }) + } + + pub fn process_queue(&mut self, vring: &mut Vring) -> bool { + let mut used_any = false; + let mem = match self.mem.as_ref() { + Some(m) => m, + None => return false, + }; + + while let Some(head) = vring.mut_queue().iter(mem).next() { + debug!("got an element in the queue"); + let len; + match Request::parse(&head, mem) { + Ok(request) => { + debug!("element is a valid request"); + let status = match request.execute( + &mut self.disk_image, + self.disk_nsectors, + mem, + &self.disk_image_id, + ) { + Ok(l) => { + len = l; + VIRTIO_BLK_S_OK + } + Err(e) => { + len = 1; + e.status() + } + }; + mem.write_obj(status, request.status_addr).unwrap(); + } + Err(err) => { + error!("failed to parse available descriptor chain: {:?}", err); + len = 0; + } + } + vring.mut_queue().add_used(mem, head.index, len); + used_any = true; + } + + used_any + } + + pub fn set_vring_worker(&mut self, vring_worker: Option>) { + self.vring_worker = vring_worker; + } +} + +impl VhostUserBackend for VhostUserBlkBackend { + fn num_queues(&self) -> usize { + NUM_QUEUES + } + + fn max_queue_size(&self) -> usize { + QUEUE_SIZE + } + + fn features(&self) -> u64 { + let mut avail_features = 1 << VIRTIO_BLK_F_MQ + | 1 << VIRTIO_BLK_F_CONFIG_WCE + | 1 << VIRTIO_F_VERSION_1 + | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); + + if self.rdonly { + avail_features |= 1 << VIRTIO_BLK_F_RO; + } + avail_features + } + + fn protocol_features(&self) -> VhostUserProtocolFeatures { + VhostUserProtocolFeatures::CONFIG + } + + fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> { + self.mem = Some(mem); + Ok(()) + } + + fn handle_event( + &mut self, + device_event: u16, + evset: epoll::Events, + vrings: &[Arc>], + ) -> VhostUserBackendResult { + if evset != epoll::Events::EPOLLIN { + warn!("invalid events operation"); + return Ok(false); + } + + debug!("event received: {:?}", device_event); + + let mut vring = vrings[0].write().unwrap(); + if self.process_queue(&mut vring) { + debug!("signalling queue"); + vring.signal_used_queue().unwrap(); + } + + Ok(false) + } + + fn get_config(&self, _offset: u32, _size: u32) -> Vec { + // self.config is a statically allocated virtio_blk_config + let buf = unsafe { + slice::from_raw_parts( + &self.config as *const virtio_blk_config as *const _, + mem::size_of::(), + ) + }; + + buf.to_vec() + } +} + +pub struct VhostUserBlkBackendConfig<'a> { + pub image: &'a str, + pub sock: &'a str, + pub readonly: bool, + pub direct: bool, +} + +impl<'a> VhostUserBlkBackendConfig<'a> { + pub fn parse(backend: &'a str) -> Result { + let params_list: Vec<&str> = backend.split(',').collect(); + + let mut image: &str = ""; + let mut sock: &str = ""; + let mut readonly: bool = false; + let mut direct: bool = false; + + for param in params_list.iter() { + if param.starts_with("image=") { + image = ¶m[6..]; + } else if param.starts_with("sock=") { + sock = ¶m[5..]; + } else if param.starts_with("readonly=") { + readonly = match param[9..].parse::() { + Ok(b) => b, + Err(_) => return Err(Error::ParseReadOnlyParam), + } + } else if param.starts_with("direct=") { + direct = match param[7..].parse::() { + Ok(b) => b, + Err(_) => return Err(Error::ParseDirectParam), + } + } + } + + if image.is_empty() { + return Err(Error::ParseImageParam); + } + if sock.is_empty() { + return Err(Error::ParseSockParam); + } + + Ok(VhostUserBlkBackendConfig { + image, + sock, + readonly, + direct, + }) + } +}