block_util: Add utilities to support io_uring

Creates a dedicated function relying on io_uring crate to execute
io_uring specific requests.

Also creates a function for checking io_uring support on the host.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

Cargo.lock

@@ -148,6 +148,8 @@ dependencies = [
 name = "block_util"
 version = "0.1.0"
 dependencies = [
+ "io-uring",
+ "libc",
  "log 0.4.11",
  "serde",
  "serde_derive",
@@ -155,6 +157,7 @@ dependencies = [
  "virtio-bindings",
  "vm-memory",
  "vm-virtio",
+ "vmm-sys-util",
 ]
 
 [[package]]
@@ -485,6 +488,15 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "io-uring"
+version = "0.3.5"
+source = "git+https://github.com/tokio-rs/io-uring.git?branch=0.4#c00d968b038263a02a72d6510edaf438b0c7b4f3"
+dependencies = [
+ "bitflags 1.2.1",
+ "libc",
+]
+
 [[package]]
 name = "ipnetwork"
 version = "0.16.0"

block_util/Cargo.toml

@@ -5,6 +5,8 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
+io-uring = { git = "https://github.com/tokio-rs/io-uring.git", branch = "0.4" }
+libc = "0.2.74"
 log = "0.4.11"
 serde = ">=1.0.27"
 serde_derive = ">=1.0.27"
@@ -12,3 +14,4 @@ serde_json = ">=1.0.9"
 virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"]}
 vm-memory = { version = "0.2.1", features = ["backend-mmap", "backend-atomic"] }
 vm-virtio = { path = "../vm-virtio" }
+vmm-sys-util = ">=0.3.1"

block_util/src/lib.rs

@@ -13,15 +13,18 @@ extern crate log;
 #[macro_use]
 extern crate serde_derive;
 
+use io_uring::{opcode, IoUring, Probe};
 use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::cmp;
 use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::os::linux::fs::MetadataExt;
+use std::os::unix::io::{AsRawFd, RawFd};
 use std::path::PathBuf;
 use std::result;
 use virtio_bindings::bindings::virtio_blk::*;
 use vm_memory::{ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
 use vm_virtio::DescriptorChain;
+use vmm_sys_util::eventfd::EventFd;
 
 const SECTOR_SHIFT: u8 = 9;
 pub const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT;
@@ -86,6 +89,8 @@ pub enum ExecuteError {
     Seek(io::Error),
     Write(GuestMemoryError),
     Unsupported(u32),
+    SubmitIoUring(io::Error),
+    GetHostAddress(GuestMemoryError),
 }
 
 impl ExecuteError {
@@ -97,6 +102,8 @@ impl ExecuteError {
             ExecuteError::Seek(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::Write(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::Unsupported(_) => VIRTIO_BLK_S_UNSUPP,
+            ExecuteError::SubmitIoUring(_) => VIRTIO_BLK_S_IOERR,
+            ExecuteError::GetHostAddress(_) => VIRTIO_BLK_S_IOERR,
         }
     }
 }
@@ -136,11 +143,11 @@ fn sector(mem: &GuestMemoryMmap, desc_addr: GuestAddress) -> result::Result<u64,
 
 pub struct Request {
     pub request_type: RequestType,
-    sector: u64,
-    data_addr: GuestAddress,
+    pub sector: u64,
+    pub data_addr: GuestAddress,
     pub data_len: u32,
     pub status_addr: GuestAddress,
-    writeback: bool,
+    pub writeback: bool,
 }
 
 impl Request {
@@ -256,6 +263,95 @@ impl Request {
         Ok(0)
     }
 
+    pub fn execute_io_uring(
+        &self,
+        mem: &GuestMemoryMmap,
+        io_uring: &mut IoUring,
+        disk_nsectors: u64,
+        disk_image_fd: RawFd,
+        disk_id: &[u8],
+        user_data: u64,
+    ) -> result::Result<bool, ExecuteError> {
+        let data_len = self.data_len;
+        let sector = self.sector;
+        let data_addr = self.data_addr;
+        let request_type = self.request_type;
+
+        let mut top: u64 = u64::from(data_len) / SECTOR_SIZE;
+        if u64::from(data_len) % SECTOR_SIZE != 0 {
+            top += 1;
+        }
+        top = top
+            .checked_add(sector)
+            .ok_or(ExecuteError::BadRequest(Error::InvalidOffset))?;
+        if top > disk_nsectors {
+            return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+        }
+
+        let buf = mem
+            .get_slice(data_addr, data_len as usize)
+            .map_err(ExecuteError::GetHostAddress)?
+            .as_ptr();
+        let offset = (sector as i64) << SECTOR_SHIFT;
+
+        let (submitter, sq, _) = io_uring.split();
+        let mut avail_sq = sq.available();
+
+        // Queue operations expected to be submitted.
+        match request_type {
+            RequestType::In => {
+                // Safe because we know the file descriptor is valid and we
+                // relied on vm-memory to provide the buffer address.
+                let _ = unsafe {
+                    avail_sq.push(
+                        opcode::Read::new(opcode::types::Fd(disk_image_fd), buf, data_len)
+                            .offset(offset)
+                            .build()
+                            .user_data(user_data),
+                    )
+                };
+            }
+            RequestType::Out => {
+                // Safe because we know the file descriptor is valid and we
+                // relied on vm-memory to provide the buffer address.
+                let _ = unsafe {
+                    avail_sq.push(
+                        opcode::Write::new(opcode::types::Fd(disk_image_fd), buf, data_len)
+                            .offset(offset)
+                            .build()
+                            .user_data(user_data),
+                    )
+                };
+            }
+            RequestType::Flush => {
+                // Safe because we know the file descriptor is valid.
+                let _ = unsafe {
+                    avail_sq.push(
+                        opcode::Fsync::new(opcode::types::Fd(disk_image_fd))
+                            .build()
+                            .user_data(user_data),
+                    )
+                };
+            }
+            RequestType::GetDeviceID => {
+                if (data_len as usize) < disk_id.len() {
+                    return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+                }
+                mem.write_slice(disk_id, data_addr)
+                    .map_err(ExecuteError::Write)?;
+                return Ok(false);
+            }
+            RequestType::Unsupported(t) => return Err(ExecuteError::Unsupported(t)),
+        }
+
+        // Update the submission queue and submit new operations to the
+        // io_uring instance.
+        avail_sq.sync();
+        submitter.submit().map_err(ExecuteError::SubmitIoUring)?;
+
+        Ok(true)
+    }
+
     pub fn set_writeback(&mut self, writeback: bool) {
         self.writeback = writeback
     }
@@ -374,3 +470,71 @@ impl Serialize for VirtioBlockGeometry {
 }
 
 unsafe impl ByteValued for VirtioBlockGeometry {}
+
+/// Check if io_uring for block device can be used on the current system, as
+/// it correctly supports the expected io_uring features.
+pub fn block_io_uring_is_supported() -> bool {
+    let error_msg = "io_uring not supported:";
+
+    // Check we can create an io_uring instance, which effectively verifies
+    // that io_uring_setup() syscall is supported.
+    let io_uring = match IoUring::new(1) {
+        Ok(io_uring) => io_uring,
+        Err(e) => {
+            info!("{} failed to create io_uring instance: {}", error_msg, e);
+            return false;
+        }
+    };
+
+    let submitter = io_uring.submitter();
+
+    let event_fd = match EventFd::new(libc::EFD_NONBLOCK) {
+        Ok(fd) => fd,
+        Err(e) => {
+            info!("{} failed to create eventfd: {}", error_msg, e);
+            return false;
+        }
+    };
+
+    // Check we can register an eventfd as this is going to be needed while
+    // using io_uring with the virtio block device. This also validates that
+    // io_uring_register() syscall is supported.
+    match submitter.register_eventfd(event_fd.as_raw_fd()) {
+        Ok(_) => {}
+        Err(e) => {
+            info!("{} failed to register eventfd: {}", error_msg, e);
+            return false;
+        }
+    }
+
+    let mut probe = Probe::new();
+
+    // Check we can register a probe to validate supported operations.
+    match submitter.register_probe(&mut probe) {
+        Ok(_) => {}
+        Err(e) => {
+            info!("{} failed to register a probe: {}", error_msg, e);
+            return false;
+        }
+    }
+
+    // Check IORING_OP_FSYNC is supported
+    if !probe.is_supported(opcode::Fsync::CODE) {
+        info!("{} IORING_OP_FSYNC operation not supported", error_msg);
+        return false;
+    }
+
+    // Check IORING_OP_READ is supported
+    if !probe.is_supported(opcode::Read::CODE) {
+        info!("{} IORING_OP_READ operation not supported", error_msg);
+        return false;
+    }
+
+    // Check IORING_OP_WRITE is supported
+    if !probe.is_supported(opcode::Write::CODE) {
+        info!("{} IORING_OP_WRITE operation not supported", error_msg);
+        return false;
+    }
+
+    true
+}