From f77d2c2d1631ea360a5437744abae90fc9fc567f Mon Sep 17 00:00:00 2001
From: Sebastien Boeuf <sebastien.boeuf@intel.com>
Date: Thu, 9 Jan 2020 16:09:19 +0100
Subject: [PATCH] pci: Add some KVM and interrupt utilities to the crate

In order to anticipate the need for both msi.rs and msix.rs to rely on
some KVM utils and InterruptRoute structure to handle the update of the
KVM GSI routes, this commit adds these utilities directly to the pci
crate. So far, these were exclusively used by the vfio crate, which is
why there were located there.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
---
 Cargo.lock     |  2 ++
 pci/Cargo.toml |  2 ++
 pci/src/lib.rs | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 59b228f39..a940ee42d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -473,6 +473,8 @@ version = "0.1.0"
 dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "devices 0.1.0",
+ "kvm-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "kvm-ioctls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "vm-allocator 0.1.0",
diff --git a/pci/Cargo.toml b/pci/Cargo.toml
index 968248b56..63cfad165 100644
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -8,6 +8,8 @@ edition = "2018"
 vm-allocator = { path = "../vm-allocator" }
 byteorder = "1.3.2"
 devices = { path = "../devices" }
+kvm-bindings = "0.2.0"
+kvm-ioctls = "0.4.0"
 libc = "0.2.60"
 log = "0.4.8"
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
diff --git a/pci/src/lib.rs b/pci/src/lib.rs
index c8b261d3e..dabd2280d 100644
--- a/pci/src/lib.rs
+++ b/pci/src/lib.rs
@@ -27,6 +27,14 @@ pub use self::device::{
 };
 pub use self::msi::MsiCap;
 pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_TABLE_ENTRY_SIZE};
+use kvm_bindings::{kvm_irq_routing, kvm_irq_routing_entry};
+use kvm_ioctls::*;
+use std::collections::HashMap;
+use std::io;
+use std::mem::size_of;
+use std::sync::{Arc, Mutex};
+use vm_allocator::SystemAllocator;
+use vmm_sys_util::eventfd::EventFd;
 
 /// PCI has four interrupt pins A->D.
 #[derive(Copy, Clone)]
@@ -42,3 +50,89 @@ impl PciInterruptPin {
         self as u32
     }
 }
+
+#[derive(Debug)]
+pub enum Error {
+    AllocateGsi,
+    EventFd(io::Error),
+    IrqFd(kvm_ioctls::Error),
+    SetGsiRouting(kvm_ioctls::Error),
+}
+
+// Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.
+fn vec_with_size_in_bytes<T: Default>(size_in_bytes: usize) -> Vec<T> {
+    let rounded_size = (size_in_bytes + size_of::<T>() - 1) / size_of::<T>();
+    let mut v = Vec::with_capacity(rounded_size);
+    v.resize_with(rounded_size, T::default);
+    v
+}
+
+// The kvm API has many structs that resemble the following `Foo` structure:
+//
+// ```
+// #[repr(C)]
+// struct Foo {
+//    some_data: u32
+//    entries: __IncompleteArrayField<__u32>,
+// }
+// ```
+//
+// In order to allocate such a structure, `size_of::<Foo>()` would be too small because it would not
+// include any space for `entries`. To make the allocation large enough while still being aligned
+// for `Foo`, a `Vec<Foo>` is created. Only the first element of `Vec<Foo>` would actually be used
+// as a `Foo`. The remaining memory in the `Vec<Foo>` is for `entries`, which must be contiguous
+// with `Foo`. This function is used to make the `Vec<Foo>` with enough space for `count` entries.
+pub fn vec_with_array_field<T: Default, F>(count: usize) -> Vec<T> {
+    let element_space = count * size_of::<F>();
+    let vec_size_bytes = size_of::<T>() + element_space;
+    vec_with_size_in_bytes(vec_size_bytes)
+}
+
+pub fn set_kvm_routes<S: ::std::hash::BuildHasher>(
+    vm_fd: VmFd,
+    gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry, S>>>,
+) -> Result<(), Error> {
+    let mut entry_vec: Vec<kvm_irq_routing_entry> = Vec::new();
+    for (_, entry) in gsi_msi_routes.lock().unwrap().iter() {
+        entry_vec.push(*entry);
+    }
+
+    let mut irq_routing =
+        vec_with_array_field::<kvm_irq_routing, kvm_irq_routing_entry>(entry_vec.len());
+    irq_routing[0].nr = entry_vec.len() as u32;
+    irq_routing[0].flags = 0;
+
+    unsafe {
+        let entries: &mut [kvm_irq_routing_entry] =
+            irq_routing[0].entries.as_mut_slice(entry_vec.len());
+        entries.copy_from_slice(&entry_vec);
+    }
+
+    vm_fd
+        .set_gsi_routing(&irq_routing[0])
+        .map_err(Error::SetGsiRouting)
+}
+
+pub struct InterruptRoute {
+    gsi: u32,
+    irq_fd: EventFd,
+}
+
+impl InterruptRoute {
+    pub fn new(allocator: &mut SystemAllocator) -> Result<Self, Error> {
+        let irq_fd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?;
+        let gsi = allocator.allocate_gsi().ok_or(Error::AllocateGsi)?;
+
+        Ok(InterruptRoute { gsi, irq_fd })
+    }
+
+    pub fn enable(&self, vm: &Arc<VmFd>) -> Result<(), Error> {
+        vm.register_irqfd(&self.irq_fd, self.gsi)
+            .map_err(Error::IrqFd)
+    }
+
+    pub fn disable(&self, vm: &Arc<VmFd>) -> Result<(), Error> {
+        vm.unregister_irqfd(&self.irq_fd, self.gsi)
+            .map_err(Error::IrqFd)
+    }
+}