From b5eab43aa504a88ffc76498aa86b557451135fee Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 24 Jul 2019 21:15:46 -0700 Subject: [PATCH] vfio: Create a global KVM VFIO device for all VFIO devices KVM does not support multiple KVM VFIO devices to be created when trying to support multiple VFIO devices. This commit creates one global KVM VFIO device being shared with every VFIO device, which makes possible the support for passing several devices through the VM. Signed-off-by: Sebastien Boeuf --- vfio/src/vfio_device.rs | 30 +++++++----------------------- vmm/src/vm.rs | 23 +++++++++++++++++++++-- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/vfio/src/vfio_device.rs b/vfio/src/vfio_device.rs index 49d881195..e26ef3db5 100644 --- a/vfio/src/vfio_device.rs +++ b/vfio/src/vfio_device.rs @@ -36,7 +36,6 @@ pub enum VfioError { UnsetContainer, ContainerSetIOMMU, GroupGetDeviceFD, - CreateVfioKvmDevice(io::Error), KvmSetDeviceAttr(io::Error), VfioDeviceGetInfo, VfioDeviceGetRegionInfo, @@ -77,9 +76,6 @@ impl fmt::Display for VfioError { "failed to set container's IOMMU driver type as VfioType1V2" ), VfioError::GroupGetDeviceFD => write!(f, "failed to get vfio device fd"), - VfioError::CreateVfioKvmDevice(e) => { - write!(f, "failed to create KVM vfio device: {}", e) - } VfioError::KvmSetDeviceAttr(e) => { write!(f, "failed to set KVM vfio device's attribute: {}", e) } @@ -201,12 +197,12 @@ impl AsRawFd for VfioContainer { struct VfioGroup { group: File, - device: DeviceFd, + device: Arc, container: VfioContainer, } impl VfioGroup { - fn new(id: u32, vm: &Arc) -> Result { + fn new(id: u32, device: Arc) -> Result { let group_path = Path::new("/dev/vfio").join(id.to_string()); let group = OpenOptions::new() .read(true) @@ -246,7 +242,7 @@ impl VfioGroup { container.set_iommu(VFIO_TYPE1v2_IOMMU)?; - let device = Self::kvm_device_add_group(vm, &group)?; + Self::kvm_device_add_group(&device, &group)?; Ok(VfioGroup { group, @@ -255,17 +251,7 @@ impl VfioGroup { }) } - fn kvm_device_add_group(vm: &VmFd, group: &File) -> Result { - let mut vfio_dev = kvm_bindings::kvm_create_device { - type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_VFIO, - fd: 0, - flags: 0, - }; - - let device_fd = vm - .create_device(&mut vfio_dev) - .map_err(VfioError::CreateVfioKvmDevice)?; - + fn kvm_device_add_group(device_fd: &Arc, group: &File) -> Result<()> { let group_fd = group.as_raw_fd(); let group_fd_ptr = &group_fd as *const i32; let dev_attr = kvm_bindings::kvm_device_attr { @@ -277,9 +263,7 @@ impl VfioGroup { device_fd .set_device_attr(&dev_attr) - .map_err(VfioError::KvmSetDeviceAttr)?; - - Ok(device_fd) + .map_err(VfioError::KvmSetDeviceAttr) } fn kvm_device_del_group(&self) -> std::result::Result<(), io::Error> { @@ -536,7 +520,7 @@ impl VfioDevice { /// Create a new vfio device, then guest read/write on this device could be /// transfered into kernel vfio. /// sysfspath specify the vfio device path in sys file system. - pub fn new(sysfspath: &Path, vm: &Arc, mem: GuestMemoryMmap) -> Result { + pub fn new(sysfspath: &Path, device_fd: Arc, mem: GuestMemoryMmap) -> Result { let uuid_path: PathBuf = [sysfspath, Path::new("iommu_group")].iter().collect(); let group_path = uuid_path.read_link().map_err(|_| VfioError::InvalidPath)?; let group_osstr = group_path.file_name().ok_or(VfioError::InvalidPath)?; @@ -545,7 +529,7 @@ impl VfioDevice { .parse::() .map_err(|_| VfioError::InvalidPath)?; - let group = VfioGroup::new(group_id, vm)?; + let group = VfioGroup::new(group_id, device_fd)?; let device_info = group.get_device(sysfspath)?; let regions = device_info.get_regions()?; let irqs = device_info.get_irqs()?; diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 27b8d16b5..21a63efe9 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -256,6 +256,9 @@ pub enum DeviceManagerError { /// Failed to map VFIO MMIO region. VfioMapRegion(VfioPciError), + + /// Failed to create the KVM device. + CreateKvmDevice(io::Error), } pub type DeviceManagerResult = result::Result; @@ -946,6 +949,17 @@ impl DeviceManager { Ok(()) } + fn create_kvm_device(vm: &Arc) -> DeviceManagerResult { + let mut vfio_dev = kvm_bindings::kvm_create_device { + type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_VFIO, + fd: 0, + flags: 0, + }; + + vm.create_device(&mut vfio_dev) + .map_err(DeviceManagerError::CreateKvmDevice) + } + fn add_vfio_devices( memory: GuestMemoryMmap, allocator: &mut SystemAllocator, @@ -956,9 +970,14 @@ impl DeviceManager { mem_slots: u32, ) -> DeviceManagerResult<()> { if let Some(device_list_cfg) = &vm_cfg.devices { + // Create the KVM VFIO device + let device_fd = DeviceManager::create_kvm_device(vm_fd)?; + let device_fd = Arc::new(device_fd); + for device_cfg in device_list_cfg.iter() { - let vfio_device = VfioDevice::new(device_cfg.path, vm_fd, memory.clone()) - .map_err(DeviceManagerError::VfioCreate)?; + let vfio_device = + VfioDevice::new(device_cfg.path, device_fd.clone(), memory.clone()) + .map_err(DeviceManagerError::VfioCreate)?; let mut vfio_pci_device = VfioPciDevice::new(vm_fd, allocator, vfio_device) .map_err(DeviceManagerError::VfioPciCreate)?;