From 868d1f690238a564ae0f692d55311da07a12a4cc Mon Sep 17 00:00:00 2001
From: Steven Dake <sdake@lambdal.com>
Date: Mon, 8 Aug 2022 05:23:04 +0000
Subject: [PATCH] pci: vfio: Preserve prefetchable BAR flag

If the BAR for the VFIO device is marked as prefetchable on the
underlying device ensure that the BAR exposed through PciConfiguration
is also marked as prefetchable.

Fixes problem where NVIDIA devices are not usable with PCI VFIO
passthrough. See related NVIDIA kernel driver bug:
 https://github.com/NVIDIA/open-gpu-kernel-modules/issues/344.

Fixes: #4451

Signed-off-by: Steven Dake <sdake@lambdal.com>
Signed-off-by: Rob Bradford <robert.bradford@intel.com>
---
 pci/src/configuration.rs |  6 ++++++
 pci/src/vfio.rs          | 18 +++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/pci/src/configuration.rs b/pci/src/configuration.rs
index d98834ef4..f2454a5ae 100644
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -999,6 +999,12 @@ impl PciBarConfiguration {
         self
     }
 
+    #[must_use]
+    pub fn set_prefetchable(mut self, prefetchable: PciBarPrefetchable) -> Self {
+        self.prefetchable = prefetchable;
+        self
+    }
+
     pub fn idx(&self) -> usize {
         self.idx
     }
diff --git a/pci/src/vfio.rs b/pci/src/vfio.rs
index 26e72f7db..d073d81bd 100644
--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -5,8 +5,9 @@
 
 use crate::{
     msi_num_enabled_vectors, BarReprogrammingParams, MsiCap, MsiConfig, MsixCap, MsixConfig,
-    PciBarConfiguration, PciBarRegionType, PciBdf, PciCapabilityId, PciClassCode, PciConfiguration,
-    PciDevice, PciDeviceError, PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
+    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciBdf, PciCapabilityId,
+    PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass,
+    MSIX_TABLE_ENTRY_SIZE,
 };
 use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
@@ -415,6 +416,7 @@ impl VfioCommon {
         while bar_id < VFIO_PCI_CONFIG_REGION_INDEX {
             let mut region_size: u64 = 0;
             let mut region_type = PciBarRegionType::Memory32BitRegion;
+            let mut prefetchable = PciBarPrefetchable::NotPrefetchable;
             let mut flags: u32 = 0;
 
             let mut restored_bar_addr = None;
@@ -467,6 +469,13 @@ impl VfioCommon {
                     false
                 };
 
+                if matches!(
+                    flags & PCI_CONFIG_BAR_PREFETCHABLE,
+                    PCI_CONFIG_BAR_PREFETCHABLE
+                ) {
+                    prefetchable = PciBarPrefetchable::Prefetchable
+                };
+
                 // To get size write all 1s
                 self.vfio_wrapper
                     .write_config_dword(bar_offset, 0xffff_ffff);
@@ -566,7 +575,8 @@ impl VfioCommon {
                 .set_index(bar_id as usize)
                 .set_address(bar_addr.raw_value())
                 .set_size(region_size)
-                .set_region_type(region_type);
+                .set_region_type(region_type)
+                .set_prefetchable(prefetchable);
 
             if bar_id == VFIO_PCI_ROM_REGION_INDEX {
                 self.configuration
@@ -1505,6 +1515,8 @@ const PCI_CONFIG_CAPABILITY_OFFSET: u32 = 0x34;
 const PCI_CONFIG_IO_BAR: u32 = 0x1;
 // 64-bit memory bar flag.
 const PCI_CONFIG_MEMORY_BAR_64BIT: u32 = 0x4;
+// Prefetchable BAR bit
+const PCI_CONFIG_BAR_PREFETCHABLE: u32 = 0x8;
 // PCI config register size (4 bytes).
 const PCI_CONFIG_REGISTER_SIZE: usize = 4;
 // Number of BARs for a PCI device