From ec0b5567c89139a2068884fcbada5431894cb274 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Tue, 20 Aug 2019 14:12:00 -0700 Subject: [PATCH] vmm: Share the guest memory instead of cloning it The VMM guest memory was cloned (copied) everywhere the code needed to have ownership of it. In order to clean the code, and in anticipation for future support of modifying this guest memory instance at runtime, it is important that every part of the code share the same instance. Because VirtioDevice implementations need to have access to it from different threads, that's why Arc must be used in this case. Signed-off-by: Sebastien Boeuf --- vfio/src/vfio_device.rs | 8 +++-- vm-virtio/src/block.rs | 4 +-- vm-virtio/src/console.rs | 4 +-- vm-virtio/src/device.rs | 2 +- vm-virtio/src/fs.rs | 4 +-- vm-virtio/src/net.rs | 4 +-- vm-virtio/src/pmem.rs | 4 +-- vm-virtio/src/rng.rs | 4 +-- vm-virtio/src/transport/pci_common_config.rs | 2 +- vm-virtio/src/transport/pci_device.rs | 4 +-- vmm/src/vm.rs | 32 +++++++++++--------- 11 files changed, 40 insertions(+), 32 deletions(-) diff --git a/vfio/src/vfio_device.rs b/vfio/src/vfio_device.rs index 2bab89e8b..2e5e1a37b 100644 --- a/vfio/src/vfio_device.rs +++ b/vfio/src/vfio_device.rs @@ -513,14 +513,18 @@ pub struct VfioDevice { group: VfioGroup, regions: Vec, irqs: HashMap, - mem: GuestMemoryMmap, + mem: Arc, } impl VfioDevice { /// Create a new vfio device, then guest read/write on this device could be /// transfered into kernel vfio. /// sysfspath specify the vfio device path in sys file system. - pub fn new(sysfspath: &Path, device_fd: Arc, mem: GuestMemoryMmap) -> Result { + pub fn new( + sysfspath: &Path, + device_fd: Arc, + mem: Arc, + ) -> Result { let uuid_path: PathBuf = [sysfspath, Path::new("iommu_group")].iter().collect(); let group_path = uuid_path.read_link().map_err(|_| VfioError::InvalidPath)?; let group_osstr = group_path.file_name().ok_or(VfioError::InvalidPath)?; diff --git a/vm-virtio/src/block.rs b/vm-virtio/src/block.rs index c1901fa1a..649f5d25a 100755 --- a/vm-virtio/src/block.rs +++ b/vm-virtio/src/block.rs @@ -321,7 +321,7 @@ impl Request { struct BlockEpollHandler { queues: Vec, - mem: GuestMemoryMmap, + mem: Arc, disk_image: T, disk_nsectors: u64, interrupt_cb: Arc, @@ -610,7 +610,7 @@ impl VirtioDevice for Block { fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, queues: Vec, mut queue_evts: Vec, diff --git a/vm-virtio/src/console.rs b/vm-virtio/src/console.rs index abd6601b3..38cefcaa4 100755 --- a/vm-virtio/src/console.rs +++ b/vm-virtio/src/console.rs @@ -54,7 +54,7 @@ unsafe impl ByteValued for VirtioConsoleConfig {} struct ConsoleEpollHandler { queues: Vec, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, in_buffer: Arc>>, out: Box, @@ -432,7 +432,7 @@ impl VirtioDevice for Console { fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, queues: Vec, mut queue_evts: Vec, diff --git a/vm-virtio/src/device.rs b/vm-virtio/src/device.rs index 63041b6b4..f93b9b168 100644 --- a/vm-virtio/src/device.rs +++ b/vm-virtio/src/device.rs @@ -67,7 +67,7 @@ pub trait VirtioDevice: Send { /// Activates this device for real usage. fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_evt: Arc, queues: Vec, queue_evts: Vec, diff --git a/vm-virtio/src/fs.rs b/vm-virtio/src/fs.rs index 0e7a894ff..9fcef9878 100644 --- a/vm-virtio/src/fs.rs +++ b/vm-virtio/src/fs.rs @@ -368,7 +368,7 @@ impl Fs { fn setup_vu( &mut self, - mem: &GuestMemoryMmap, + mem: &Arc, queues: Vec, queue_evts: Vec, ) -> Result> { @@ -527,7 +527,7 @@ impl VirtioDevice for Fs { fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, queues: Vec, queue_evts: Vec, diff --git a/vm-virtio/src/net.rs b/vm-virtio/src/net.rs index d90edd61b..b8ab8170e 100644 --- a/vm-virtio/src/net.rs +++ b/vm-virtio/src/net.rs @@ -115,7 +115,7 @@ fn vnet_hdr_len() -> usize { } struct NetEpollHandler { - mem: GuestMemoryMmap, + mem: Arc, tap: Tap, rx: RxVirtio, tx: TxVirtio, @@ -572,7 +572,7 @@ impl VirtioDevice for Net { fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, mut queues: Vec, mut queue_evts: Vec, diff --git a/vm-virtio/src/pmem.rs b/vm-virtio/src/pmem.rs index ada32c8b2..8e5a1079d 100644 --- a/vm-virtio/src/pmem.rs +++ b/vm-virtio/src/pmem.rs @@ -154,7 +154,7 @@ impl Request { struct PmemEpollHandler { queue: Queue, - mem: GuestMemoryMmap, + mem: Arc, disk: File, interrupt_cb: Arc, queue_evt: EventFd, @@ -382,7 +382,7 @@ impl VirtioDevice for Pmem { fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, mut queues: Vec, mut queue_evts: Vec, diff --git a/vm-virtio/src/rng.rs b/vm-virtio/src/rng.rs index ebd4ee0ea..da3095bb5 100755 --- a/vm-virtio/src/rng.rs +++ b/vm-virtio/src/rng.rs @@ -32,7 +32,7 @@ const KILL_EVENT: DeviceEventT = 1; struct RngEpollHandler { queues: Vec, - mem: GuestMemoryMmap, + mem: Arc, random_file: File, interrupt_cb: Arc, queue_evt: EventFd, @@ -237,7 +237,7 @@ impl VirtioDevice for Rng { fn activate( &mut self, - mem: GuestMemoryMmap, + mem: Arc, interrupt_cb: Arc, queues: Vec, mut queue_evts: Vec, diff --git a/vm-virtio/src/transport/pci_common_config.rs b/vm-virtio/src/transport/pci_common_config.rs index a3446f265..ad2e80480 100644 --- a/vm-virtio/src/transport/pci_common_config.rs +++ b/vm-virtio/src/transport/pci_common_config.rs @@ -271,7 +271,7 @@ mod tests { } fn activate( &mut self, - _mem: GuestMemoryMmap, + _mem: Arc, _interrupt_evt: Arc, _queues: Vec, _queue_evts: Vec, diff --git a/vm-virtio/src/transport/pci_device.rs b/vm-virtio/src/transport/pci_device.rs index 8332818f8..5f5fc0e3e 100755 --- a/vm-virtio/src/transport/pci_device.rs +++ b/vm-virtio/src/transport/pci_device.rs @@ -235,7 +235,7 @@ pub struct VirtioPciDevice { queue_evts: Vec, // Guest memory - memory: Option, + memory: Option>, // Setting PCI BAR settings_bar: u8, @@ -244,7 +244,7 @@ pub struct VirtioPciDevice { impl VirtioPciDevice { /// Constructs a new PCI transport for the given virtio device. pub fn new( - memory: GuestMemoryMmap, + memory: Arc, device: Box, msix_num: u16, ) -> Result { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index cc6c6d189..c1f79380e 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -518,7 +518,7 @@ impl Vcpu { } struct VmInfo<'a> { - memory: GuestMemoryMmap, + memory: &'a Arc, vm_fd: &'a Arc, vm_cfg: &'a VmConfig<'a>, } @@ -697,7 +697,7 @@ impl DeviceManager { .map_err(DeviceManagerError::CreateVirtioConsole)?; DeviceManager::add_virtio_pci_device( Box::new(virtio_console_device), - vm_info.memory.clone(), + vm_info.memory, allocator, vm_info.vm_fd, &mut pci, @@ -812,7 +812,7 @@ impl DeviceManager { DeviceManager::add_virtio_pci_device( block, - vm_info.memory.clone(), + vm_info.memory, allocator, vm_info.vm_fd, pci, @@ -849,7 +849,7 @@ impl DeviceManager { DeviceManager::add_virtio_pci_device( Box::new(virtio_net_device), - vm_info.memory.clone(), + vm_info.memory, allocator, vm_info.vm_fd, pci, @@ -876,7 +876,7 @@ impl DeviceManager { DeviceManager::add_virtio_pci_device( Box::new(virtio_rng_device), - vm_info.memory.clone(), + vm_info.memory, allocator, vm_info.vm_fd, pci, @@ -965,7 +965,7 @@ impl DeviceManager { DeviceManager::add_virtio_pci_device( Box::new(virtio_fs_device), - vm_info.memory.clone(), + vm_info.memory, allocator, vm_info.vm_fd, pci, @@ -1046,7 +1046,7 @@ impl DeviceManager { DeviceManager::add_virtio_pci_device( Box::new(virtio_pmem_device), - vm_info.memory.clone(), + vm_info.memory, allocator, vm_info.vm_fd, pci, @@ -1113,7 +1113,7 @@ impl DeviceManager { fn add_virtio_pci_device( virtio_device: Box, - memory: GuestMemoryMmap, + memory: &Arc, allocator: &mut SystemAllocator, vm_fd: &Arc, pci: &mut PciConfigIo, @@ -1126,7 +1126,7 @@ impl DeviceManager { 0 }; - let mut virtio_pci_device = VirtioPciDevice::new(memory, virtio_device, msix_num) + let mut virtio_pci_device = VirtioPciDevice::new(memory.clone(), virtio_device, msix_num) .map_err(DeviceManagerError::VirtioDevice)?; let bars = virtio_pci_device @@ -1317,7 +1317,7 @@ impl AsRawFd for EpollContext { pub struct Vm<'a> { fd: Arc, kernel: File, - memory: GuestMemoryMmap, + memory: Arc, vcpus: Vec>, devices: DeviceManager, cpuid: CpuId, @@ -1485,8 +1485,12 @@ impl<'a> Vm<'a> { .ok_or(Error::MemoryRangeAllocation)?; } + // Convert the guest memory into an Arc. The point being able to use it + // anywhere in the code, no matter which thread might use it. + let guest_memory = Arc::new(guest_memory); + let vm_info = VmInfo { - memory: guest_memory.clone(), + memory: &guest_memory, vm_fd: &fd, vm_cfg: &config, }; @@ -1533,7 +1537,7 @@ impl<'a> Vm<'a> { let cmdline_cstring = CString::new(self.config.cmdline.args.clone()).map_err(|_| Error::CmdLine)?; let entry_addr = match linux_loader::loader::Elf::load( - &self.memory, + self.memory.as_ref(), None, &mut self.kernel, Some(arch::HIMEM_START), @@ -1541,7 +1545,7 @@ impl<'a> Vm<'a> { Ok(entry_addr) => entry_addr, Err(linux_loader::loader::Error::InvalidElfMagicNumber) => { linux_loader::loader::BzImage::load( - &self.memory, + self.memory.as_ref(), None, &mut self.kernel, Some(arch::HIMEM_START), @@ -1552,7 +1556,7 @@ impl<'a> Vm<'a> { }; linux_loader::loader::load_cmdline( - &self.memory, + self.memory.as_ref(), self.config.cmdline.offset, &cmdline_cstring, )