From b67e0b3dad39acd518337afb449af60f94383e92 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Mon, 6 May 2019 10:27:40 -0700 Subject: [PATCH] vmm: Use virtio-blk to support booting from disk image After the virtio-blk device support has been introduced in the previous commit, the vmm need to rely on this new device to boot from disk images instead of initrd built into the kernel. In order to achieve the proper support of virtio-blk, this commit had to handle a few things: - Register an ioevent fd for each virtqueue. This important to be notified from the virtio driver that something has been written on the queue. - Fix the retrieval of 64bits BAR address. This is needed to provide the right address which need to be registered as the notification address from the virtio driver. - Fix the write_bar and read_bar functions. They were both assuming to be provided with an address, from which they were trying to find the associated offset. But the reality is that the offset is directly provided by the Bus layer. - Register a new virtio-blk device as a virtio-pci device from the vm.rs code. When the VM is started, it expects a block device to be created, using this block device as the VM rootfs. Signed-off-by: Sebastien Boeuf --- Cargo.lock | 182 ++++++++++++++++++++++++++ pci/src/configuration.rs | 11 +- pci/src/device.rs | 3 +- vm-virtio/src/lib.rs | 5 + vm-virtio/src/transport/pci_device.rs | 44 +++++-- vmm/Cargo.toml | 2 + vmm/src/vm.rs | 102 +++++++++++++-- 7 files changed, 327 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67ff80520..c3860bfe1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,11 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "autocfg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bitflags" version = "0.9.1" @@ -71,6 +76,14 @@ dependencies = [ "vmm 0.1.0", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "devices" version = "0.1.0" @@ -91,6 +104,11 @@ dependencies = [ "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "kvm-bindings" version = "0.1.1" @@ -168,6 +186,110 @@ dependencies = [ "proc-macro2 0.4.29 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "redox_syscall" version = "0.1.51" @@ -191,6 +313,14 @@ dependencies = [ "syn 0.15.33 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.6.0" @@ -206,6 +336,19 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempfile" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "termion" version = "1.5.1" @@ -239,6 +382,10 @@ name = "vec_map" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "virtio-bindings" +version = "0.1.0" + [[package]] name = "vm-allocator" version = "0.1.0" @@ -255,6 +402,23 @@ dependencies = [ "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "vm-virtio" +version = "0.1.0" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "devices 0.1.0", + "epoll 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pci 0.1.0", + "tempfile 3.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-bindings 0.1.0", + "vm-allocator 0.1.0", + "vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)", + "vmm-sys-util 0.1.0 (git+https://github.com/sameo/vmm-sys-util)", +] + [[package]] name = "vmm" version = "0.1.0" @@ -268,7 +432,9 @@ dependencies = [ "linux-loader 0.1.0 (git+https://github.com/sameo/linux-loader)", "pci 0.1.0", "qcow 0.1.0", + "vm-allocator 0.1.0", "vm-memory 0.1.0 (git+https://github.com/rust-vmm/vm-memory)", + "vm-virtio 0.1.0", "vmm-sys-util 0.1.0 (git+https://github.com/sameo/vmm-sys-util)", ] @@ -302,12 +468,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +"checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" "checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" "checksum clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b8c532887f1a292d17de05ae858a8fe50a301e196f9ef0ddb7ccd0d1d00f180" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum epoll 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3f0680f2a6f2a17fa7a8668a27c54e45e1ad1cf8a632f56a7c19b9e4e3bbe8a" +"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum kvm-bindings 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c223e8703d2eb76d990c5f58e29c85b0f6f50e24b823babde927948e7c71fc03" "checksum kvm-ioctls 0.1.0 (git+https://github.com/rust-vmm/kvm-ioctls)" = "" "checksum libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)" = "e962c7641008ac010fa60a7dfdc1712449f29c44ef2d4702394aea943ee75047" @@ -315,11 +484,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" "checksum proc-macro2 0.4.29 (registry+https://github.com/rust-lang/crates.io-index)" = "64c827cea7a7ab30ce4593e5e04d7a11617ad6ece2fa230605a78b00ff965316" "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" +"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +"checksum rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum remain 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3bec2543b50be4539fdc27fde082e218cf4c3895358ca77f5c52fe930589e209" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum syn 0.15.33 (registry+https://github.com/rust-lang/crates.io-index)" = "ec52cd796e5f01d0067225a5392e70084acc4c0013fa71d55166d38a8b307836" +"checksum tempfile 3.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "b86c784c88d98c801132806dadd3819ed29d8600836c4088e855cdf3e178ed8a" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/pci/src/configuration.rs b/pci/src/configuration.rs index c54869175..17b3519df 100755 --- a/pci/src/configuration.rs +++ b/pci/src/configuration.rs @@ -402,13 +402,20 @@ impl PciConfiguration { Ok(config.reg_idx) } - /// Returns the address of the given BAR region. - pub fn get_bar_addr(&self, bar_num: usize) -> u32 { + /// Returns the address of the given 32 bits BAR region. + pub fn get_bar32_addr(&self, bar_num: usize) -> u32 { let bar_idx = BAR0_REG + bar_num; self.registers[bar_idx] & BAR_MEM_ADDR_MASK } + /// Returns the address of the given 64 bits BAR region. + pub fn get_bar64_addr(&self, bar_num: usize) -> u64 { + let bar_idx = BAR0_REG + bar_num; + + u64::from(self.registers[bar_idx] & BAR_MEM_ADDR_MASK) | (u64::from(self.registers[bar_idx+1]) << 32) + } + /// Configures the IRQ line and pin used by this device. pub fn set_irq(&mut self, line: u8, pin: PciInterruptPin) { // `pin` is 1-based in the pci config space. diff --git a/pci/src/device.rs b/pci/src/device.rs index 94415f027..53868ad3d 100755 --- a/pci/src/device.rs +++ b/pci/src/device.rs @@ -5,7 +5,6 @@ use crate::configuration::{self, PciConfiguration}; use crate::PciInterruptPin; use devices::BusDevice; -use kvm_ioctls::*; use std; use std::fmt::{self, Display}; use vm_allocator::SystemAllocator; @@ -60,7 +59,7 @@ pub trait PciDevice: BusDevice { /// Gets a list of ioeventfds that should be registered with the running VM. The list is /// returned as a Vec of (eventfd, addr, datamatch) tuples. - fn ioeventfds(&self) -> Vec<(&EventFd, u64, NoDatamatch)> { + fn ioeventfds(&self) -> Vec<(&EventFd, u64, u64)> { Vec::new() } /// Gets the configuration registers of the Pci Device. diff --git a/vm-virtio/src/lib.rs b/vm-virtio/src/lib.rs index b8b1f4ee2..201acbb89 100644 --- a/vm-virtio/src/lib.rs +++ b/vm-virtio/src/lib.rs @@ -17,11 +17,13 @@ use std::fmt; use std::fs::File; use std::io; +mod block; mod device; mod queue; pub mod transport; +pub use self::block::*; pub use self::device::*; pub use self::queue::*; @@ -123,4 +125,7 @@ pub enum Error { event: DeviceEventT, }, IoError(io::Error), + EpollCreateFd(io::Error), + EpollCtl(io::Error), + EpollWait(io::Error), } diff --git a/vm-virtio/src/transport/pci_device.rs b/vm-virtio/src/transport/pci_device.rs index 4f67a0cc4..64945b140 100755 --- a/vm-virtio/src/transport/pci_device.rs +++ b/vm-virtio/src/transport/pci_device.rs @@ -218,6 +218,18 @@ impl VirtioPciDevice { }) } + /// Gets the list of queue events that must be triggered whenever the VM writes to + /// `virtio::NOTIFY_REG_OFFSET` past the MMIO base. Each event must be triggered when the + /// value being written equals the index of the event in this list. + pub fn queue_evts(&self) -> &[EventFd] { + self.queue_evts.as_slice() + } + + /// Gets the event this device uses to interrupt the VM when the used queue is changed. + pub fn interrupt_evt(&self) -> Option<&EventFd> { + self.interrupt_evt.as_ref() + } + fn is_driver_ready(&self) -> bool { let ready_bits = (DEVICE_ACKNOWLEDGE | DEVICE_DRIVER | DEVICE_DRIVER_OK | DEVICE_FEATURES_OK) as u8; @@ -305,6 +317,24 @@ impl PciDevice for VirtioPciDevice { &mut self.configuration } + fn ioeventfds(&self) -> Vec<(&EventFd, u64, u64)> { + let bar0 = self + .configuration + .get_bar64_addr(self.settings_bar as usize); + let notify_base = bar0 + NOTIFICATION_BAR_OFFSET; + self.queue_evts() + .iter() + .enumerate() + .map(|(i, event)| { + ( + event, + notify_base + i as u64 * u64::from(NOTIFY_OFF_MULTIPLIER), + i as u64, + ) + }) + .collect() + } + fn allocate_bars( &mut self, allocator: &mut SystemAllocator, @@ -326,8 +356,9 @@ impl PciDevice for VirtioPciDevice { })? as u8; println!( - "VIRTIO PCI BAR starts at 0x{:x}", - virtio_pci_bar_addr.raw_value() + "VIRTIO PCI BAR starts at 0x{:x}, size 0x{:x}", + virtio_pci_bar_addr.raw_value(), + CAPABILITY_BAR_SIZE ); ranges.push((virtio_pci_bar_addr, CAPABILITY_BAR_SIZE)); @@ -349,10 +380,7 @@ impl PciDevice for VirtioPciDevice { Ok(ranges) } - fn read_bar(&mut self, addr: u64, data: &mut [u8]) { - // The driver is only allowed to do aligned, properly sized access. - let bar0 = u64::from(self.configuration.get_bar_addr(self.settings_bar as usize)); - let offset = addr - bar0; + fn read_bar(&mut self, offset: u64, data: &mut [u8]) { match offset { o if o < COMMON_CONFIG_BAR_OFFSET + COMMON_CONFIG_SIZE => self.common_config.read( o - COMMON_CONFIG_BAR_OFFSET, @@ -380,9 +408,7 @@ impl PciDevice for VirtioPciDevice { } } - fn write_bar(&mut self, addr: u64, data: &[u8]) { - let bar0 = u64::from(self.configuration.get_bar_addr(self.settings_bar as usize)); - let offset = addr - bar0; + fn write_bar(&mut self, offset: u64, data: &[u8]) { match offset { o if o < COMMON_CONFIG_BAR_OFFSET + COMMON_CONFIG_SIZE => self.common_config.write( o - COMMON_CONFIG_BAR_OFFSET, diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index 3782dfd02..c8009404f 100755 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -14,6 +14,8 @@ libc = ">=0.2.39" pci = {path = "../pci"} qcow = { path = "../qcow" } linux-loader = { git = "https://github.com/sameo/linux-loader" } +vm-virtio = { path = "../vm-virtio" } +vm-allocator = { path = "../vm-allocator" } vmm-sys-util = { git = "https://github.com/sameo/vmm-sys-util" } [dependencies.vm-memory] diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 16fd8bd66..a3e3cc5c9 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -9,7 +9,9 @@ extern crate epoll; extern crate kvm_ioctls; extern crate libc; extern crate linux_loader; +extern crate vm_allocator; extern crate vm_memory; +extern crate vm_virtio; extern crate vmm_sys_util; use kvm_bindings::{kvm_pit_config, kvm_userspace_memory_region, KVM_PIT_SPEAKER_DUMMY}; @@ -17,18 +19,20 @@ use kvm_ioctls::*; use libc::{c_void, siginfo_t, EFD_NONBLOCK}; use linux_loader::cmdline; use linux_loader::loader::KernelLoader; -use pci::{PciConfigIo, PciRoot}; +use pci::{PciConfigIo, PciDevice, PciInterruptPin, PciRoot}; use std::ffi::CString; -use std::fs::File; +use std::fs::{File, OpenOptions}; use std::io::{self, stdout}; use std::os::unix::io::{AsRawFd, RawFd}; use std::path::Path; use std::sync::{Arc, Barrier, Mutex}; use std::{result, str, thread}; +use vm_allocator::SystemAllocator; use vm_memory::{ Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestUsize, MmapError, }; +use vm_virtio::transport::VirtioPciDevice; use vmm_sys_util::signal::register_signal_handler; use vmm_sys_util::terminal::Terminal; use vmm_sys_util::EventFd; @@ -39,6 +43,7 @@ pub const DEFAULT_MEMORY: GuestUsize = 512; const DEFAULT_CMDLINE: &str = "console=ttyS0 reboot=k panic=1 nomodules \ i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd"; const CMDLINE_OFFSET: GuestAddress = GuestAddress(0x20000); +const X86_64_IRQ_BASE: u32 = 5; // CPUID feature bits const ECX_HYPERVISOR_SHIFT: u32 = 31; // Hypervisor bit. @@ -114,8 +119,32 @@ pub enum Error { /// Write to the serial console failed. Serial(vmm_sys_util::Error), + /// Cannot allocate IRQ. + AllocateIrq, + + /// Cannot allocate PCI BARs + AllocateBars(pci::PciDeviceError), + + /// Cannot register ioevent. + RegisterIoevent(io::Error), + /// Cannot configure the IRQ. Irq(io::Error), + + /// Cannot create virtio device + VirtioDevice, + + /// Cannot add PCI device + AddPciDevice(pci::PciRootError), + + /// Cannot open disk path + Disk(io::Error), + + /// Cannot create virtio-blk device + CreateVirtioBlock(io::Error), + + /// Cannot create the system allocator + CreateSystemAllocator, } pub type Result = result::Result; @@ -213,6 +242,7 @@ impl<'a> Default for VmConfig<'a> { struct DeviceManager { io_bus: devices::Bus, + mmio_bus: devices::Bus, // Serial port on 0x3f8 serial: Arc>, @@ -227,8 +257,9 @@ struct DeviceManager { } impl DeviceManager { - fn new() -> Result { + fn new(memory: GuestMemoryMmap, allocator: &mut SystemAllocator, vm_fd: &VmFd) -> Result { let io_bus = devices::Bus::new(); + let mut mmio_bus = devices::Bus::new(); let serial_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFd)?; let serial = Arc::new(Mutex::new(devices::legacy::Serial::new_out( serial_evt.try_clone().map_err(Error::EventFd)?, @@ -240,11 +271,52 @@ impl DeviceManager { exit_evt.try_clone().map_err(Error::EventFd)?, ))); - let pci_root = PciRoot::new(None); + let mut pci_root = PciRoot::new(None); + + // Open block device path + let raw_img: File = OpenOptions::new() + .read(true) + .write(true) + .open("/foo/bar/rootfs.img") + .map_err(Error::Disk)?; + + let virtio_block_device = + vm_virtio::Block::new(raw_img, false).map_err(Error::CreateVirtioBlock)?; + let virtio_block_device = Box::new(virtio_block_device); + let mut virtio_pci_device = + VirtioPciDevice::new(memory, virtio_block_device).map_err(|_| Error::VirtioDevice)?; + let bars = virtio_pci_device + .allocate_bars(allocator) + .map_err(Error::AllocateBars)?; + + for (event, addr, _) in virtio_pci_device.ioeventfds() { + let io_addr = IoEventAddress::Mmio(addr); + println!("Register ioevent at 0x{:x}", addr); + vm_fd + .register_ioevent(event.as_raw_fd(), &io_addr, NoDatamatch) + .map_err(Error::RegisterIoevent)?; + } + + // Assign IRQ to the virtio-blk device + let irqfd = EventFd::new(EFD_NONBLOCK).map_err(Error::EventFd)?; + let irq_num = allocator.allocate_irq().ok_or(Error::AllocateIrq)?; + vm_fd + .register_irqfd(irqfd.as_raw_fd(), irq_num) + .map_err(Error::Irq)?; + // Let's use irq line INTA for now. + virtio_pci_device.assign_irq(irqfd, irq_num as u32, PciInterruptPin::IntA); + + let virtio_pci_device = Arc::new(Mutex::new(virtio_pci_device)); + + pci_root + .add_device(virtio_pci_device.clone(), &mut mmio_bus, bars) + .map_err(Error::AddPciDevice)?; + let pci = Arc::new(Mutex::new(PciConfigIo::new(pci_root))); Ok(DeviceManager { io_bus, + mmio_bus, serial, serial_evt, i8042, @@ -396,7 +468,18 @@ impl<'a> Vm<'a> { .map_err(Error::VmSetup)?; Vm::patch_cpuid(&mut cpuid); - let device_manager = DeviceManager::new().map_err(|_| Error::DeviceManager)?; + // Let's allocate 64 GiB of addressable MMIO space, starting at 0. + let mut allocator = SystemAllocator::new( + None, + None, + GuestAddress(0), + 1 << 36 as GuestUsize, + X86_64_IRQ_BASE, + ) + .ok_or(Error::CreateSystemAllocator)?; + + let device_manager = DeviceManager::new(guest_memory.clone(), &mut allocator, &fd) + .map_err(|_| Error::DeviceManager)?; fd.register_irqfd(device_manager.serial_evt.as_raw_fd(), 4) .map_err(Error::Irq)?; @@ -507,6 +590,7 @@ impl<'a> Vm<'a> { for cpu_id in 0..vcpu_count { println!("Starting VCPU {:?}", cpu_id); let io_bus = self.devices.io_bus.clone(); + let mmio_bus = self.devices.mmio_bus.clone(); let mut vcpu = Vcpu::new(cpu_id, &self)?; vcpu.configure(entry_addr, &self)?; @@ -541,11 +625,11 @@ impl<'a> Vm<'a> { VcpuExit::IoOut(addr, data) => { io_bus.write(u64::from(addr), data); } - VcpuExit::MmioRead(addr, _data) => { - println!("MMIO R -- addr: {:#x}", addr); + VcpuExit::MmioRead(addr, data) => { + mmio_bus.read(addr as u64, data); } - VcpuExit::MmioWrite(addr, _data) => { - println!("MMIO W -- addr: {:#x}", addr); + VcpuExit::MmioWrite(addr, data) => { + mmio_bus.write(addr as u64, data); } VcpuExit::Unknown => { println!("Unknown");