Merge 0461c3080e into 4fd5070f5d

For MSHV we always create frozen partition, so we
resume the VM during boot. Also during pause and resume
VM events we call hypervisor specific API.

Signed-off-by: Muminul Islam <muislam@microsoft.com>

.github/workflows/taplo.yaml

@@ -0,0 +1,21 @@
+name: Cargo.toml Formatting (taplo)
+on:
+  pull_request:
+    paths: 
+      - '**/Cargo.toml'
+
+jobs:
+  cargo_toml_format:
+    name: Cargo.toml Formatting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install build dependencies
+        run: sudo apt-get update && sudo apt-get -yqq install build-essential libssl-dev
+      - name: Install taplo
+        run: cargo install taplo-cli --locked
+      - name: Check formatting
+        run: taplo fmt --check

.taplo.toml

@@ -0,0 +1,5 @@
+include = ["**/Cargo.toml"]
+
+[formatting]
+reoder_arrays = true
+reorder_keys = true

Cargo.toml

@@ -1,13 +1,13 @@
 [package]
+authors = ["The Cloud Hypervisor Authors"]
+build = "build.rs"
+default-run = "cloud-hypervisor"
+description = "Open source Virtual Machine Monitor (VMM) that runs on top of KVM"
+edition = "2021"
+homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
+license = "LICENSE-APACHE & LICENSE-BSD-3-Clause"
 name = "cloud-hypervisor"
 version = "39.0.0"
-authors = ["The Cloud Hypervisor Authors"]
-edition = "2021"
-default-run = "cloud-hypervisor"
-build = "build.rs"
-license = "LICENSE-APACHE & LICENSE-BSD-3-Clause"
-description = "Open source Virtual Machine Monitor (VMM) that runs on top of KVM"
-homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 # Minimum buildable version:
 # Keep in sync with version in .github/workflows/build.yaml
 # Policy on MSRV (see #4318):
@@ -18,15 +18,15 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 rust-version = "1.77.0"
 
 [profile.release]
-lto = true
 codegen-units = 1
+lto = true
 opt-level = "s"
 strip = true
 
 [profile.profiling]
+debug = true
 inherits = "release"
 strip = false
-debug = true
 
 [dependencies]
 anyhow = "1.0.81"
@@ -43,11 +43,11 @@ seccompiler = "0.4.0"
 serde_json = "1.0.115"
 signal-hook = "0.3.17"
 thiserror = "1.0.60"
-tpm = { path = "tpm"}
+tpm = { path = "tpm" }
 tracer = { path = "tracer" }
+vm-memory = "0.14.1"
 vmm = { path = "vmm" }
 vmm-sys-util = "0.12.1"
-vm-memory = "0.14.1"
 zbus = { version = "3.15.2", optional = true }
 
 [dev-dependencies]
@@ -61,9 +61,9 @@ wait-timeout = "0.2.0"
 # Please adjust `vmm::feature_list()` accordingly when changing the
 # feature list below
 [features]
-default = ["kvm", "io_uring"]
 dbus_api = ["zbus", "vmm/dbus_api"]
-dhat-heap = ["dhat", "vmm/dhat-heap"] # For heap profiling
+default = ["kvm", "io_uring"]
+dhat-heap = ["dhat", "vmm/dhat-heap"]       # For heap profiling
 guest_debug = ["vmm/guest_debug"]
 igvm = ["vmm/igvm", "mshv"]
 io_uring = ["vmm/io_uring"]
@@ -75,27 +75,27 @@ tracing = ["vmm/tracing", "tracer/tracing"]
 
 [workspace]
 members = [
-    "api_client",
-    "arch",
-    "block",
-    "devices",
-    "event_monitor",
-    "hypervisor",
-    "net_gen",
-    "net_util",
-    "option_parser",
-    "pci",
-    "performance-metrics",
-    "rate_limiter",
-    "serial_buffer",
-    "test_infra",
-    "tracer",
-    "vhost_user_block",
-    "vhost_user_net",
-    "virtio-devices",
-    "vmm",
-    "vm-allocator",
-    "vm-device",
-    "vm-migration",
-    "vm-virtio"
+  "api_client",
+  "arch",
+  "block",
+  "devices",
+  "event_monitor",
+  "hypervisor",
+  "net_gen",
+  "net_util",
+  "option_parser",
+  "pci",
+  "performance-metrics",
+  "rate_limiter",
+  "serial_buffer",
+  "test_infra",
+  "tracer",
+  "vhost_user_block",
+  "vhost_user_net",
+  "virtio-devices",
+  "vmm",
+  "vm-allocator",
+  "vm-device",
+  "vm-migration",
+  "vm-virtio",
 ]

api_client/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "api_client"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "api_client"
+version = "0.1.0"
 
 [dependencies]
 vmm-sys-util = "0.12.1"

arch/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "arch"
-version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 edition = "2021"
+name = "arch"
+version = "0.1.0"
 
 [features]
 default = []
@@ -19,7 +19,10 @@ log = "0.4.21"
 serde = { version = "1.0.197", features = ["rc", "derive"] }
 thiserror = "1.0.60"
 uuid = "1.8.0"
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-bitmap",
+] }
 vm-migration = { path = "../vm-migration" }
 vmm-sys-util = { version = "0.12.1", features = ["with-serde"] }
 

block/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
+authors = ["The Cloud Hypervisor Authors", "The Chromium OS Authors"]
+edition = "2021"
 name = "block"
 version = "0.1.0"
-edition = "2021"
-authors = ["The Cloud Hypervisor Authors", "The Chromium OS Authors"]
 
 [features]
 default = []
@@ -21,6 +21,10 @@ thiserror = "1.0.60"
 uuid = { version = "1.8.0", features = ["v4"] }
 virtio-bindings = { version = "0.2.2", features = ["virtio-v5_0_0"] }
 virtio-queue = "0.12.0"
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+  "backend-bitmap",
+] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.12.1"

devices/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
-name = "devices"
-version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 edition = "2021"
+name = "devices"
+version = "0.1.0"
 
 [dependencies]
-acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"  }
+acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main" }
 anyhow = "1.0.81"
 arch = { path = "../arch" }
 bitflags = "2.5.0"

event_monitor/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "event_monitor"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "event_monitor"
+version = "0.1.0"
 
 [dependencies]
 flume = "0.11.0"

fuzz/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
-name = "cloud-hypervisor-fuzz"
-version = "0.0.0"
 authors = ["Automatically generated"]
-publish = false
 edition = "2021"
+name = "cloud-hypervisor-fuzz"
+publish = false
+version = "0.0.0"
 
 [package.metadata]
 cargo-fuzz = true
@@ -24,12 +24,12 @@ once_cell = "1.19.0"
 seccompiler = "0.4.0"
 virtio-devices = { path = "../virtio-devices" }
 virtio-queue = "0.12.0"
-vmm = { path = "../vmm", features = ["guest_debug"]}
-vmm-sys-util = "0.12.1"
+vm-device = { path = "../vm-device" }
 vm-memory = "0.14.1"
 vm-migration = { path = "../vm-migration" }
-vm-device = { path = "../vm-device" }
 vm-virtio = { path = "../vm-virtio" }
+vmm = { path = "../vmm", features = ["guest_debug"] }
+vmm-sys-util = "0.12.1"
 
 [dependencies.cloud-hypervisor]
 path = ".."
@@ -39,97 +39,97 @@ path = ".."
 members = ["."]
 
 [[bin]]
+doc = false
 name = "balloon"
 path = "fuzz_targets/balloon.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "block"
 path = "fuzz_targets/block.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "cmos"
 path = "fuzz_targets/cmos.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "console"
 path = "fuzz_targets/console.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "http_api"
 path = "fuzz_targets/http_api.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "iommu"
 path = "fuzz_targets/iommu.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "linux_loader"
 path = "fuzz_targets/linux_loader.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "linux_loader_cmdline"
 path = "fuzz_targets/linux_loader_cmdline.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "mem"
 path = "fuzz_targets/mem.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "net"
 path = "fuzz_targets/net.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "pmem"
 path = "fuzz_targets/pmem.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "qcow"
 path = "fuzz_targets/qcow.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "rng"
 path = "fuzz_targets/rng.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "serial"
 path = "fuzz_targets/serial.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "vhdx"
 path = "fuzz_targets/vhdx.rs"
 test = false
-doc = false
 
 [[bin]]
+doc = false
 name = "watchdog"
 path = "fuzz_targets/watchdog.rs"
 test = false
-doc = false

hypervisor/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
-name = "hypervisor"
-version = "0.1.0"
 authors = ["Microsoft Authors"]
 edition = "2021"
 license = "Apache-2.0 OR BSD-3-Clause"
+name = "hypervisor"
+version = "0.1.0"
 
 [features]
 kvm = ["kvm-ioctls", "kvm-bindings", "vfio-ioctls/kvm"]
@@ -14,26 +14,34 @@ tdx = []
 [dependencies]
 anyhow = "1.0.81"
 byteorder = "1.5.0"
-igvm = { version = "0.2.0", optional  = true }
-igvm_defs = { version = "0.2.0", optional  = true }
-libc = "0.2.153"
-log = "0.4.21"
+igvm = { version = "0.2.0", optional = true }
+igvm_defs = { version = "0.2.0", optional = true }
 kvm-bindings = { version = "0.8.1", optional = true, features = ["serde"] }
 kvm-ioctls = { version = "0.17.0", optional = true }
-mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
-mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
+libc = "0.2.153"
+log = "0.4.21"
+mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = [
+  "with-serde",
+  "fam-wrappers",
+], optional = true }
+mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional = true }
 serde = { version = "1.0.197", features = ["rc", "derive"] }
-serde_with = { version = "3.7.0", default-features = false, features = ["macros"] }
-vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic"] }
-vmm-sys-util = { version = "0.12.1", features = ["with-serde"] }
+serde_with = { version = "3.7.0", default-features = false, features = [
+  "macros",
+] }
 thiserror = "1.0.60"
+vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+] }
+vmm-sys-util = { version = "0.12.1", features = ["with-serde"] }
 
 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
-optional = true
-version = "1.21.0"
 default-features = false
 features = ["std", "decoder", "op_code_info", "instr_info", "fast_fmt"]
+optional = true
+version = "1.21.0"
 
 [dev-dependencies]
 env_logger = "0.11.3"

hypervisor/src/kvm/aarch64/mod.rs

@@ -106,12 +106,23 @@ pub fn is_system_register(regid: u64) -> bool {
 }
 
 pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
-    if !kvm.check_extension(Cap::SignalMsi) {
-        return Err(KvmError::CapabilityMissing(Cap::SignalMsi));
-    }
-    if !kvm.check_extension(Cap::OneReg) {
-        return Err(KvmError::CapabilityMissing(Cap::OneReg));
+    macro_rules! check_extension {
+        ($cap:expr) => {
+            if !kvm.check_extension($cap) {
+                return Err(KvmError::CapabilityMissing($cap));
+            }
+        };
     }
+
+    // SetGuestDebug is required but some kernels have it implemented without the capability flag.
+    check_extension!(Cap::ImmediateExit);
+    check_extension!(Cap::Ioeventfd);
+    check_extension!(Cap::Irqchip);
+    check_extension!(Cap::Irqfd);
+    check_extension!(Cap::IrqRouting);
+    check_extension!(Cap::MpState);
+    check_extension!(Cap::OneReg);
+    check_extension!(Cap::UserMemory);
     Ok(())
 }
 

hypervisor/src/kvm/x86_64/mod.rs

@@ -32,29 +32,37 @@ pub use {
 /// Check KVM extension for Linux
 ///
 pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
-    if !kvm.check_extension(Cap::SignalMsi) {
-        return Err(KvmError::CapabilityMissing(Cap::SignalMsi));
-    }
-    if !kvm.check_extension(Cap::TscDeadlineTimer) {
-        return Err(KvmError::CapabilityMissing(Cap::TscDeadlineTimer));
-    }
-    if !kvm.check_extension(Cap::SplitIrqchip) {
-        return Err(KvmError::CapabilityMissing(Cap::SplitIrqchip));
-    }
-    if !kvm.check_extension(Cap::SetIdentityMapAddr) {
-        return Err(KvmError::CapabilityMissing(Cap::SetIdentityMapAddr));
-    }
-    if !kvm.check_extension(Cap::SetTssAddr) {
-        return Err(KvmError::CapabilityMissing(Cap::SetTssAddr));
-    }
-    if !kvm.check_extension(Cap::ImmediateExit) {
-        return Err(KvmError::CapabilityMissing(Cap::ImmediateExit));
-    }
-    if !kvm.check_extension(Cap::GetTscKhz) {
-        return Err(KvmError::CapabilityMissing(Cap::GetTscKhz));
+    macro_rules! check_extension {
+        ($cap:expr) => {
+            if !kvm.check_extension($cap) {
+                return Err(KvmError::CapabilityMissing($cap));
+            }
+        };
     }
+
+    // DeviceCtrl, EnableCap, and SetGuestDebug are also required, but some kernels have
+    // the features implemented without the capability flags.
+    check_extension!(Cap::AdjustClock);
+    check_extension!(Cap::ExtCpuid);
+    check_extension!(Cap::GetTscKhz);
+    check_extension!(Cap::ImmediateExit);
+    check_extension!(Cap::Ioeventfd);
+    check_extension!(Cap::Irqchip);
+    check_extension!(Cap::Irqfd);
+    check_extension!(Cap::IrqRouting);
+    check_extension!(Cap::MpState);
+    check_extension!(Cap::SetIdentityMapAddr);
+    check_extension!(Cap::SetTssAddr);
+    check_extension!(Cap::SplitIrqchip);
+    check_extension!(Cap::TscDeadlineTimer);
+    check_extension!(Cap::UserMemory);
+    check_extension!(Cap::UserNmi);
+    check_extension!(Cap::VcpuEvents);
+    check_extension!(Cap::Xcrs);
+    check_extension!(Cap::Xsave);
     Ok(())
 }
+
 #[derive(Clone, Serialize, Deserialize)]
 pub struct VcpuKvmState {
     pub cpuid: Vec<CpuIdEntry>,

hypervisor/src/mshv/mod.rs

@@ -310,6 +310,13 @@ impl hypervisor::Hypervisor for MshvHypervisor {
         )
         .map_err(|e| hypervisor::HypervisorError::SetPartitionProperty(e.into()))?;
 
+        // Always create a frozen partition
+        fd.set_partition_property(
+            hv_partition_property_code_HV_PARTITION_PROPERTY_TIME_FREEZE,
+            1u64,
+        )
+        .map_err(|e| hypervisor::HypervisorError::SetPartitionProperty(e.into()))?;
+
         let vm_fd = Arc::new(fd);
 
         #[cfg(target_arch = "x86_64")]
@@ -2076,4 +2083,36 @@ impl vm::Vm for MshvVm {
     fn get_preferred_target(&self, kvi: &mut VcpuInit) -> vm::Result<()> {
         unimplemented!()
     }
+
+    /// Pause the VM
+    fn pause(&self) -> vm::Result<()> {
+        // Freeze the partition
+        self.fd
+            .set_partition_property(
+                hv_partition_property_code_HV_PARTITION_PROPERTY_TIME_FREEZE,
+                1u64,
+            )
+            .map_err(|e| {
+                vm::HypervisorVmError::SetVmProperty(anyhow!(
+                    "Failed to set partition property: {}",
+                    e
+                ))
+            })
+    }
+
+    /// Resume the VM
+    fn resume(&self) -> vm::Result<()> {
+        // Resuming the partition using TIME_FREEZE property
+        self.fd
+            .set_partition_property(
+                hv_partition_property_code_HV_PARTITION_PROPERTY_TIME_FREEZE,
+                0u64,
+            )
+            .map_err(|e| {
+                vm::HypervisorVmError::SetVmProperty(anyhow!(
+                    "Failed to set partition property: {}",
+                    e
+                ))
+            })
+    }
 }

hypervisor/src/vm.rs

@@ -231,6 +231,10 @@ pub enum HypervisorVmError {
     ///
     #[error("Failed to complete isolated import: {0}")]
     CompleteIsolatedImport(#[source] anyhow::Error),
+    /// Failed to set VM property
+    ///
+    #[error("Failed to set VM property: {0}")]
+    SetVmProperty(#[source] anyhow::Error),
 }
 ///
 /// Result type for returning from a function

net_gen/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "net_gen"
-version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 edition = "2021"
+name = "net_gen"
+version = "0.1.0"
 
 [dependencies]
 vmm-sys-util = "0.12.1"

net_util/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "net_util"
-version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 edition = "2021"
+name = "net_util"
+version = "0.1.0"
 
 [dependencies]
 epoll = "4.3.3"
@@ -11,11 +11,15 @@ libc = "0.2.153"
 log = "0.4.21"
 net_gen = { path = "../net_gen" }
 rate_limiter = { path = "../rate_limiter" }
-serde = {version = "1.0.197",features = ["derive"]}
+serde = { version = "1.0.197", features = ["derive"] }
 thiserror = "1.0.60"
 virtio-bindings = "0.2.2"
 virtio-queue = "0.12.0"
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+  "backend-bitmap",
+] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.12.1"
 

option_parser/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "option_parser"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "option_parser"
+version = "0.1.0"

pci/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "pci"
-version = "0.1.0"
 authors = ["Samuel Ortiz <sameo@linux.intel.com>"]
 edition = "2021"
+name = "pci"
+version = "0.1.0"
 
 [features]
 default = []
@@ -13,16 +13,21 @@ mshv = ["vfio-ioctls/mshv"]
 anyhow = "1.0.81"
 byteorder = "1.5.0"
 hypervisor = { path = "../hypervisor" }
-vfio-bindings = { git = "https://github.com/rust-vmm/vfio", branch = "main", features = ["fam-wrappers"] }
-vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
-vfio_user = { git = "https://github.com/rust-vmm/vfio-user", branch = "main" }
-
-vmm-sys-util = "0.12.1"
 libc = "0.2.153"
 log = "0.4.21"
 serde = { version = "1.0.197", features = ["derive"] }
 thiserror = "1.0.60"
+vfio-bindings = { git = "https://github.com/rust-vmm/vfio", branch = "main", features = [
+  "fam-wrappers",
+] }
+vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
+vfio_user = { git = "https://github.com/rust-vmm/vfio-user", branch = "main" }
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+  "backend-bitmap",
+] }
 vm-migration = { path = "../vm-migration" }
+vmm-sys-util = "0.12.1"

performance-metrics/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
+authors = ["The Cloud Hypervisor Authors"]
+build = "../build.rs"
+edition = "2021"
 name = "performance-metrics"
 version = "0.1.0"
-authors = ["The Cloud Hypervisor Authors"]
-edition = "2021"
-build = "../build.rs"
 
 [dependencies]
 clap = { version = "4.5.4", features = ["wrap_help"] }

rate_limiter/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
+edition = "2021"
 name = "rate_limiter"
 version = "0.1.0"
-edition = "2021"
 
 [dependencies]
 epoll = "4.3.3"

serial_buffer/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2021"
 name = "serial_buffer"
 version = "0.1.0"
-authors = ["The Cloud Hypervisor Authors"]
-edition = "2021"

src/bin/ch-remote.rs

@@ -987,7 +987,7 @@ fn main() {
         )
         .subcommand(
             Command::new("remove-device")
-                .about("Remove VFIO device")
+                .about("Remove VFIO and PCI device")
                 .arg(Arg::new("id").index(1).help("<device_id>")),
         )
         .subcommand(Command::new("info").about("Info on the VM"))

test_infra/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "test_infra"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "test_infra"
+version = "0.1.0"
 
 [dependencies]
 dirs = "5.0.1"

tpm/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "tpm"
-edition = "2021"
 authors = ["Microsoft Authors"]
+edition = "2021"
 license = "Apache-2.0"
+name = "tpm"
 version = "0.1.0"
 
 [dependencies]

tracer/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "tracer"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "tracer"
+version = "0.1.0"
 
 [dependencies]
 libc = "0.2.153"

vhost_user_block/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
+authors = ["The Cloud Hypervisor Authors"]
+build = "../build.rs"
+edition = "2021"
 name = "vhost_user_block"
 version = "0.1.0"
-authors = ["The Cloud Hypervisor Authors"]
-edition = "2021"
-build = "../build.rs"
 
 [dependencies]
-clap = { version = "4.5.4", features = ["wrap_help","cargo"] }
 block = { path = "../block" }
+clap = { version = "4.5.4", features = ["wrap_help", "cargo"] }
 env_logger = "0.11.3"
 epoll = "4.3.3"
 libc = "0.2.153"

vhost_user_net/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
+authors = ["The Cloud Hypervisor Authors"]
+build = "../build.rs"
+edition = "2021"
 name = "vhost_user_net"
 version = "0.1.0"
-authors = ["The Cloud Hypervisor Authors"]
-edition = "2021"
-build = "../build.rs"
 
 [dependencies]
-clap = { version = "4.5.4", features = ["wrap_help","cargo"] }
+clap = { version = "4.5.4", features = ["wrap_help", "cargo"] }
 env_logger = "0.11.3"
 epoll = "4.3.3"
 libc = "0.2.153"

virtio-devices/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "virtio-devices"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "virtio-devices"
+version = "0.1.0"
 
 [features]
 default = []
@@ -24,15 +24,26 @@ rate_limiter = { path = "../rate_limiter" }
 seccompiler = "0.4.0"
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.115"
-serde_with = { version = "3.7.0", default-features = false, features = ["macros"] }
+serde_with = { version = "3.7.0", default-features = false, features = [
+  "macros",
+] }
 serial_buffer = { path = "../serial_buffer" }
 thiserror = "1.0.60"
-vhost = { version = "0.11.0", features = ["vhost-user-frontend", "vhost-user-backend", "vhost-kern", "vhost-vdpa"] }
+vhost = { version = "0.11.0", features = [
+  "vhost-user-frontend",
+  "vhost-user-backend",
+  "vhost-kern",
+  "vhost-vdpa",
+] }
 virtio-bindings = { version = "0.2.2", features = ["virtio-v5_0_0"] }
 virtio-queue = "0.12.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+  "backend-bitmap",
+] }
 vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.12.1"

vm-allocator/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "vm-allocator"
-version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 edition = "2021"
+name = "vm-allocator"
+version = "0.1.0"
 
 [dependencies]
 libc = "0.2.153"

vm-device/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "vm-device"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "vm-device"
+version = "0.1.0"
 
 [features]
 default = []
@@ -12,8 +12,8 @@ mshv = ["vfio-ioctls/mshv"]
 [dependencies]
 anyhow = "1.0.81"
 hypervisor = { path = "../hypervisor" }
-thiserror = "1.0.60"
 serde = { version = "1.0.197", features = ["rc", "derive"] }
+thiserror = "1.0.60"
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vm-memory = { version = "0.14.1", features = ["backend-mmap"] }
 vmm-sys-util = "0.12.1"

vm-migration/Cargo.toml

@@ -1,12 +1,15 @@
 [package]
-name = "vm-migration"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "vm-migration"
+version = "0.1.0"
 
 [dependencies]
 anyhow = "1.0.81"
-thiserror = "1.0.60"
 serde = { version = "1.0.197", features = ["rc", "derive"] }
 serde_json = "1.0.115"
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic"] }
+thiserror = "1.0.60"
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+] }

vm-virtio/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "vm-virtio"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "vm-virtio"
+version = "0.1.0"
 
 [features]
 default = []
@@ -10,4 +10,8 @@ default = []
 [dependencies]
 log = "0.4.21"
 virtio-queue = "0.12.0"
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+  "backend-bitmap",
+] }

vmm/Cargo.toml

@@ -1,15 +1,15 @@
 [package]
-name = "vmm"
-version = "0.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
+name = "vmm"
+version = "0.1.0"
 
 [features]
-default = []
 dbus_api = ["blocking", "futures", "zbus"]
-dhat-heap = ["dhat"] # For heap profiling
+default = []
+dhat-heap = ["dhat"]                                                         # For heap profiling
 guest_debug = ["kvm", "gdbstub", "gdbstub_arch"]
-igvm = ["hex", "dep:igvm", "igvm_defs",  "mshv-bindings", "range_map_vec"]
+igvm = ["hex", "dep:igvm", "igvm_defs", "mshv-bindings", "range_map_vec"]
 io_uring = ["block/io_uring"]
 kvm = ["hypervisor/kvm", "vfio-ioctls/kvm", "vm-device/kvm", "pci/kvm"]
 mshv = ["hypervisor/mshv", "vfio-ioctls/mshv", "vm-device/mshv", "pci/mshv"]
@@ -18,7 +18,7 @@ tdx = ["arch/tdx", "hypervisor/tdx"]
 tracing = ["tracer/tracing"]
 
 [dependencies]
-acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"  }
+acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main" }
 anyhow = "1.0.81"
 arc-swap = "1.7.1"
 arch = { path = "../arch" }
@@ -37,13 +37,16 @@ gdbstub = { version = "0.7.1", optional = true }
 gdbstub_arch = { version = "0.3.0", optional = true }
 hex = { version = "0.4.3", optional = true }
 hypervisor = { path = "../hypervisor" }
-igvm = { version = "0.2.0", optional  = true }
-igvm_defs = { version = "0.2.0", optional  = true }
+igvm = { version = "0.2.0", optional = true }
+igvm_defs = { version = "0.2.0", optional = true }
 libc = "0.2.153"
 linux-loader = { version = "0.11.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.21"
 micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
-mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
+mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = [
+  "with-serde",
+  "fam-wrappers",
+], optional = true }
 net_util = { path = "../net_util" }
 once_cell = "1.19.0"
 option_parser = { path = "../option_parser" }
@@ -64,9 +67,13 @@ virtio-devices = { path = "../virtio-devices" }
 virtio-queue = "0.12.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.14.1", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.14.1", features = [
+  "backend-mmap",
+  "backend-atomic",
+  "backend-bitmap",
+] }
 vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = { version = "0.12.1", features = ["with-serde"] }
 zbus = { version = "3.15.2", optional = true }
-zerocopy = { version = "0.7.32", features = ["alloc","derive"] }
+zerocopy = { version = "0.7.32", features = ["alloc", "derive"] }

vmm/src/vm.rs

@@ -319,6 +319,9 @@ pub enum Error {
 
     #[error("Error injecting NMI")]
     ErrorNmi,
+
+    #[error("Error resuming the VM: {0}")]
+    ResumeVm(#[source] hypervisor::HypervisorVmError),
 }
 pub type Result<T> = result::Result<T, Error>;
 
@@ -2173,6 +2176,11 @@ impl Vm {
             self.vm.tdx_finalize().map_err(Error::FinalizeTdx)?;
         }
 
+        // Resume the vm for MSHV
+        if current_state == VmState::Created {
+            self.vm.resume().map_err(Error::ResumeVm)?;
+        }
+
         self.cpu_manager
             .lock()
             .unwrap()
@@ -2487,6 +2495,10 @@ impl Pausable for Vm {
         self.cpu_manager.lock().unwrap().pause()?;
         self.device_manager.lock().unwrap().pause()?;
 
+        self.vm
+            .pause()
+            .map_err(|e| MigratableError::Pause(anyhow!("Could not pause the VM: {}", e)))?;
+
         *state = new_state;
 
         event!("vm", "paused");
@@ -2495,6 +2507,7 @@ impl Pausable for Vm {
 
     fn resume(&mut self) -> std::result::Result<(), MigratableError> {
         event!("vm", "resuming");
+        let current_state = self.get_state().unwrap();
         let mut state = self
             .state
             .try_write()
@@ -2514,6 +2527,13 @@ impl Pausable for Vm {
                 })?;
             }
         }
+
+        if current_state == VmState::Paused {
+            self.vm
+                .resume()
+                .map_err(|e| MigratableError::Resume(anyhow!("Could not resume the VM: {}", e)))?;
+        }
+
         self.device_manager.lock().unwrap().resume()?;
 
         // And we're back to the Running state.