block: merge qcow, vhdx and block_util into block crate

This commit merges crates `qcow`, `vhdx` and `block_util` into the
crate `block`, which can allow `qcow` to use functions from `block_util`
without introducing a circular crate dependency.

This commit is based on crosvm implementation:
f2eecc4152

Signed-off-by: Yu Li <liyu.yukiteru@bytedance.com>
This commit is contained in:
Yu Li 2023-07-12 10:24:28 +08:00 committed by Rob Bradford
parent 47fbe9af1e
commit 447cad3861
38 changed files with 117 additions and 201 deletions

75
Cargo.lock generated
View File

@ -273,6 +273,28 @@ version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
[[package]]
name = "block"
version = "0.1.0"
dependencies = [
"byteorder",
"crc32c",
"io-uring",
"libc",
"log",
"remain",
"smallvec",
"thiserror",
"uuid",
"versionize",
"versionize_derive",
"virtio-bindings",
"virtio-queue",
"vm-memory",
"vm-virtio",
"vmm-sys-util",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -282,26 +304,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "block_util"
version = "0.1.0"
dependencies = [
"io-uring",
"libc",
"log",
"qcow",
"smallvec",
"thiserror",
"versionize",
"versionize_derive",
"vhdx",
"virtio-bindings",
"virtio-queue",
"vm-memory",
"vm-virtio",
"vmm-sys-util",
]
[[package]]
name = "blocking"
version = "1.3.1"
@ -1452,17 +1454,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "qcow"
version = "0.1.0"
dependencies = [
"byteorder",
"libc",
"log",
"remain",
"vmm-sys-util",
]
[[package]]
name = "quote"
version = "1.0.29"
@ -2070,19 +2061,6 @@ dependencies = [
"vmm-sys-util",
]
[[package]]
name = "vhdx"
version = "0.1.0"
dependencies = [
"byteorder",
"crc32c",
"libc",
"log",
"remain",
"thiserror",
"uuid",
]
[[package]]
name = "vhost"
version = "0.7.0"
@ -2114,13 +2092,12 @@ name = "vhost_user_block"
version = "0.1.0"
dependencies = [
"argh",
"block_util",
"block",
"env_logger",
"epoll",
"libc",
"log",
"option_parser",
"qcow",
"vhost",
"vhost-user-backend",
"virtio-bindings",
@ -2159,7 +2136,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"arc-swap",
"block_util",
"block",
"byteorder",
"epoll",
"event_monitor",
@ -2268,7 +2245,7 @@ dependencies = [
"arc-swap",
"arch",
"bitflags 2.3.3",
"block_util",
"block",
"blocking",
"devices",
"epoll",
@ -2285,7 +2262,6 @@ dependencies = [
"once_cell",
"option_parser",
"pci",
"qcow",
"seccompiler",
"serde",
"serde_json",
@ -2298,7 +2274,6 @@ dependencies = [
"versionize_derive",
"vfio-ioctls",
"vfio_user",
"vhdx",
"virtio-devices",
"virtio-queue",
"vm-allocator",

View File

@ -80,7 +80,7 @@ tracing = ["vmm/tracing", "tracer/tracing"]
members = [
"api_client",
"arch",
"block_util",
"block",
"devices",
"event_monitor",
"hypervisor",
@ -89,12 +89,10 @@ members = [
"option_parser",
"pci",
"performance-metrics",
"qcow",
"rate_limiter",
"serial_buffer",
"test_infra",
"tracer",
"vhdx",
"vhost_user_block",
"vhost_user_net",
"virtio-devices",

View File

@ -1,26 +1,27 @@
[package]
name = "block_util"
name = "block"
version = "0.1.0"
authors = ["The Cloud Hypervisor Authors"]
edition = "2021"
authors = ["The Cloud Hypervisor Authors", "The Chromium OS Authors"]
[features]
default = []
io_uring = ["dep:io-uring"]
[dependencies]
byteorder = "1.4.3"
crc32c = "0.6.3"
io-uring = { version = "0.5.13", optional = true }
libc = "0.2.139"
log = "0.4.17"
qcow = { path = "../qcow" }
remain = "0.2.6"
smallvec = "1.10.0"
thiserror = "1.0.40"
uuid = { version = "1.3.4", features = ["v4"] }
versionize = "0.1.10"
versionize_derive = "0.1.4"
vhdx = { path = "../vhdx" }
virtio-bindings = { version = "0.2.0", features = ["virtio-v5_0_0"] }
virtio-queue = "0.8.0"
vm-memory = { version = "0.11.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
vm-virtio = { path = "../vm-virtio" }
vmm-sys-util = "0.11.0"

View File

@ -17,6 +17,7 @@ pub mod fixed_vhd;
/// Enabled with the `"io_uring"` feature
pub mod fixed_vhd_async;
pub mod fixed_vhd_sync;
pub mod qcow;
pub mod qcow_sync;
#[cfg(feature = "io_uring")]
/// Async primitives based on `io-uring`
@ -25,6 +26,7 @@ pub mod qcow_sync;
pub mod raw_async;
pub mod raw_sync;
pub mod vhd;
pub mod vhdx;
pub mod vhdx_sync;
use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};

View File

@ -2,17 +2,16 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-BSD-3-Clause file.
#[macro_use]
extern crate log;
mod qcow_raw_file;
mod raw_file;
mod refcount;
mod vec_cache;
use crate::qcow_raw_file::QcowRawFile;
use crate::refcount::RefCount;
use crate::vec_cache::{CacheMap, Cacheable, VecCache};
use crate::qcow::{
qcow_raw_file::QcowRawFile,
refcount::RefCount,
vec_cache::{CacheMap, Cacheable, VecCache},
};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use libc::{EINVAL, ENOSPC, ENOTSUP};
use remain::sorted;
@ -25,7 +24,7 @@ use vmm_sys_util::{
write_zeroes::WriteZeroesAt,
};
pub use crate::raw_file::RawFile;
pub use crate::qcow::raw_file::RawFile;
#[sorted]
#[derive(Debug)]
@ -362,8 +361,8 @@ fn max_refcount_clusters(refcount_order: u32, cluster_size: u32, num_clusters: u
/// # Example
///
/// ```
/// # use block::qcow::{self, QcowFile, RawFile};
/// # use std::io::{Read, Seek, SeekFrom};
/// # use qcow::{self, QcowFile, RawFile};
/// # fn test(file: std::fs::File) -> std::io::Result<()> {
/// let mut raw_img = RawFile::new(file, false);
/// let mut q = QcowFile::from(raw_img).expect("Can't open qcow file");

View File

@ -7,8 +7,10 @@ use std::io;
use libc::EINVAL;
use crate::qcow_raw_file::QcowRawFile;
use crate::vec_cache::{CacheMap, Cacheable, VecCache};
use crate::qcow::{
qcow_raw_file::QcowRawFile,
vec_cache::{CacheMap, Cacheable, VecCache},
};
#[derive(Debug)]
pub enum Error {

View File

@ -3,8 +3,8 @@
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use crate::async_io::{AsyncIo, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult};
use crate::qcow::{QcowFile, RawFile, Result as QcowResult};
use crate::AsyncAdaptor;
use qcow::{QcowFile, RawFile, Result as QcowResult};
use std::collections::VecDeque;
use std::fs::File;
use std::io::{Seek, SeekFrom};

View File

@ -2,15 +2,30 @@
//
// SPDX-License-Identifier: Apache-2.0
use crate::vhdx_bat::{BatEntry, VhdxBatError};
use crate::vhdx_header::{self, RegionInfo, RegionTableEntry, VhdxHeader, VhdxHeaderError};
use crate::vhdx_io::{self, VhdxIoError};
use crate::vhdx_metadata::{DiskSpec, VhdxMetadataError};
use crate::vhdx::{
vhdx_bat::{BatEntry, VhdxBatError},
vhdx_header::{RegionInfo, RegionTableEntry, VhdxHeader, VhdxHeaderError},
vhdx_io::VhdxIoError,
vhdx_metadata::{DiskSpec, VhdxMetadataError},
};
use byteorder::{BigEndian, ByteOrder};
use remain::sorted;
use std::collections::btree_map::BTreeMap;
use std::fs::File;
use std::io::{Read, Seek, SeekFrom, Write};
use thiserror::Error;
use uuid::Uuid;
macro_rules! div_round_up {
($n:expr,$d:expr) => {
($n + $d - 1) / $d
};
}
mod vhdx_bat;
mod vhdx_header;
mod vhdx_io;
mod vhdx_metadata;
#[sorted]
#[derive(Error, Debug)]
@ -208,3 +223,15 @@ impl Clone for Vhdx {
}
}
}
pub(crate) fn uuid_from_guid(buf: &[u8]) -> Uuid {
// The first 3 fields of UUID are stored in Big Endian format, and
// the last 8 bytes are stored as byte array. Therefore, we read the
// first 3 fields in Big Endian format instead of Little Endian.
Uuid::from_fields_le(
BigEndian::read_u32(&buf[0..4]),
BigEndian::read_u16(&buf[4..6]),
BigEndian::read_u16(&buf[6..8]),
buf[8..16].try_into().unwrap(),
)
}

View File

@ -2,8 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
use crate::vhdx_header::RegionTableEntry;
use crate::vhdx_metadata::DiskSpec;
use crate::vhdx::{vhdx_header::RegionTableEntry, vhdx_metadata::DiskSpec};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use remain::sorted;
use std::fs::File;

View File

@ -344,7 +344,7 @@ impl RegionTableEntry {
// SAFETY: the assertion above makes sure the buffer size is correct.
let mut region_table_entry = unsafe { *(buffer.as_ptr() as *mut RegionTableEntry) };
let uuid = crate::uuid_from_guid(buffer);
let uuid = crate::vhdx::uuid_from_guid(buffer);
region_table_entry.guid = uuid;
Ok(region_table_entry)

View File

@ -2,8 +2,10 @@
//
// SPDX-License-Identifier: Apache-2.0
use crate::vhdx_bat::{self, BatEntry, VhdxBatError};
use crate::vhdx_metadata::{self, DiskSpec};
use crate::vhdx::{
vhdx_bat::{self, BatEntry, VhdxBatError},
vhdx_metadata::{self, DiskSpec},
};
use remain::sorted;
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom, Write};

View File

@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
use crate::vhdx_header::RegionTableEntry;
use crate::vhdx::vhdx_header::RegionTableEntry;
use byteorder::{LittleEndian, ReadBytesExt};
use remain::sorted;
use std::fs::File;
@ -303,7 +303,7 @@ impl MetadataTableEntry {
// SAFETY: the assertion above makes sure the buffer size is correct.
let mut metadata_table_entry = unsafe { *(buffer.as_ptr() as *mut MetadataTableEntry) };
let uuid = crate::uuid_from_guid(buffer);
let uuid = crate::vhdx::uuid_from_guid(buffer);
metadata_table_entry.item_id = uuid;
if metadata_table_entry.length > METADATA_LENGTH_MAX {

View File

@ -3,11 +3,11 @@
// SPDX-License-Identifier: Apache-2.0
use crate::async_io::{AsyncIo, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult};
use crate::vhdx::{Result as VhdxResult, Vhdx};
use crate::AsyncAdaptor;
use std::collections::VecDeque;
use std::fs::File;
use std::sync::{Arc, Mutex, MutexGuard};
use vhdx::vhdx::{Result as VhdxResult, Vhdx};
use vmm_sys_util::eventfd::EventFd;
pub struct VhdxDiskSync {

42
fuzz/Cargo.lock generated
View File

@ -107,18 +107,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
[[package]]
name = "block_util"
name = "block"
version = "0.1.0"
dependencies = [
"byteorder",
"crc32c",
"io-uring",
"libc",
"log",
"qcow",
"remain",
"smallvec",
"thiserror",
"uuid",
"versionize",
"versionize_derive",
"vhdx",
"virtio-bindings",
"virtio-queue",
"vm-memory",
@ -175,7 +177,7 @@ dependencies = [
name = "cloud-hypervisor-fuzz"
version = "0.0.0"
dependencies = [
"block_util",
"block",
"cloud-hypervisor",
"devices",
"epoll",
@ -185,9 +187,7 @@ dependencies = [
"micro_http",
"net_util",
"once_cell",
"qcow",
"seccompiler",
"vhdx",
"virtio-devices",
"virtio-queue",
"vm-device",
@ -495,17 +495,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "qcow"
version = "0.1.0"
dependencies = [
"byteorder",
"libc",
"log",
"remain",
"vmm-sys-util",
]
[[package]]
name = "quote"
version = "1.0.29"
@ -802,19 +791,6 @@ dependencies = [
"vmm-sys-util",
]
[[package]]
name = "vhdx"
version = "0.1.0"
dependencies = [
"byteorder",
"crc32c",
"libc",
"log",
"remain",
"thiserror",
"uuid",
]
[[package]]
name = "vhost"
version = "0.7.0"
@ -839,7 +815,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"arc-swap",
"block_util",
"block",
"byteorder",
"epoll",
"event_monitor",
@ -948,7 +924,7 @@ dependencies = [
"arc-swap",
"arch",
"bitflags 2.3.3",
"block_util",
"block",
"devices",
"epoll",
"event_monitor",
@ -961,7 +937,6 @@ dependencies = [
"once_cell",
"option_parser",
"pci",
"qcow",
"seccompiler",
"serde",
"serde_json",
@ -974,7 +949,6 @@ dependencies = [
"versionize_derive",
"vfio-ioctls",
"vfio_user",
"vhdx",
"virtio-devices",
"virtio-queue",
"vm-allocator",

View File

@ -9,7 +9,7 @@ edition = "2021"
cargo-fuzz = true
[dependencies]
block_util = { path = "../block_util" }
block = { path = "../block" }
devices = { path = "../devices" }
epoll = "4.3.1"
libc = "0.2.147"
@ -18,9 +18,7 @@ linux-loader = { version = "0.9.0", features = ["elf", "bzimage", "pe"] }
micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
net_util = { path = "../net_util" }
once_cell = "1.18.0"
qcow = { path = "../qcow" }
seccompiler = "0.3.0"
vhdx = { path = "../vhdx" }
virtio-devices = { path = "../virtio-devices" }
virtio-queue = "0.8.0"
vmm = { path = "../vmm" }

View File

@ -8,7 +8,7 @@
#![no_main]
use block_util::{async_io::DiskFile, raw_sync::RawFileDiskSync};
use block::{async_io::DiskFile, raw_sync::RawFileDiskSync};
use libfuzzer_sys::fuzz_target;
use seccompiler::SeccompAction;
use std::ffi;

View File

@ -4,7 +4,7 @@
#![no_main]
use libfuzzer_sys::fuzz_target;
use qcow::{QcowFile, RawFile};
use block::qcow::{QcowFile, RawFile};
use std::ffi;
use std::fs::File;
use std::io::{self, Cursor, Read, Seek, SeekFrom, Write};

View File

@ -8,7 +8,7 @@ use std::ffi;
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::os::unix::io::{FromRawFd, RawFd};
use vhdx::vhdx::Vhdx;
use block::vhdx::Vhdx;
// Populate the corpus directory with a test file:
// truncate -s 16M /tmp/source

View File

@ -1,16 +0,0 @@
[package]
name = "qcow"
version = "0.1.0"
authors = ["The Chromium OS Authors"]
edition = "2021"
license = "BSD-3-Clause"
[lib]
path = "src/qcow.rs"
[dependencies]
byteorder = "1.4.3"
libc = "0.2.139"
log = "0.4.17"
remain = "0.2.10"
vmm-sys-util = "0.11.0"

View File

@ -1,15 +0,0 @@
[package]
name = "vhdx"
version = "0.1.0"
authors = ["The Cloud Hypervisor Authors"]
edition = "2021"
license = "Apache-2.0"
[dependencies]
byteorder = "1.4.3"
crc32c = "0.6.3"
libc = "0.2.139"
log = "0.4.17"
remain = "0.2.10"
thiserror = "1.0.40"
uuid = { version = "1.3.4", features = ["v4"] }

View File

@ -1,30 +0,0 @@
// Copyright © 2021 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
use byteorder::{BigEndian, ByteOrder};
use uuid::Uuid;
macro_rules! div_round_up {
($n:expr,$d:expr) => {
($n + $d - 1) / $d
};
}
pub mod vhdx;
mod vhdx_bat;
mod vhdx_header;
mod vhdx_io;
mod vhdx_metadata;
pub(crate) fn uuid_from_guid(buf: &[u8]) -> Uuid {
// The first 3 fields of UUID are stored in Big Endian format, and
// the last 8 bytes are stored as byte array. Therefore, we read the
// first 3 fields in Big Endian format instead of Little Endian.
Uuid::from_fields_le(
BigEndian::read_u32(&buf[0..4]),
BigEndian::read_u16(&buf[4..6]),
BigEndian::read_u16(&buf[6..8]),
buf[8..16].try_into().unwrap(),
)
}

View File

@ -7,13 +7,12 @@ build = "../build.rs"
[dependencies]
argh = "0.1.9"
block_util = { path = "../block_util" }
block = { path = "../block" }
env_logger = "0.10.0"
epoll = "4.3.3"
libc = "0.2.139"
log = "0.4.17"
option_parser = { path = "../option_parser" }
qcow = { path = "../qcow" }
vhost = { version = "0.7.0", features = ["vhost-user-slave"] }
vhost-user-backend = "0.9.0"
virtio-bindings = "0.2.0"

View File

@ -8,11 +8,14 @@
//
// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)
use block_util::{build_disk_image_id, Request, VirtioBlockConfig};
use block::{
build_disk_image_id,
qcow::{self, ImageType, QcowFile},
Request, VirtioBlockConfig,
};
use libc::EFD_NONBLOCK;
use log::*;
use option_parser::{OptionParser, OptionParserError, Toggle};
use qcow::{self, ImageType, QcowFile};
use std::fs::File;
use std::fs::OpenOptions;
use std::io::Read;

View File

@ -10,7 +10,7 @@ default = []
[dependencies]
anyhow = "1.0.71"
arc-swap = "1.5.1"
block_util = { path = "../block_util" }
block = { path = "../block" }
byteorder = "1.4.3"
epoll = "4.3.3"
event_monitor = { path = "../event_monitor" }

View File

@ -19,7 +19,7 @@ use crate::thread_helper::spawn_virtio_thread;
use crate::GuestMemoryMmap;
use crate::VirtioInterrupt;
use anyhow::anyhow;
use block_util::{
use block::{
async_io::AsyncIo, async_io::AsyncIoError, async_io::DiskFile, build_disk_image_id, Request,
RequestType, VirtioBlockConfig,
};
@ -63,11 +63,11 @@ const LATENCY_SCALE: u64 = 10000;
#[derive(Error, Debug)]
pub enum Error {
#[error("Failed to parse the request: {0}")]
RequestParsing(block_util::Error),
RequestParsing(block::Error),
#[error("Failed to execute the request: {0}")]
RequestExecuting(block_util::ExecuteError),
RequestExecuting(block::ExecuteError),
#[error("Failed to complete the request: {0}")]
RequestCompleting(block_util::Error),
RequestCompleting(block::Error),
#[error("Missing the expected entry in the list of requests")]
MissingEntryRequestList,
#[error("The asynchronous request returned with failure")]

View File

@ -9,7 +9,7 @@ use crate::thread_helper::spawn_virtio_thread;
use crate::vhost_user::VhostUserCommon;
use crate::{GuestMemoryMmap, GuestRegionMmap};
use crate::{VirtioInterrupt, VIRTIO_F_IOMMU_PLATFORM};
use block_util::VirtioBlockConfig;
use block::VirtioBlockConfig;
use seccompiler::SeccompAction;
use std::mem;
use std::result;

View File

@ -8,7 +8,7 @@ edition = "2021"
default = []
dbus_api = ["blocking", "futures", "zbus"]
guest_debug = ["kvm", "gdbstub", "gdbstub_arch"]
io_uring = ["block_util/io_uring"]
io_uring = ["block/io_uring"]
kvm = ["hypervisor/kvm", "vfio-ioctls/kvm", "vm-device/kvm", "pci/kvm"]
mshv = ["hypervisor/mshv", "vfio-ioctls/mshv", "vm-device/mshv", "pci/mshv"]
tdx = ["arch/tdx", "hypervisor/tdx"]
@ -20,7 +20,7 @@ anyhow = "1.0.71"
arc-swap = "1.5.1"
arch = { path = "../arch" }
bitflags = "2.3.3"
block_util = { path = "../block_util" }
block = { path = "../block" }
blocking = { version = "1.3.0", optional = true }
devices = { path = "../devices" }
epoll = "4.3.3"
@ -37,7 +37,6 @@ net_util = { path = "../net_util" }
once_cell = "1.18.0"
option_parser = { path = "../option_parser" }
pci = { path = "../pci" }
qcow = { path = "../qcow" }
seccompiler = "0.3.0"
serde = { version = "1.0.164", features = ["rc", "derive"] }
serde_json = "1.0.96"
@ -50,7 +49,6 @@ versionize = "0.1.10"
versionize_derive = "0.1.4"
vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
vfio_user = { git = "https://github.com/rust-vmm/vfio-user", branch = "main" }
vhdx = { path = "../vhdx" }
virtio-devices = { path = "../virtio-devices" }
virtio-queue = "0.8.0"
vm-allocator = { path = "../vm-allocator" }

View File

@ -34,13 +34,13 @@ use arch::layout::{APIC_START, IOAPIC_SIZE, IOAPIC_START};
use arch::NumaNodes;
#[cfg(target_arch = "aarch64")]
use arch::{DeviceType, MmioDeviceInfo};
use block_util::{
use block::{
async_io::DiskFile, block_io_uring_is_supported, detect_image_type,
fixed_vhd_sync::FixedVhdDiskSync, qcow_sync::QcowDiskSync, raw_sync::RawFileDiskSync,
vhdx_sync::VhdxDiskSync, ImageType,
fixed_vhd_sync::FixedVhdDiskSync, qcow, qcow_sync::QcowDiskSync, raw_sync::RawFileDiskSync,
vhdx, vhdx_sync::VhdxDiskSync, ImageType,
};
#[cfg(feature = "io_uring")]
use block_util::{fixed_vhd_async::FixedVhdDiskAsync, raw_async::RawFileDisk};
use block::{fixed_vhd_async::FixedVhdDiskAsync, raw_async::RawFileDisk};
#[cfg(target_arch = "aarch64")]
use devices::gic;
#[cfg(target_arch = "x86_64")]
@ -424,7 +424,7 @@ pub enum DeviceManagerError {
CreateQcowDiskSync(qcow::Error),
/// Failed to create FixedVhdxDiskSync
CreateFixedVhdxDiskSync(vhdx::vhdx::VhdxError),
CreateFixedVhdxDiskSync(vhdx::VhdxError),
/// Failed to add DMA mapping handler to virtio-mem device.
AddDmaMappingHandlerVirtioMem(virtio_devices::mem::Error),