External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2024-11-05 11:31:14 +00:00
Code Issues Packages Projects Releases Wiki Activity
e0125653b1
cloud-hypervisor/virtio-devices/src/iommu.rs
Philipp Schuster ad6c0ee52b virtio-devices: properly join all threads on Drop 
This change is important to do a proper resource cleanup. We decided
to do this repetitive approach as VirtioCommon can't implement Drop
without major changes to the corresponding code. Also, devices such as
Net can't easily use the epoll_threads-abstraction from VirtioCommon as
it has multiple threads with different semantics.

Signed-off-by: Philipp Schuster <philipp.schuster@cyberus-technology.de>
2023-01-12 18:03:33 +00:00

1118 lines
37 KiB
Rust
Raw Blame History

// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
use super::Error as DeviceError;
use super::{
ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon, VirtioDevice,
VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1,
};
use crate::seccomp_filters::Thread;
use crate::thread_helper::spawn_virtio_thread;
use crate::GuestMemoryMmap;
use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
use anyhow::anyhow;
use seccompiler::SeccompAction;
use std::collections::BTreeMap;
use std::io;
use std::mem::size_of;
use std::ops::Bound::Included;
use std::os::unix::io::AsRawFd;
use std::result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Barrier, Mutex, RwLock};
use thiserror::Error;
use versionize::{VersionMap, Versionize, VersionizeResult};
use versionize_derive::Versionize;
use virtio_queue::{DescriptorChain, Queue, QueueT};
use vm_device::dma_mapping::ExternalDmaMapping;
use vm_memory::{
Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
GuestMemoryError, GuestMemoryLoadGuard,
};
use vm_migration::VersionMapped;
use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
use vm_virtio::AccessPlatform;
use vmm_sys_util::eventfd::EventFd;
/// Queues sizes
const QUEUE_SIZE: u16 = 256;
const NUM_QUEUES: usize = 2;
const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES];
/// New descriptors are pending on the request queue.
/// "requestq" is meant to be used anytime an action is required to be
/// performed on behalf of the guest driver.
const REQUEST_Q_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;
/// New descriptors are pending on the event queue.
/// "eventq" lets the device report any fault or other asynchronous event to
/// the guest driver.
const _EVENT_Q_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2;
/// PROBE properties size.
/// This is the minimal size to provide at least one RESV_MEM property.
/// Because virtio-iommu expects one MSI reserved region, we must provide it,
/// otherwise the driver in the guest will define a predefined one between
/// 0x8000000 and 0x80FFFFF, which is only relevant for ARM architecture, but
/// will conflict with x86.
const PROBE_PROP_SIZE: u32 =
(size_of::<VirtioIommuProbeProperty>() + size_of::<VirtioIommuProbeResvMem>()) as u32;
/// Virtio IOMMU features
#[allow(unused)]
const VIRTIO_IOMMU_F_INPUT_RANGE: u32 = 0;
#[allow(unused)]
const VIRTIO_IOMMU_F_DOMAIN_RANGE: u32 = 1;
#[allow(unused)]
const VIRTIO_IOMMU_F_MAP_UNMAP: u32 = 2;
#[allow(unused)]
const VIRTIO_IOMMU_F_BYPASS: u32 = 3;
const VIRTIO_IOMMU_F_PROBE: u32 = 4;
#[allow(unused)]
const VIRTIO_IOMMU_F_MMIO: u32 = 5;
const VIRTIO_IOMMU_F_BYPASS_CONFIG: u32 = 6;
// Support 2MiB and 4KiB page sizes.
const VIRTIO_IOMMU_PAGE_SIZE_MASK: u64 = (2 << 20) | (4 << 10);
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuRange32 {
start: u32,
end: u32,
}
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuRange64 {
start: u64,
end: u64,
}
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuConfig {
page_size_mask: u64,
input_range: VirtioIommuRange64,
domain_range: VirtioIommuRange32,
probe_size: u32,
bypass: u8,
_reserved: [u8; 7],
}
/// Virtio IOMMU request type
const VIRTIO_IOMMU_T_ATTACH: u8 = 1;
const VIRTIO_IOMMU_T_DETACH: u8 = 2;
const VIRTIO_IOMMU_T_MAP: u8 = 3;
const VIRTIO_IOMMU_T_UNMAP: u8 = 4;
const VIRTIO_IOMMU_T_PROBE: u8 = 5;
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
struct VirtioIommuReqHead {
type_: u8,
_reserved: [u8; 3],
}
/// Virtio IOMMU request status
const VIRTIO_IOMMU_S_OK: u8 = 0;
#[allow(unused)]
const VIRTIO_IOMMU_S_IOERR: u8 = 1;
#[allow(unused)]
const VIRTIO_IOMMU_S_UNSUPP: u8 = 2;
#[allow(unused)]
const VIRTIO_IOMMU_S_DEVERR: u8 = 3;
#[allow(unused)]
const VIRTIO_IOMMU_S_INVAL: u8 = 4;
#[allow(unused)]
const VIRTIO_IOMMU_S_RANGE: u8 = 5;
#[allow(unused)]
const VIRTIO_IOMMU_S_NOENT: u8 = 6;
#[allow(unused)]
const VIRTIO_IOMMU_S_FAULT: u8 = 7;
#[allow(unused)]
const VIRTIO_IOMMU_S_NOMEM: u8 = 8;
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuReqTail {
status: u8,
_reserved: [u8; 3],
}
/// ATTACH request
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
struct VirtioIommuReqAttach {
domain: u32,
endpoint: u32,
flags: u32,
_reserved: [u8; 4],
}
const VIRTIO_IOMMU_ATTACH_F_BYPASS: u32 = 1;
/// DETACH request
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
struct VirtioIommuReqDetach {
domain: u32,
endpoint: u32,
_reserved: [u8; 8],
}
/// Virtio IOMMU request MAP flags
#[allow(unused)]
const VIRTIO_IOMMU_MAP_F_READ: u32 = 1;
#[allow(unused)]
const VIRTIO_IOMMU_MAP_F_WRITE: u32 = 1 << 1;
#[allow(unused)]
const VIRTIO_IOMMU_MAP_F_MMIO: u32 = 1 << 2;
#[allow(unused)]
const VIRTIO_IOMMU_MAP_F_MASK: u32 =
VIRTIO_IOMMU_MAP_F_READ | VIRTIO_IOMMU_MAP_F_WRITE | VIRTIO_IOMMU_MAP_F_MMIO;
/// MAP request
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
struct VirtioIommuReqMap {
domain: u32,
virt_start: u64,
virt_end: u64,
phys_start: u64,
_flags: u32,
}
/// UNMAP request
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
struct VirtioIommuReqUnmap {
domain: u32,
virt_start: u64,
virt_end: u64,
_reserved: [u8; 4],
}
/// Virtio IOMMU request PROBE types
#[allow(unused)]
const VIRTIO_IOMMU_PROBE_T_NONE: u16 = 0;
const VIRTIO_IOMMU_PROBE_T_RESV_MEM: u16 = 1;
#[allow(unused)]
const VIRTIO_IOMMU_PROBE_T_MASK: u16 = 0xfff;
/// PROBE request
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuReqProbe {
endpoint: u32,
_reserved: [u64; 8],
}
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuProbeProperty {
type_: u16,
length: u16,
}
/// Virtio IOMMU request PROBE property RESV_MEM subtypes
#[allow(unused)]
const VIRTIO_IOMMU_RESV_MEM_T_RESERVED: u8 = 0;
const VIRTIO_IOMMU_RESV_MEM_T_MSI: u8 = 1;
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
#[allow(dead_code)]
struct VirtioIommuProbeResvMem {
subtype: u8,
_reserved: [u8; 3],
start: u64,
end: u64,
}
/// Virtio IOMMU fault flags
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_F_READ: u32 = 1;
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_F_WRITE: u32 = 1 << 1;
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_F_EXEC: u32 = 1 << 2;
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_F_ADDRESS: u32 = 1 << 8;
/// Virtio IOMMU fault reasons
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_R_UNKNOWN: u32 = 0;
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_R_DOMAIN: u32 = 1;
#[allow(unused)]
const VIRTIO_IOMMU_FAULT_R_MAPPING: u32 = 2;
/// Fault reporting through eventq
#[allow(unused)]
#[derive(Copy, Clone, Debug, Default)]
#[repr(packed)]
struct VirtioIommuFault {
reason: u8,
reserved: [u8; 3],
flags: u32,
endpoint: u32,
reserved2: [u8; 4],
address: u64,
}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuRange32 {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuRange64 {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuConfig {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqHead {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqTail {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqAttach {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqDetach {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqMap {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqUnmap {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuReqProbe {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuProbeProperty {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuProbeResvMem {}
// SAFETY: data structure only contain integers and have no implicit padding
unsafe impl ByteValued for VirtioIommuFault {}
#[derive(Error, Debug)]
enum Error {
#[error("Guest gave us bad memory addresses: {0}")]
GuestMemory(GuestMemoryError),
#[error("Guest gave us a write only descriptor that protocol says to read from")]
UnexpectedWriteOnlyDescriptor,
#[error("Guest gave us a read only descriptor that protocol says to write to")]
UnexpectedReadOnlyDescriptor,
#[error("Guest gave us too few descriptors in a descriptor chain")]
DescriptorChainTooShort,
#[error("Guest gave us a buffer that was too short to use")]
BufferLengthTooSmall,
#[error("Guest sent us invalid request")]
InvalidRequest,
#[error("Guest sent us invalid ATTACH request")]
InvalidAttachRequest,
#[error("Guest sent us invalid DETACH request")]
InvalidDetachRequest,
#[error("Guest sent us invalid MAP request")]
InvalidMapRequest,
#[error("Invalid to map because the domain is in bypass mode")]
InvalidMapRequestBypassDomain,
#[error("Invalid to map because the domain is missing")]
InvalidMapRequestMissingDomain,
#[error("Guest sent us invalid UNMAP request")]
InvalidUnmapRequest,
#[error("Invalid to unmap because the domain is in bypass mode")]
InvalidUnmapRequestBypassDomain,
#[error("Invalid to unmap because the domain is missing")]
InvalidUnmapRequestMissingDomain,
#[error("Guest sent us invalid PROBE request")]
InvalidProbeRequest,
#[error("Failed to performing external mapping: {0}")]
ExternalMapping(io::Error),
#[error("Failed to performing external unmapping: {0}")]
ExternalUnmapping(io::Error),
#[error("Failed adding used index: {0}")]
QueueAddUsed(virtio_queue::Error),
}
struct Request {}
impl Request {
// Parse the available vring buffer. Based on the hashmap table of external
// mappings required from various devices such as VFIO or vhost-user ones,
// this function might update the hashmap table of external mappings per
// domain.
// Basically, the VMM knows about the device_id <=> mapping relationship
// before running the VM, but at runtime, a new domain <=> mapping hashmap
// is created based on the information provided from the guest driver for
// virtio-iommu (giving the link device_id <=> domain).
fn parse(
desc_chain: &mut DescriptorChain<GuestMemoryLoadGuard<GuestMemoryMmap>>,
mapping: &Arc<IommuMapping>,
ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
msi_iova_space: (u64, u64),
) -> result::Result<usize, Error> {
let desc = desc_chain
.next()
.ok_or(Error::DescriptorChainTooShort)
.map_err(|e| {
error!("Missing head descriptor");
e
})?;
// The descriptor contains the request type which MUST be readable.
if desc.is_write_only() {
return Err(Error::UnexpectedWriteOnlyDescriptor);
}
if (desc.len() as usize) < size_of::<VirtioIommuReqHead>() {
return Err(Error::InvalidRequest);
}
let req_head: VirtioIommuReqHead = desc_chain
.memory()
.read_obj(desc.addr())
.map_err(Error::GuestMemory)?;
let req_offset = size_of::<VirtioIommuReqHead>();
let desc_size_left = (desc.len() as usize) - req_offset;
let req_addr = if let Some(addr) = desc.addr().checked_add(req_offset as u64) {
addr
} else {
return Err(Error::InvalidRequest);
};
let (msi_iova_start, msi_iova_end) = msi_iova_space;
// Create the reply
let mut reply: Vec<u8> = Vec::new();
let mut status = VIRTIO_IOMMU_S_OK;
let mut hdr_len = 0;
let result = (|| {
match req_head.type_ {
VIRTIO_IOMMU_T_ATTACH => {
if desc_size_left != size_of::<VirtioIommuReqAttach>() {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidAttachRequest);
}
let req: VirtioIommuReqAttach = desc_chain
.memory()
.read_obj(req_addr as GuestAddress)
.map_err(Error::GuestMemory)?;
debug!("Attach request {:?}", req);
// Copy the value to use it as a proper reference.
let domain_id = req.domain;
let endpoint = req.endpoint;
let bypass =
(req.flags & VIRTIO_IOMMU_ATTACH_F_BYPASS) == VIRTIO_IOMMU_ATTACH_F_BYPASS;
// Add endpoint associated with specific domain
mapping
.endpoints
.write()
.unwrap()
.insert(endpoint, domain_id);
// Add new domain with no mapping if the entry didn't exist yet
let mut domains = mapping.domains.write().unwrap();
let domain = Domain {
mappings: BTreeMap::new(),
bypass,
};
domains.entry(domain_id).or_insert_with(|| domain);
}
VIRTIO_IOMMU_T_DETACH => {
if desc_size_left != size_of::<VirtioIommuReqDetach>() {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidDetachRequest);
}
let req: VirtioIommuReqDetach = desc_chain
.memory()
.read_obj(req_addr as GuestAddress)
.map_err(Error::GuestMemory)?;
debug!("Detach request {:?}", req);
// Copy the value to use it as a proper reference.
let domain_id = req.domain;
let endpoint = req.endpoint;
// Remove endpoint associated with specific domain
mapping.endpoints.write().unwrap().remove(&endpoint);
// After all endpoints have been successfully detached from a
// domain, the domain can be removed. This means we must remove
// the mappings associated with this domain.
if mapping
.endpoints
.write()
.unwrap()
.iter()
.filter(|(_, &d)| d == domain_id)
.count()
== 0
{
mapping.domains.write().unwrap().remove(&domain_id);
}
}
VIRTIO_IOMMU_T_MAP => {
if desc_size_left != size_of::<VirtioIommuReqMap>() {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidMapRequest);
}
let req: VirtioIommuReqMap = desc_chain
.memory()
.read_obj(req_addr as GuestAddress)
.map_err(Error::GuestMemory)?;
debug!("Map request {:?}", req);
// Copy the value to use it as a proper reference.
let domain_id = req.domain;
if let Some(domain) = mapping.domains.read().unwrap().get(&domain_id) {
if domain.bypass {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidMapRequestBypassDomain);
}
} else {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidMapRequestMissingDomain);
}
// Find the list of endpoints attached to the given domain.
let endpoints: Vec<u32> = mapping
.endpoints
.write()
.unwrap()
.iter()
.filter(|(_, &d)| d == domain_id)
.map(|(&e, _)| e)
.collect();
// Trigger external mapping if necessary.
for endpoint in endpoints {
if let Some(ext_map) = ext_mapping.get(&endpoint) {
let size = req.virt_end - req.virt_start + 1;
ext_map
.map(req.virt_start, req.phys_start, size)
.map_err(Error::ExternalMapping)?;
}
}
// Add new mapping associated with the domain
mapping
.domains
.write()
.unwrap()
.get_mut(&domain_id)
.unwrap()
.mappings
.insert(
req.virt_start,
Mapping {
gpa: req.phys_start,
size: req.virt_end - req.virt_start + 1,
},
);
}
VIRTIO_IOMMU_T_UNMAP => {
if desc_size_left != size_of::<VirtioIommuReqUnmap>() {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidUnmapRequest);
}
let req: VirtioIommuReqUnmap = desc_chain
.memory()
.read_obj(req_addr as GuestAddress)
.map_err(Error::GuestMemory)?;
debug!("Unmap request {:?}", req);
// Copy the value to use it as a proper reference.
let domain_id = req.domain;
let virt_start = req.virt_start;
if let Some(domain) = mapping.domains.read().unwrap().get(&domain_id) {
if domain.bypass {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidUnmapRequestBypassDomain);
}
} else {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidUnmapRequestMissingDomain);
}
// Find the list of endpoints attached to the given domain.
let endpoints: Vec<u32> = mapping
.endpoints
.write()
.unwrap()
.iter()
.filter(|(_, &d)| d == domain_id)
.map(|(&e, _)| e)
.collect();
// Trigger external unmapping if necessary.
for endpoint in endpoints {
if let Some(ext_map) = ext_mapping.get(&endpoint) {
let size = req.virt_end - virt_start + 1;
ext_map
.unmap(virt_start, size)
.map_err(Error::ExternalUnmapping)?;
}
}
// Remove mapping associated with the domain
mapping
.domains
.write()
.unwrap()
.get_mut(&domain_id)
.unwrap()
.mappings
.remove(&virt_start);
}
VIRTIO_IOMMU_T_PROBE => {
if desc_size_left != size_of::<VirtioIommuReqProbe>() {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidProbeRequest);
}
let req: VirtioIommuReqProbe = desc_chain
.memory()
.read_obj(req_addr as GuestAddress)
.map_err(Error::GuestMemory)?;
debug!("Probe request {:?}", req);
let probe_prop = VirtioIommuProbeProperty {
type_: VIRTIO_IOMMU_PROBE_T_RESV_MEM,
length: size_of::<VirtioIommuProbeResvMem>() as u16,
};
reply.extend_from_slice(probe_prop.as_slice());
let resv_mem = VirtioIommuProbeResvMem {
subtype: VIRTIO_IOMMU_RESV_MEM_T_MSI,
start: msi_iova_start,
end: msi_iova_end,
..Default::default()
};
reply.extend_from_slice(resv_mem.as_slice());
hdr_len = PROBE_PROP_SIZE;
}
_ => {
status = VIRTIO_IOMMU_S_INVAL;
return Err(Error::InvalidRequest);
}
}
Ok(())
})();
let status_desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?;
// The status MUST always be writable
if !status_desc.is_write_only() {
return Err(Error::UnexpectedReadOnlyDescriptor);
}
if status_desc.len() < hdr_len + size_of::<VirtioIommuReqTail>() as u32 {
return Err(Error::BufferLengthTooSmall);
}
let tail = VirtioIommuReqTail {
status,
..Default::default()
};
reply.extend_from_slice(tail.as_slice());
// Make sure we return the result of the request to the guest before
// we return a potential error internally.
desc_chain
.memory()
.write_slice(reply.as_slice(), status_desc.addr())
.map_err(Error::GuestMemory)?;
// Return the error if the result was not Ok().
result?;
Ok((hdr_len as usize) + size_of::<VirtioIommuReqTail>())
}
}
struct IommuEpollHandler {
mem: GuestMemoryAtomic<GuestMemoryMmap>,
request_queue: Queue,
_event_queue: Queue,
interrupt_cb: Arc<dyn VirtioInterrupt>,
request_queue_evt: EventFd,
_event_queue_evt: EventFd,
kill_evt: EventFd,
pause_evt: EventFd,
mapping: Arc<IommuMapping>,
ext_mapping: Arc<Mutex<BTreeMap<u32, Arc<dyn ExternalDmaMapping>>>>,
msi_iova_space: (u64, u64),
}
impl IommuEpollHandler {
fn request_queue(&mut self) -> Result<bool, Error> {
let mut used_descs = false;
while let Some(mut desc_chain) = self.request_queue.pop_descriptor_chain(self.mem.memory())
{
let len = Request::parse(
&mut desc_chain,
&self.mapping,
&self.ext_mapping.lock().unwrap(),
self.msi_iova_space,
)?;
self.request_queue
.add_used(desc_chain.memory(), desc_chain.head_index(), len as u32)
.map_err(Error::QueueAddUsed)?;
used_descs = true;
}
Ok(used_descs)
}
fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> {
self.interrupt_cb
.trigger(VirtioInterruptType::Queue(queue_index))
.map_err(|e| {
error!("Failed to signal used queue: {:?}", e);
DeviceError::FailedSignalingUsedQueue(e)
})
}
fn run(
&mut self,
paused: Arc<AtomicBool>,
paused_sync: Arc<Barrier>,
) -> result::Result<(), EpollHelperError> {
let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?;
helper.add_event(self.request_queue_evt.as_raw_fd(), REQUEST_Q_EVENT)?;
helper.run(paused, paused_sync, self)?;
Ok(())
}
}
impl EpollHelperHandler for IommuEpollHandler {
fn handle_event(
&mut self,
_helper: &mut EpollHelper,
event: &epoll::Event,
) -> result::Result<(), EpollHelperError> {
let ev_type = event.data as u16;
match ev_type {
REQUEST_Q_EVENT => {
self.request_queue_evt.read().map_err(|e| {
EpollHelperError::HandleEvent(anyhow!("Failed to get queue event: {:?}", e))
})?;
let needs_notification = self.request_queue().map_err(|e| {
EpollHelperError::HandleEvent(anyhow!(
"Failed to process request queue : {:?}",
e
))
})?;
if needs_notification {
self.signal_used_queue(0).map_err(|e| {
EpollHelperError::HandleEvent(anyhow!(
"Failed to signal used queue: {:?}",
e
))
})?;
}
}
_ => {
return Err(EpollHelperError::HandleEvent(anyhow!(
"Unexpected event: {}",
ev_type
)));
}
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, Versionize)]
struct Mapping {
gpa: u64,
size: u64,
}
#[derive(Clone, Debug)]
struct Domain {
mappings: BTreeMap<u64, Mapping>,
bypass: bool,
}
#[derive(Debug)]
pub struct IommuMapping {
// Domain related to an endpoint.
endpoints: Arc<RwLock<BTreeMap<u32, u32>>>,
// Information related to each domain.
domains: Arc<RwLock<BTreeMap<u32, Domain>>>,
// Global flag indicating if endpoints that are not attached to any domain
// are in bypass mode.
bypass: AtomicBool,
}
impl DmaRemapping for IommuMapping {
fn translate_gva(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
debug!("Translate GVA addr 0x{:x}", addr);
if let Some(domain_id) = self.endpoints.read().unwrap().get(&id) {
if let Some(domain) = self.domains.read().unwrap().get(domain_id) {
// Directly return identity mapping in case the domain is in
// bypass mode.
if domain.bypass {
return Ok(addr);
}
let range_start = if VIRTIO_IOMMU_PAGE_SIZE_MASK > addr {
0
} else {
addr - VIRTIO_IOMMU_PAGE_SIZE_MASK
};
for (&key, &value) in domain
.mappings
.range((Included(&range_start), Included(&addr)))
{
if addr >= key && addr < key + value.size {
let new_addr = addr - key + value.gpa;
debug!("Into GPA addr 0x{:x}", new_addr);
return Ok(new_addr);
}
}
}
} else if self.bypass.load(Ordering::Acquire) {
return Ok(addr);
}
Err(io::Error::new(
io::ErrorKind::Other,
format!("failed to translate GVA addr 0x{addr:x}"),
))
}
fn translate_gpa(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
debug!("Translate GPA addr 0x{:x}", addr);
if let Some(domain_id) = self.endpoints.read().unwrap().get(&id) {
if let Some(domain) = self.domains.read().unwrap().get(domain_id) {
// Directly return identity mapping in case the domain is in
// bypass mode.
if domain.bypass {
return Ok(addr);
}
for (&key, &value) in domain.mappings.iter() {
if addr >= value.gpa && addr < value.gpa + value.size {
let new_addr = addr - value.gpa + key;
debug!("Into GVA addr 0x{:x}", new_addr);
return Ok(new_addr);
}
}
}
} else if self.bypass.load(Ordering::Acquire) {
return Ok(addr);
}
Err(io::Error::new(
io::ErrorKind::Other,
format!("failed to translate GPA addr 0x{addr:x}"),
))
}
}
#[derive(Debug)]
pub struct AccessPlatformMapping {
id: u32,
mapping: Arc<IommuMapping>,
}
impl AccessPlatformMapping {
pub fn new(id: u32, mapping: Arc<IommuMapping>) -> Self {
AccessPlatformMapping { id, mapping }
}
}
impl AccessPlatform for AccessPlatformMapping {
fn translate_gva(&self, base: u64, _size: u64) -> std::result::Result<u64, std::io::Error> {
self.mapping.translate_gva(self.id, base)
}
fn translate_gpa(&self, base: u64, _size: u64) -> std::result::Result<u64, std::io::Error> {
self.mapping.translate_gpa(self.id, base)
}
}
pub struct Iommu {
common: VirtioCommon,
id: String,
config: VirtioIommuConfig,
mapping: Arc<IommuMapping>,
ext_mapping: Arc<Mutex<BTreeMap<u32, Arc<dyn ExternalDmaMapping>>>>,
seccomp_action: SeccompAction,
exit_evt: EventFd,
msi_iova_space: (u64, u64),
}
type EndpointsState = Vec<(u32, u32)>;
type DomainsState = Vec<(u32, (Vec<(u64, Mapping)>, bool))>;
#[derive(Versionize)]
pub struct IommuState {
avail_features: u64,
acked_features: u64,
endpoints: EndpointsState,
domains: DomainsState,
}
impl VersionMapped for IommuState {}
impl Iommu {
pub fn new(
id: String,
seccomp_action: SeccompAction,
exit_evt: EventFd,
msi_iova_space: (u64, u64),
state: Option<IommuState>,
) -> io::Result<(Self, Arc<IommuMapping>)> {
let (avail_features, acked_features, endpoints, domains, paused) =
if let Some(state) = state {
info!("Restoring virtio-iommu {}", id);
(
state.avail_features,
state.acked_features,
state.endpoints.into_iter().collect(),
state
.domains
.into_iter()
.map(|(k, v)| {
(
k,
Domain {
mappings: v.0.into_iter().collect(),
bypass: v.1,
},
)
})
.collect(),
true,
)
} else {
let avail_features = 1u64 << VIRTIO_F_VERSION_1
| 1u64 << VIRTIO_IOMMU_F_MAP_UNMAP
| 1u64 << VIRTIO_IOMMU_F_PROBE
| 1u64 << VIRTIO_IOMMU_F_BYPASS_CONFIG;
(avail_features, 0, BTreeMap::new(), BTreeMap::new(), false)
};
let config = VirtioIommuConfig {
page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
probe_size: PROBE_PROP_SIZE,
..Default::default()
};
let mapping = Arc::new(IommuMapping {
endpoints: Arc::new(RwLock::new(endpoints)),
domains: Arc::new(RwLock::new(domains)),
bypass: AtomicBool::new(true),
});
Ok((
Iommu {
id,
common: VirtioCommon {
device_type: VirtioDeviceType::Iommu as u32,
queue_sizes: QUEUE_SIZES.to_vec(),
avail_features,
acked_features,
paused_sync: Some(Arc::new(Barrier::new(2))),
min_queues: NUM_QUEUES as u16,
paused: Arc::new(AtomicBool::new(paused)),
..Default::default()
},
config,
mapping: mapping.clone(),
ext_mapping: Arc::new(Mutex::new(BTreeMap::new())),
seccomp_action,
exit_evt,
msi_iova_space,
},
mapping,
))
}
fn state(&self) -> IommuState {
IommuState {
avail_features: self.common.avail_features,
acked_features: self.common.acked_features,
endpoints: self
.mapping
.endpoints
.read()
.unwrap()
.clone()
.into_iter()
.collect(),
domains: self
.mapping
.domains
.read()
.unwrap()
.clone()
.into_iter()
.map(|(k, v)| (k, (v.mappings.into_iter().collect(), v.bypass)))
.collect(),
}
}
fn update_bypass(&mut self) {
// Use bypass from config if VIRTIO_IOMMU_F_BYPASS_CONFIG has been negotiated
if !self
.common
.feature_acked(VIRTIO_IOMMU_F_BYPASS_CONFIG.into())
{
return;
}
let bypass = self.config.bypass == 1;
info!("Updating bypass mode to {}", bypass);
self.mapping.bypass.store(bypass, Ordering::Release);
}
pub fn add_external_mapping(&mut self, device_id: u32, mapping: Arc<dyn ExternalDmaMapping>) {
self.ext_mapping.lock().unwrap().insert(device_id, mapping);
}
#[cfg(fuzzing)]
pub fn wait_for_epoll_threads(&mut self) {
self.common.wait_for_epoll_threads();
}
}
impl Drop for Iommu {
fn drop(&mut self) {
if let Some(kill_evt) = self.common.kill_evt.take() {
// Ignore the result because there is nothing we can do about it.
let _ = kill_evt.write(1);
}
self.common.wait_for_epoll_threads();
}
}
impl VirtioDevice for Iommu {
fn device_type(&self) -> u32 {
self.common.device_type
}
fn queue_max_sizes(&self) -> &[u16] {
&self.common.queue_sizes
}
fn features(&self) -> u64 {
self.common.avail_features
}
fn ack_features(&mut self, value: u64) {
self.common.ack_features(value)
}
fn read_config(&self, offset: u64, data: &mut [u8]) {
self.read_config_from_slice(self.config.as_slice(), offset, data);
}
fn write_config(&mut self, offset: u64, data: &[u8]) {
// The "bypass" field is the only mutable field
let bypass_offset =
(&self.config.bypass as *const _ as u64) - (&self.config as *const _ as u64);
if offset != bypass_offset || data.len() != std::mem::size_of_val(&self.config.bypass) {
error!(
"Attempt to write to read-only field: offset {:x} length {}",
offset,
data.len()
);
return;
}
self.config.bypass = data[0];
self.update_bypass();
}
fn activate(
&mut self,
mem: GuestMemoryAtomic<GuestMemoryMmap>,
interrupt_cb: Arc<dyn VirtioInterrupt>,
mut queues: Vec<(usize, Queue, EventFd)>,
) -> ActivateResult {
self.common.activate(&queues, &interrupt_cb)?;
let (kill_evt, pause_evt) = self.common.dup_eventfds();
let (_, request_queue, request_queue_evt) = queues.remove(0);
let (_, _event_queue, _event_queue_evt) = queues.remove(0);
let mut handler = IommuEpollHandler {
mem,
request_queue,
_event_queue,
interrupt_cb,
request_queue_evt,
_event_queue_evt,
kill_evt,
pause_evt,
mapping: self.mapping.clone(),
ext_mapping: self.ext_mapping.clone(),
msi_iova_space: self.msi_iova_space,
};
let paused = self.common.paused.clone();
let paused_sync = self.common.paused_sync.clone();
let mut epoll_threads = Vec::new();
spawn_virtio_thread(
&self.id,
&self.seccomp_action,
Thread::VirtioIommu,
&mut epoll_threads,
&self.exit_evt,
move || handler.run(paused, paused_sync.unwrap()),
)?;
self.common.epoll_threads = Some(epoll_threads);
event!("virtio-device", "activated", "id", &self.id);
Ok(())
}
fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
let result = self.common.reset();
event!("virtio-device", "reset", "id", &self.id);
result
}
}
impl Pausable for Iommu {
fn pause(&mut self) -> result::Result<(), MigratableError> {
self.common.pause()
}
fn resume(&mut self) -> result::Result<(), MigratableError> {
self.common.resume()
}
}
impl Snapshottable for Iommu {
fn id(&self) -> String {
self.id.clone()
}
fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
Snapshot::new_from_versioned_state(&self.state())
}
}
impl Transportable for Iommu {}
impl Migratable for Iommu {}
Reference in New Issue View Git Blame Copy Permalink