net_util, virtio-devices, vmm: Accept multiple TAP fds

This patch enables multi-queue support for creating virtio-net devices by
accepting multiple TAP fds, e.g. '--net fds=3:7'.

Fixes: #2164

Signed-off-by: Bo Chen <chen.bo@intel.com>

net_util/src/tap.rs

@@ -153,7 +153,7 @@ impl Tap {
         Self::open_named("vmtap%d", num_queue_pairs, None)
     }
 
-    pub fn from_tap_fd(fd: RawFd) -> Result<Tap> {
+    pub fn from_tap_fd(fd: RawFd, num_queue_pairs: usize) -> Result<Tap> {
         // Ensure that the file is opened non-blocking, this is particularly
         // needed when opened via the shell for macvtap.
         let ret = unsafe {
@@ -181,6 +181,9 @@ impl Tap {
             let ifru_flags = ifreq.ifr_ifru.ifru_flags.as_mut();
             *ifru_flags =
                 (net_gen::IFF_TAP | net_gen::IFF_NO_PI | net_gen::IFF_VNET_HDR) as c_short;
+            if num_queue_pairs > 1 {
+                *ifru_flags |= net_gen::IFF_MULTI_QUEUE as c_short;
+            }
         }
         let ret = unsafe { ioctl_with_mut_ref(&tap_file, net_gen::TUNSETIFF(), &mut ifreq) };
         if ret < 0 && IoError::last_os_error().raw_os_error().unwrap() != libc::EEXIST {
@@ -591,7 +594,7 @@ mod tests {
 
         let orig_tap = Tap::new(1).unwrap();
         let fd = orig_tap.as_raw_fd();
-        let _new_tap = Tap::from_tap_fd(fd).unwrap();
+        let _new_tap = Tap::from_tap_fd(fd, 1).unwrap();
     }
 
     #[test]

virtio-devices/src/net.rs

@@ -297,21 +297,28 @@ impl Net {
         )
     }
 
-    pub fn from_tap_fd(
+    pub fn from_tap_fds(
         id: String,
-        fd: RawFd,
+        fds: &[RawFd],
         guest_mac: Option<MacAddr>,
         iommu: bool,
         queue_size: u16,
         seccomp_action: SeccompAction,
     ) -> Result<Self> {
-        let tap = Tap::from_tap_fd(fd).map_err(Error::TapError)?;
+        let mut taps: Vec<Tap> = Vec::new();
+        let num_queue_pairs = fds.len();
+
+        for fd in fds.iter() {
+            let tap = Tap::from_tap_fd(*fd, num_queue_pairs).map_err(Error::TapError)?;
+            taps.push(tap);
+        }
+
         Self::new_with_tap(
             id,
-            vec![tap],
+            taps,
             guest_mac,
             iommu,
-            2,
+            num_queue_pairs * 2,
             queue_size,
             seccomp_action,
         )

vmm/src/config.rs

@@ -107,6 +107,8 @@ pub enum ValidationError {
     CpuTopologyZeroPart,
     /// Virtio needs a min of 2 queues
     VnetQueueLowerThan2,
+    /// The input queue number for virtio_net must match the number of input fds
+    VnetQueueFdMismatch,
 }
 
 type ValidationResult<T> = std::result::Result<T, ValidationError>;
@@ -132,6 +134,10 @@ impl fmt::Display for ValidationError {
                 "Product of CPU topology parts does not match maximum vCPUs"
             ),
             VnetQueueLowerThan2 => write!(f, "Number of queues to virtio_net less than 2"),
+            VnetQueueFdMismatch => write!(
+                f,
+                "Number of queues to virtio_net does not match the number of input FDs"
+            ),
         }
     }
 }
@@ -765,7 +771,7 @@ pub struct NetConfig {
     #[serde(default)]
     pub id: Option<String>,
     #[serde(default)]
-    pub fd: Option<i32>,
+    pub fds: Option<Vec<i32>>,
 }
 
 fn default_netconfig_tap() -> Option<String> {
@@ -806,14 +812,14 @@ impl Default for NetConfig {
             vhost_user: false,
             vhost_socket: None,
             id: None,
-            fd: None,
+            fds: None,
         }
     }
 }
 
 impl NetConfig {
     pub const SYNTAX: &'static str = "Network parameters \
-    \"tap=<if_name>,ip=<ip_addr>,mask=<net_mask>,mac=<mac_addr>,fd=<fd>,iommu=on|off,\
+    \"tap=<if_name>,ip=<ip_addr>,mask=<net_mask>,mac=<mac_addr>,fd=<fd1:fd2...>,iommu=on|off,\
     num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,\
     vhost_user=<vhost_user_enable>,socket=<vhost_user_socket_path>,id=<device_id>\"";
 
@@ -869,7 +875,11 @@ impl NetConfig {
             .0;
         let vhost_socket = parser.get("socket");
         let id = parser.get("id");
-        let fd = parser.convert("fd").map_err(Error::ParseNetwork)?;
+        let fds = parser
+            .convert::<IntegerList>("fd")
+            .map_err(Error::ParseNetwork)?
+            .map(|v| v.0.iter().map(|e| *e as i32).collect());
+
         let config = NetConfig {
             tap,
             ip,
@@ -882,7 +892,7 @@ impl NetConfig {
             vhost_user,
             vhost_socket,
             id,
-            fd,
+            fds,
         };
         config.validate().map_err(Error::Validation)?;
         Ok(config)
@@ -891,6 +901,11 @@ impl NetConfig {
         if self.num_queues < 2 {
             return Err(ValidationError::VnetQueueLowerThan2);
         }
+
+        if self.fds.is_some() && self.fds.as_ref().unwrap().len() * 2 != self.num_queues {
+            return Err(ValidationError::VnetQueueFdMismatch);
+        }
+
         Ok(())
     }
 }
@@ -1949,10 +1964,11 @@ mod tests {
         );
 
         assert_eq!(
-            NetConfig::parse("mac=de:ad:be:ef:12:34,fd=3")?,
+            NetConfig::parse("mac=de:ad:be:ef:12:34,fd=3:7,num_queues=4")?,
             NetConfig {
                 mac: MacAddr::parse_str("de:ad:be:ef:12:34").unwrap(),
-                fd: Some(3),
+                fds: Some(vec![3, 7]),
+                num_queues: 4,
                 ..Default::default()
             }
         );

vmm/src/device_manager.rs

@@ -1790,11 +1790,11 @@ impl DeviceManager {
                     )
                     .map_err(DeviceManagerError::CreateVirtioNet)?,
                 ))
-            } else if let Some(fd) = net_cfg.fd {
+            } else if let Some(fds) = &net_cfg.fds {
                 Arc::new(Mutex::new(
-                    virtio_devices::Net::from_tap_fd(
+                    virtio_devices::Net::from_tap_fds(
                         id.clone(),
-                        fd,
+                        fds,
                         Some(net_cfg.mac),
                         net_cfg.iommu,
                         net_cfg.queue_size,