vmm: Unix socket backend for serial port

Cloud-Hypervisor takes a path for Unix socket, where it will listen
on. Users can connect to the other end of the socket and access serial
port on the guest.

    "--serial socket=/path/to/socket" is the cmdline option to pass to
cloud-hypervisor.

Users can use socat like below to access guest's serial port once the
guest starts to boot:

    socat -,crnl UNIX-CONNECT:/path/to/socket

Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com>

devices/src/legacy/serial.rs

@@ -166,8 +166,8 @@ impl Serial {
         Self::new(id, interrupt, None, state)
     }
 
-    pub fn set_out(&mut self, out: Box<dyn io::Write + Send>) {
+    pub fn set_out(&mut self, out: Option<Box<dyn io::Write + Send>>) {
-        self.out = Some(out);
+        self.out = out;
     }
 
     /// Queues raw bytes for the guest to read and signals the interrupt if the line status would

devices/src/legacy/uart_pl011.rs

@@ -201,8 +201,8 @@ impl Pl011 {
         }
     }
 
-    pub fn set_out(&mut self, out: Box<dyn io::Write + Send>) {
+    pub fn set_out(&mut self, out: Option<Box<dyn io::Write + Send>>) {
-        self.out = Some(out);
+        self.out = out;
     }
 
     fn state(&self) -> Pl011State {

src/main.rs

@@ -813,11 +813,13 @@ mod unit_tests {
                 file: None,
                 mode: ConsoleOutputMode::Null,
                 iommu: false,
+                socket: None,
             },
             console: ConsoleConfig {
                 file: None,
                 mode: ConsoleOutputMode::Tty,
                 iommu: false,
+                socket: None,
             },
             devices: None,
             user_devices: None,

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -951,9 +951,11 @@ components:
       properties:
         file:
           type: string
+        socket:
+          type: string
         mode:
           type: string
-          enum: [Off, Pty, Tty, File, Null]
+          enum: [Off, Pty, Tty, File, Socket, Null]
         iommu:
           type: boolean
           default: false

vmm/src/config.rs

@@ -111,6 +111,8 @@ pub enum ValidationError {
     KernelMissing,
     /// Missing file value for console
     ConsoleFileMissing,
+    /// Missing socket path for console
+    ConsoleSocketPathMissing,
     /// Max is less than boot
     CpusMaxLowerThanBoot,
     /// Both socket and path specified
@@ -185,6 +187,7 @@ impl fmt::Display for ValidationError {
             DoubleTtyMode => write!(f, "Console mode tty specified for both serial and console"),
             KernelMissing => write!(f, "No kernel specified"),
             ConsoleFileMissing => write!(f, "Path missing when using file console mode"),
+            ConsoleSocketPathMissing => write!(f, "Path missing when using socket console mode"),
             CpusMaxLowerThanBoot => write!(f, "Max CPUs lower than boot CPUs"),
             DiskSocketAndPath => write!(f, "Disk path and vhost socket both provided"),
             VhostUserRequiresSharedMemory => {
@@ -1341,10 +1344,12 @@ impl ConsoleConfig {
             .add_valueless("tty")
             .add_valueless("null")
             .add("file")
-            .add("iommu");
+            .add("iommu")
+            .add("socket");
         parser.parse(console).map_err(Error::ParseConsole)?;
 
         let mut file: Option<PathBuf> = default_consoleconfig_file();
+        let mut socket: Option<PathBuf> = None;
         let mut mode: ConsoleOutputMode = ConsoleOutputMode::Off;
 
         if parser.is_set("off") {
@@ -1360,6 +1365,11 @@ impl ConsoleConfig {
                 Some(PathBuf::from(parser.get("file").ok_or(
                     Error::Validation(ValidationError::ConsoleFileMissing),
                 )?));
+        } else if parser.is_set("socket") {
+            mode = ConsoleOutputMode::Socket;
+            socket = Some(PathBuf::from(parser.get("socket").ok_or(
+                Error::Validation(ValidationError::ConsoleSocketPathMissing),
+            )?));
         } else {
             return Err(Error::ParseConsoleInvalidModeGiven);
         }
@@ -1369,7 +1379,12 @@ impl ConsoleConfig {
             .unwrap_or(Toggle(false))
             .0;
 
-        Ok(Self { file, mode, iommu })
+        Ok(Self {
+            file,
+            mode,
+            iommu,
+            socket,
+        })
     }
 }
 
@@ -2659,6 +2674,7 @@ mod tests {
                 mode: ConsoleOutputMode::Off,
                 iommu: false,
                 file: None,
+                socket: None,
             }
         );
         assert_eq!(
@@ -2667,6 +2683,7 @@ mod tests {
                 mode: ConsoleOutputMode::Pty,
                 iommu: false,
                 file: None,
+                socket: None,
             }
         );
         assert_eq!(
@@ -2675,6 +2692,7 @@ mod tests {
                 mode: ConsoleOutputMode::Tty,
                 iommu: false,
                 file: None,
+                socket: None,
             }
         );
         assert_eq!(
@@ -2683,6 +2701,7 @@ mod tests {
                 mode: ConsoleOutputMode::Null,
                 iommu: false,
                 file: None,
+                socket: None,
             }
         );
         assert_eq!(
@@ -2690,7 +2709,8 @@ mod tests {
             ConsoleConfig {
                 mode: ConsoleOutputMode::File,
                 iommu: false,
-                file: Some(PathBuf::from("/tmp/console"))
+                file: Some(PathBuf::from("/tmp/console")),
+                socket: None,
             }
         );
         assert_eq!(
@@ -2699,6 +2719,7 @@ mod tests {
                 mode: ConsoleOutputMode::Null,
                 iommu: true,
                 file: None,
+                socket: None,
             }
         );
         assert_eq!(
@@ -2706,7 +2727,17 @@ mod tests {
             ConsoleConfig {
                 mode: ConsoleOutputMode::File,
                 iommu: true,
-                file: Some(PathBuf::from("/tmp/console"))
+                file: Some(PathBuf::from("/tmp/console")),
+                socket: None,
+            }
+        );
+        assert_eq!(
+            ConsoleConfig::parse("socket=/tmp/serial.sock,iommu=on")?,
+            ConsoleConfig {
+                mode: ConsoleOutputMode::Socket,
+                iommu: true,
+                file: None,
+                socket: Some(PathBuf::from("/tmp/serial.sock")),
             }
         );
         Ok(())
@@ -2852,11 +2883,13 @@ mod tests {
                 file: None,
                 mode: ConsoleOutputMode::Null,
                 iommu: false,
+                socket: None,
             },
             console: ConsoleConfig {
                 file: None,
                 mode: ConsoleOutputMode::Tty,
                 iommu: false,
+                socket: None,
             },
             devices: None,
             user_devices: None,

vmm/src/device_manager.rs

@@ -395,6 +395,9 @@ pub enum DeviceManagerError {
     /// No support for device passthrough
     NoDevicePassthroughSupport,
 
+    /// No socket option support for console device
+    NoSocketOptionSupportForConsoleDevice,
+
     /// Failed to resize virtio-balloon
     VirtioBalloonResize(virtio_devices::balloon::Error),
 
@@ -1995,6 +1998,9 @@ impl DeviceManager {
                     Endpoint::File(stdout)
                 }
             }
+            ConsoleOutputMode::Socket => {
+                return Err(DeviceManagerError::NoSocketOptionSupportForConsoleDevice);
+            }
             ConsoleOutputMode::Null => Endpoint::Null,
             ConsoleOutputMode::Off => return Ok(None),
         };
@@ -2074,15 +2080,19 @@ impl DeviceManager {
                 let _ = self.set_raw_mode(&out);
                 Some(Box::new(out))
             }
-            ConsoleOutputMode::Off | ConsoleOutputMode::Null => None,
+            ConsoleOutputMode::Off | ConsoleOutputMode::Null | ConsoleOutputMode::Socket => None,
         };
         if serial_config.mode != ConsoleOutputMode::Off {
             let serial = self.add_serial_device(interrupt_manager, serial_writer)?;
             self.serial_manager = match serial_config.mode {
-                ConsoleOutputMode::Pty | ConsoleOutputMode::Tty => {
+                ConsoleOutputMode::Pty | ConsoleOutputMode::Tty | ConsoleOutputMode::Socket => {
-                    let serial_manager =
+                    let serial_manager = SerialManager::new(
-                        SerialManager::new(serial, self.serial_pty.clone(), serial_config.mode)
+                        serial,
-                            .map_err(DeviceManagerError::CreateSerialManager)?;
+                        self.serial_pty.clone(),
+                        serial_config.mode,
+                        serial_config.socket,
+                    )
+                    .map_err(DeviceManagerError::CreateSerialManager)?;
                     if let Some(mut serial_manager) = serial_manager {
                         serial_manager
                             .start_thread(

vmm/src/lib.rs

@@ -2274,11 +2274,13 @@ mod unit_tests {
                 file: None,
                 mode: ConsoleOutputMode::Null,
                 iommu: false,
+                socket: None,
             },
             console: ConsoleConfig {
                 file: None,
                 mode: ConsoleOutputMode::Tty,
                 iommu: false,
+                socket: None,
             },
             devices: None,
             user_devices: None,

vmm/src/serial_manager.rs

@@ -13,8 +13,11 @@ use libc::EFD_NONBLOCK;
 use serial_buffer::SerialBuffer;
 use std::fs::File;
 use std::io::Read;
-use std::os::unix::io::{AsRawFd, FromRawFd};
+use std::net::Shutdown;
+use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
+use std::os::unix::net::{UnixListener, UnixStream};
 use std::panic::AssertUnwindSafe;
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::{io, result, thread};
@@ -54,6 +57,22 @@ pub enum Error {
     /// Cannot spawn SerialManager thread.
     #[error("Error spawning SerialManager thread: {0}")]
     SpawnSerialManager(#[source] io::Error),
+
+    /// Cannot bind to Unix socket
+    #[error("Error binding to socket: {0}")]
+    BindUnixSocket(#[source] io::Error),
+
+    /// Cannot accept connection from Unix socket
+    #[error("Error accepting connection: {0}")]
+    AcceptConnection(#[source] io::Error),
+
+    /// Cannot clone the UnixStream
+    #[error("Error cloning UnixStream: {0}")]
+    CloneUnixStream(#[source] io::Error),
+
+    /// Cannot shutdown the connection
+    #[error("Error shutting down a connection: {0}")]
+    ShutdownConnection(#[source] io::Error),
 }
 pub type Result<T> = result::Result<T, Error>;
 
@@ -62,6 +81,7 @@ pub type Result<T> = result::Result<T, Error>;
 pub enum EpollDispatch {
     File = 0,
     Kill = 1,
+    Socket = 2,
     Unknown,
 }
 
@@ -71,6 +91,7 @@ impl From<u64> for EpollDispatch {
         match v {
             0 => File,
             1 => Kill,
+            2 => Socket,
             _ => Unknown,
         }
     }
@@ -86,6 +107,7 @@ pub struct SerialManager {
     kill_evt: EventFd,
     handle: Option<thread::JoinHandle<()>>,
     pty_write_out: Option<Arc<AtomicBool>>,
+    mode: ConsoleOutputMode,
 }
 
 impl SerialManager {
@@ -94,6 +116,7 @@ impl SerialManager {
         #[cfg(target_arch = "aarch64")] serial: Arc<Mutex<Pl011>>,
         pty_pair: Option<Arc<Mutex<PtyPair>>>,
         mode: ConsoleOutputMode,
+        socket: Option<PathBuf>,
     ) -> Result<Option<Self>> {
         let in_file = match mode {
             ConsoleOutputMode::Pty => {
@@ -130,6 +153,16 @@ impl SerialManager {
                     return Ok(None);
                 }
             }
+            ConsoleOutputMode::Socket => {
+                if let Some(socket_path) = socket {
+                    let listener =
+                        UnixListener::bind(socket_path.as_path()).map_err(Error::BindUnixSocket)?;
+                    // SAFETY: listener is valid and will return valid fd
+                    unsafe { File::from_raw_fd(listener.into_raw_fd()) }
+                } else {
+                    return Ok(None);
+                }
+            }
             _ => return Ok(None),
         };
 
@@ -144,11 +177,17 @@ impl SerialManager {
         )
         .map_err(Error::Epoll)?;
 
+        let epoll_fd_data = if mode == ConsoleOutputMode::Socket {
+            EpollDispatch::Socket
+        } else {
+            EpollDispatch::File
+        };
+
         epoll::ctl(
             epoll_fd,
             epoll::ControlOptions::EPOLL_CTL_ADD,
             in_file.as_raw_fd(),
-            epoll::Event::new(epoll::Events::EPOLLIN, EpollDispatch::File as u64),
+            epoll::Event::new(epoll::Events::EPOLLIN, epoll_fd_data as u64),
         )
         .map_err(Error::Epoll)?;
 
@@ -158,7 +197,11 @@ impl SerialManager {
             pty_write_out = Some(write_out.clone());
             let writer = in_file.try_clone().map_err(Error::FileClone)?;
             let buffer = SerialBuffer::new(Box::new(writer), write_out);
-            serial.as_ref().lock().unwrap().set_out(Box::new(buffer));
+            serial
+                .as_ref()
+                .lock()
+                .unwrap()
+                .set_out(Some(Box::new(buffer)));
         }
 
         // Use 'File' to enforce closing on 'epoll_fd'
@@ -172,6 +215,7 @@ impl SerialManager {
             kill_evt,
             handle: None,
             pty_write_out,
+            mode,
         }))
     }
 
@@ -212,6 +256,10 @@ impl SerialManager {
         let mut in_file = self.in_file.try_clone().map_err(Error::FileClone)?;
         let serial = self.serial.clone();
         let pty_write_out = self.pty_write_out.clone();
+        //SAFETY: in_file is has a valid fd
+        let listener = unsafe { UnixListener::from_raw_fd(self.in_file.as_raw_fd()) };
+        let mut reader: Option<UnixStream> = None;
+        let mode = self.mode.clone();
 
         // In case of PTY, we want to be able to detect a connection on the
         // other end of the PTY. This is done by detecting there's no event
@@ -250,7 +298,7 @@ impl SerialManager {
                             }
                         };
 
-                        if num_events == 0 {
+                        if mode != ConsoleOutputMode::Socket && num_events == 0 {
                             // This very specific case happens when the serial is connected
                             // to a PTY. We know EPOLLHUP is always present when there's nothing
                             // connected at the other end of the PTY. That's why getting no event
@@ -266,11 +314,67 @@ impl SerialManager {
                                     let event = event.data;
                                     warn!("Unknown serial manager loop event: {}", event);
                                 }
+                                EpollDispatch::Socket => {
+                                    // New connection request arrived.
+                                    // Shutdown the previous connection, if any
+                                    if let Some(previous_reader) = reader {
+                                        previous_reader
+                                            .shutdown(Shutdown::Both)
+                                            .map_err(Error::AcceptConnection)?;
+                                    }
+                                    // Events on the listening socket will be connection requests.
+                                    // Accept them, create a reader and a writer.
+                                    let (unix_stream, _) =
+                                        listener.accept().map_err(Error::AcceptConnection)?;
+                                    let writer =
+                                        unix_stream.try_clone().map_err(Error::CloneUnixStream)?;
+                                    reader = Some(
+                                        unix_stream.try_clone().map_err(Error::CloneUnixStream)?,
+                                    );
+
+                                    epoll::ctl(
+                                        epoll_fd,
+                                        epoll::ControlOptions::EPOLL_CTL_ADD,
+                                        unix_stream.into_raw_fd(),
+                                        epoll::Event::new(
+                                            epoll::Events::EPOLLIN,
+                                            EpollDispatch::File as u64,
+                                        ),
+                                    )
+                                    .map_err(Error::Epoll)?;
+                                    serial.lock().unwrap().set_out(Some(Box::new(writer)));
+                                }
                                 EpollDispatch::File => {
                                     if event.events & libc::EPOLLIN as u32 != 0 {
                                         let mut input = [0u8; 64];
-                                        let count =
+                                        let count = match mode {
-                                            in_file.read(&mut input).map_err(Error::ReadInput)?;
+                                            ConsoleOutputMode::Socket => {
+                                                if let Some(mut serial_reader) = reader.as_ref() {
+                                                    let count = serial_reader
+                                                        .read(&mut input)
+                                                        .map_err(Error::ReadInput)?;
+                                                    if count == 0 {
+                                                        info!("Remote end closed serial socket");
+                                                        serial_reader
+                                                            .shutdown(Shutdown::Both)
+                                                            .map_err(Error::ShutdownConnection)?;
+
+                                                        reader = None;
+                                                        serial
+                                                            .as_ref()
+                                                            .lock()
+                                                            .unwrap()
+                                                            .set_out(None);
+                                                    }
+                                                    count
+                                                } else {
+                                                    0
+                                                }
+                                            }
+                                            _ => in_file
+                                                .read(&mut input)
+                                                .map_err(Error::ReadInput)?,
+                                        };
 
                                         // Replace "\n" with "\r" to deal with Windows SAC (#1170)
                                         if count == 1 && input[0] == 0x0a {

vmm/src/vm_config.rs

@@ -440,6 +440,7 @@ pub enum ConsoleOutputMode {
     Pty,
     Tty,
     File,
+    Socket,
     Null,
 }
 
@@ -450,6 +451,7 @@ pub struct ConsoleConfig {
     pub mode: ConsoleOutputMode,
     #[serde(default)]
     pub iommu: bool,
+    pub socket: Option<PathBuf>,
 }
 
 pub fn default_consoleconfig_file() -> Option<PathBuf> {
@@ -555,6 +557,7 @@ pub fn default_serial() -> ConsoleConfig {
         file: None,
         mode: ConsoleOutputMode::Null,
         iommu: false,
+        socket: None,
     }
 }
 
@@ -563,6 +566,7 @@ pub fn default_console() -> ConsoleConfig {
         file: None,
         mode: ConsoleOutputMode::Tty,
         iommu: false,
+        socket: None,
     }
 }