display/rdp: implement Unix.Map

Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>

qemu-display/Cargo.toml

@@ -36,6 +36,7 @@ windows = { version = "0.58", features = [
     "Win32_Networking_WinSock",
     "Win32_Foundation",
     "Win32_System_IO",
+    "Win32_System_Memory",
     "Win32_System_Threading",
 ] }
 

qemu-display/src/console.rs

@@ -119,19 +119,24 @@ impl Console {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, handler))]
     pub async fn set_map_listener<H: ConsoleListenerMapHandler>(&self, handler: H) -> Result<bool> {
-        if let Some(l) = &*self.listener.write().unwrap() {
+        let listener = self.listener.write().unwrap().take();
-            return l
+        match listener {
-                .object_server()
+            Some(l) => {
-                .at(
+                let res = l
-                    "/org/qemu/Display1/Listener",
+                    .object_server()
-                    ConsoleListenerMap::new(handler),
+                    .at(
-                )
+                        "/org/qemu/Display1/Listener",
-                .await
+                        ConsoleListenerMap::new(handler),
-                .map_err(|e| e.into());
+                    )
+                    .await
+                    .map_err(|e| e.into());
+                *self.listener.write().unwrap() = Some(l);
+                res
+            }
+            _ => Err(crate::Error::Failed("Must call register first!".into())),
         }
-
-        Err(crate::Error::Failed("Must call register first!".into()))
     }
 
     #[cfg(windows)]

qemu-display/src/console_listener.rs

@@ -1,11 +1,13 @@
 #[cfg(windows)]
-use crate::win32::Fd;
+use crate::win32::{Fd, Mmap};
 use derivative::Derivative;
 #[cfg(unix)]
-use std::os::unix::io::{AsRawFd, IntoRawFd, RawFd};
+use memmap2::Mmap;
-use std::{
+use std::ops::Drop;
-    ops::Drop,
+#[cfg(unix)]
-    os::fd::{AsFd, OwnedFd},
+use std::os::{
+    fd::{AsFd, OwnedFd},
+    unix::io::{AsRawFd, IntoRawFd, RawFd},
 };
 #[cfg(unix)]
 use zbus::zvariant::Fd;
@@ -47,6 +49,58 @@ pub struct ScanoutMap {
     pub format: u32,
 }
 
+#[derive(Debug)]
+pub struct ScanoutMmap {
+    scanout: ScanoutMap,
+    #[cfg(unix)]
+    mmap: Mmap,
+    #[cfg(windows)]
+    mmap: Mmap,
+}
+
+impl ScanoutMap {
+    pub fn mmap(self) -> std::io::Result<ScanoutMmap> {
+        let len = self.height as usize * self.stride as usize;
+        let offset = self.offset;
+
+        #[cfg(unix)]
+        let mmap = {
+            let fd = self.fd.as_raw_fd();
+            unsafe {
+                memmap2::MmapOptions::new()
+                    .len(len)
+                    .offset(offset.into())
+                    .map(fd)
+            }?
+        };
+
+        #[cfg(windows)]
+        let mmap = {
+            let handle = windows::Win32::Foundation::HANDLE(self.handle as _); // taking ownership
+            Mmap::new(handle, offset.try_into().unwrap(), len)?
+        };
+
+        Ok(ScanoutMmap {
+            scanout: self,
+            mmap,
+        })
+    }
+}
+
+impl ScanoutMmap {
+    pub fn as_ref(&self) -> &[u8] {
+        self.mmap.as_ref()
+    }
+
+    pub fn stride(&self) -> u32 {
+        self.scanout.stride
+    }
+
+    pub fn format(&self) -> u32 {
+        self.scanout.format
+    }
+}
+
 #[derive(Debug, Copy, Clone)]
 pub struct UpdateMap {
     pub x: i32,

qemu-display/src/util.rs

@@ -1,7 +1,5 @@
 use crate::Result;
 
-use std::os::fd::RawFd;
-
 #[cfg(unix)]
 use std::os::unix::net::UnixStream;
 #[cfg(windows)]
@@ -32,8 +30,3 @@ pub fn prepare_uds_pass(#[cfg(windows)] peer_pid: u32, us: &UnixStream) -> Resul
         p.duplicate_socket(SOCKET(us.as_raw_socket() as _))
     }
 }
-
-#[cfg(unix)]
-pub unsafe fn mmap(fd: RawFd, len: usize, offset: u64) -> std::io::Result<memmap2::Mmap> {
-    memmap2::MmapOptions::new().len(len).offset(offset).map(fd)
-}

qemu-display/src/win32.rs

@@ -1,8 +1,13 @@
 use std::io;
+use tracing::warn;
+
 use windows::Win32::{
     Foundation::{CloseHandle, HANDLE},
     Networking::WinSock::{WSADuplicateSocketW, SOCKET, WSAPROTOCOL_INFOW},
-    System::Threading::PROCESS_ACCESS_RIGHTS,
+    System::{
+        Memory::{MapViewOfFile, UnmapViewOfFile, FILE_MAP_READ, MEMORY_MAPPED_VIEW_ADDRESS},
+        Threading::PROCESS_ACCESS_RIGHTS,
+    },
 };
 
 #[cfg(feature = "qmp")]
@@ -117,3 +122,48 @@ pub(crate) fn unix_stream_get_peer_pid(stream: &UnixStream) -> Result<u32, std::
 
     Ok(ret)
 }
+
+#[derive(Debug)]
+pub(crate) struct Mmap {
+    handle: HANDLE,
+    ptr: MEMORY_MAPPED_VIEW_ADDRESS,
+    offset: isize,
+    size: usize,
+}
+
+#[cfg(windows)]
+impl Drop for Mmap {
+    fn drop(&mut self) {
+        unsafe {
+            if let Err(err) = UnmapViewOfFile(self.ptr) {
+                warn!("error while unmap: {}", err);
+            }
+            if let Err(err) = CloseHandle(self.handle) {
+                warn!("error while closing mmap: {}", err);
+            }
+        }
+    }
+}
+
+#[cfg(windows)]
+impl Mmap {
+    // FIXME: remove and replace with memmap2, use offset properly
+    pub(crate) fn new(handle: HANDLE, offset: usize, size: usize) -> std::io::Result<Self> {
+        let ptr = unsafe { MapViewOfFile(handle, FILE_MAP_READ, 0, 0, offset + size) };
+        if ptr.Value.is_null() {
+            return Err(io::Error::last_os_error());
+        }
+        Ok(Self {
+            handle,
+            ptr,
+            offset: offset.try_into().unwrap(),
+            size,
+        })
+    }
+
+    pub(crate) fn as_ref(&self) -> &[u8] {
+        unsafe {
+            std::slice::from_raw_parts(self.ptr.Value.cast::<u8>().offset(self.offset), self.size)
+        }
+    }
+}

qemu-rdp/src/server/display.rs

@@ -4,7 +4,10 @@ use std::{
 };
 
 use anyhow::Result;
-use qemu_display::{Console, ConsoleListenerHandler, Cursor, Display, MouseSet, Scanout, Update};
+use qemu_display::{
+    Console, ConsoleListenerHandler, ConsoleListenerMapHandler, Cursor, Display, MouseSet, Scanout,
+    ScanoutMap, ScanoutMmap, Update, UpdateMap,
+};
 
 use ironrdp::{
     connector::DesktopSize,
@@ -14,7 +17,7 @@ use ironrdp::{
         RdpServerDisplayUpdates,
     },
 };
-use tracing::debug;
+use tracing::{debug, warn};
 
 use crate::{cast, utils::PixmanFormat};
 
@@ -45,7 +48,9 @@ impl DisplayHandler {
         let desktop_size = DesktopSize { width, height };
         let listener = Listener::new(sender, desktop_size);
         self.console.unregister_listener();
-        self.console.register_listener(listener).await?;
+        self.console.register_listener(listener.clone()).await?;
+        #[cfg(any(windows, unix))]
+        self.console.set_map_listener(listener.clone()).await?;
 
         Ok(DisplayUpdates { receiver })
     }
@@ -94,6 +99,7 @@ struct Inner {
     sender: queue::Sender,
     desktop_size: DesktopSize,
     cursor_hot: (i32, i32),
+    scanout_mmap: Option<ScanoutMmap>,
 }
 
 #[derive(Clone)]
@@ -107,6 +113,7 @@ impl Listener {
             sender,
             desktop_size,
             cursor_hot: (0, 0),
+            scanout_mmap: None,
         };
         Self {
             inner: Arc::new(Mutex::new(inner)),
@@ -143,6 +150,45 @@ impl Listener {
     }
 }
 
+#[async_trait::async_trait]
+impl ConsoleListenerMapHandler for Listener {
+    async fn scanout_map(&mut self, scanout: ScanoutMap) {
+        self.set_desktop_size(scanout.width, scanout.height).await;
+        match scanout.mmap() {
+            Ok(mmap) => self.inner.lock().unwrap().scanout_mmap = Some(mmap),
+            Err(err) => warn!("Failed to mmap: {}", err),
+        }
+    }
+
+    async fn update_map(&mut self, update: UpdateMap) {
+        let update = {
+            let inner = self.inner.lock().unwrap();
+            let Some(mmap) = &inner.scanout_mmap else {
+                warn!("Update with no map!");
+                return;
+            };
+            let (stride, format) = (mmap.stride(), mmap.format());
+            if format != 0x20020888 {
+                warn!("Format not yet supported: {:X}", format);
+                return;
+            }
+            // FIXME: use arc of data
+            let data = mmap.as_ref()[update.y as usize * stride as usize + update.x as usize * 4..]
+                .to_vec();
+            Update {
+                x: update.x,
+                y: update.y,
+                w: update.w,
+                h: update.h,
+                stride,
+                format,
+                data,
+            }
+        };
+        self.update(update).await
+    }
+}
+
 #[async_trait::async_trait]
 impl ConsoleListenerHandler for Listener {
     async fn scanout(&mut self, scanout: Scanout) {
@@ -256,6 +302,10 @@ impl ConsoleListenerHandler for Listener {
     }
 
     fn interfaces(&self) -> Vec<String> {
-        vec![]
+        if cfg!(unix) {
+            vec!["org.qemu.Display1.Listener.Unix.Map".to_string()]
+        } else {
+            vec![]
+        }
     }
 }

qemu-rdw/src/display.rs

@@ -15,12 +15,11 @@ use std::os::unix::io::IntoRawFd;
 mod imp {
     use super::*;
     use gtk::subclass::prelude::*;
-    use qemu_display::{memmap2::Mmap, util::mmap};
+    use qemu_display::ScanoutMmap;
     #[cfg(any(windows, unix))]
     use std::cell::RefCell;
     #[cfg(windows)]
     use std::ffi::c_void;
-    use std::os::fd::AsRawFd;
     #[cfg(windows)]
     use windows::Win32::Foundation::{CloseHandle, HANDLE};
 
@@ -84,10 +83,7 @@ mod imp {
     pub struct Display {
         pub(crate) console: OnceCell<Console>,
         keymap: Cell<Option<&'static [u16]>>,
-        #[cfg(unix)]
+        scanout_map: RefCell<Option<ScanoutMmap>>,
-        scanout_map: RefCell<Option<(Mmap, u32)>>,
-        #[cfg(windows)]
-        scanout_map: RefCell<Option<(MemoryMap, u32)>>,
     }
 
     #[glib::object_subclass]
@@ -333,39 +329,37 @@ mod imp {
                                             );
                                             continue;
                                         }
-                                        let map = unsafe {
-                                            mmap(
-                                                scanout.fd.as_raw_fd(),
-                                                scanout.height as usize * scanout.stride as usize,
-                                                scanout.offset.into(),
-                                            )
-                                        }
-                                        .unwrap();
                                         this.obj().set_display_size(Some((
                                             scanout.width as _,
                                             scanout.height as _,
                                         )));
-                                        this.scanout_map.replace(Some((map, scanout.stride)));
+                                        let map = match scanout.mmap() {
+                                            Ok(map) => Some(map),
+                                            Err(err) => {
+                                                log::warn!("Failed to mmap: {}", err);
+                                                continue;
+                                            }
+                                        };
+                                        this.scanout_map.replace(map);
                                         let _ = wait_tx.send(());
                                     }
                                     #[cfg(unix)]
                                     UpdateMap(u) => {
                                         log::debug!("{u:?}");
                                         let scanout_map = this.scanout_map.borrow();
-                                        let Some((map, stride)) = scanout_map.as_ref() else {
+                                        let Some(map) = scanout_map.as_ref() else {
                                             log::warn!("No mapped scanout!");
                                             continue;
                                         };
-                                        let stride = *stride;
                                         let bytes = map.as_ref();
                                         this.obj().update_area(
                                             u.x as _,
                                             u.y as _,
                                             u.w as _,
                                             u.h as _,
-                                            stride as _,
+                                            map.stride() as _,
                                             Some(
-                                                &bytes[u.y as usize * stride as usize
+                                                &bytes[u.y as usize * map.stride() as usize
                                                     + u.x as usize * 4..],
                                             ),
                                         );