More progress with inputs

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = ["qemu-display-listener", "qemu-gtk4"]
 
-#[patch.crates-io]
-#zbus = { path = '/home/elmarco/src/zbus/zbus' }
-
+[patch.crates-io]
+zbus = { path = '/home/elmarco/src/zbus/zbus' }
+zvariant = { path = '/home/elmarco/src/zbus/zvariant' }

qemu-display-listener/Cargo.toml

@@ -12,3 +12,6 @@ derivative = "2.1.3"
 zvariant = "2.4.0"
 libc = "0.2.86"
 glib = { git = "https://github.com/gtk-rs/gtk-rs", optional = true }
+enumflags2 = { version = "0.6.4", features = ["serde"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_repr = "0.1.6"

qemu-display-listener/src/console.rs

@@ -7,7 +7,7 @@ use std::{os::unix::io::AsRawFd, thread};
 use zbus::{dbus_proxy, export::zvariant::Fd};
 
 use crate::Result;
-use crate::{Event, Listener};
+use crate::{Event, KeyboardProxy, Listener, MouseProxy};
 
 #[dbus_proxy(default_service = "org.qemu", interface = "org.qemu.Display1.Console")]
 pub trait Console {
@@ -34,16 +34,24 @@ pub trait Console {
 #[derivative(Debug)]
 pub struct Console {
     #[derivative(Debug = "ignore")]
-    proxy: ConsoleProxy<'static>,
+    pub proxy: ConsoleProxy<'static>,
+    #[derivative(Debug = "ignore")]
+    pub keyboard: KeyboardProxy<'static>,
+    #[derivative(Debug = "ignore")]
+    pub mouse: MouseProxy<'static>,
 }
 
 impl Console {
     pub fn new(conn: &zbus::Connection, idx: u32) -> Result<Self> {
-        let proxy = ConsoleProxy::new_for_owned_path(
-            conn.clone(),
-            format!("/org/qemu/Display1/Console_{}", idx),
-        )?;
-        Ok(Self { proxy })
+        let obj_path = format!("/org/qemu/Display1/Console_{}", idx);
+        let proxy = ConsoleProxy::new_for_owned_path(conn.clone(), obj_path.clone())?;
+        let keyboard = KeyboardProxy::new_for_owned_path(conn.clone(), obj_path.clone())?;
+        let mouse = MouseProxy::new_for_owned_path(conn.clone(), obj_path)?;
+        Ok(Self {
+            proxy,
+            keyboard,
+            mouse,
+        })
     }
 
     pub fn label(&self) -> Result<String> {
@@ -68,7 +76,7 @@ impl Console {
             let mut s = zbus::ObjectServer::new(&c);
             let err = Rc::new(RefCell::new(None));
             s.at(
-                &zvariant::ObjectPath::from_str_unchecked("/org/qemu/Display1/Listener"),
+                "/org/qemu/Display1/Listener",
                 Listener::new(tx, err.clone()),
             )
             .unwrap();
@@ -100,18 +108,18 @@ impl Console {
             let mut s = zbus::ObjectServer::new(&c);
             let err = Rc::new(RefCell::new(None));
             s.at(
-                &zvariant::ObjectPath::from_str_unchecked("/org/qemu/Display1/Listener"),
+                "/org/qemu/Display1/Listener",
                 Listener::new(tx, err.clone()),
             )
             .unwrap();
             loop {
                 if let Err(e) = s.try_handle_next() {
                     eprintln!("Listener DBus error: {}", e);
-                    return;
+                    break;
                 }
                 if let Some(e) = &*err.borrow() {
                     eprintln!("Listener channel error: {}", e);
-                    return;
+                    break;
                 }
             }
         });

qemu-display-listener/src/keyboard.rs

@@ -0,0 +1,24 @@
+use enumflags2::BitFlags;
+use serde::{Deserialize, Serialize};
+use zbus::dbus_proxy;
+use zvariant::derive::Type;
+
+#[repr(u32)]
+#[derive(Type, BitFlags, Debug, PartialEq, Copy, Clone, Serialize, Deserialize)]
+pub enum KeyboardModifiers {
+    Scroll = 0x1,
+    Num = 0x2,
+    Caps = 0x4,
+}
+
+#[dbus_proxy(default_service = "org.qemu", interface = "org.qemu.Display1.Keyboard")]
+pub trait Keyboard {
+    /// Press method
+    fn press(&self, keycode: u32) -> zbus::Result<()>;
+
+    /// Release method
+    fn release(&self, keycode: u32) -> zbus::Result<()>;
+
+    #[dbus_proxy(property)]
+    fn modifiers(&self) -> zbus::Result<BitFlags<KeyboardModifiers>>;
+}

qemu-display-listener/src/lib.rs

@@ -9,6 +9,12 @@ pub use vm::*;
 mod console;
 pub use console::*;
 
+mod keyboard;
+pub use keyboard::*;
+
+mod mouse;
+pub use mouse::*;
+
 mod listener;
 pub use listener::*;
 

qemu-display-listener/src/listener.rs

@@ -1,26 +1,28 @@
 use std::cell::RefCell;
+use std::ops::Drop;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::rc::Rc;
 use std::sync::mpsc::{SendError, Sender};
-use std::ops::Drop;
 
 use zbus::{dbus_interface, export::zvariant::Fd};
 
 #[derive(Debug)]
 pub struct Scanout {
-        fd: RawFd,
-        width: u32,
-        height: u32,
-        stride: u32,
-        fourcc: u32,
-        modifier: u64,
-        y0_top: bool,
+    pub fd: RawFd,
+    pub width: u32,
+    pub height: u32,
+    pub stride: u32,
+    pub fourcc: u32,
+    pub modifier: u64,
+    pub y0_top: bool,
 }
 
 impl Drop for Scanout {
     fn drop(&mut self) {
         if self.fd >= 0 {
-            unsafe { libc::close(self.fd); }
+            unsafe {
+                libc::close(self.fd);
+            }
         }
     }
 }
@@ -51,6 +53,7 @@ pub enum Event {
         data: Vec<u8>,
     },
     Scanout(Scanout),
+    Disconnected,
 }
 
 pub(crate) trait EventSender {
@@ -134,3 +137,9 @@ impl<E: EventSender> Listener<E> {
         }
     }
 }
+
+impl<E: EventSender> Drop for Listener<E> {
+    fn drop(&mut self) {
+        self.send(Event::Disconnected)
+    }
+}

qemu-display-listener/src/mouse.rs

@@ -0,0 +1,27 @@
+use serde_repr::{Deserialize_repr, Serialize_repr};
+use zbus::dbus_proxy;
+use zvariant::derive::Type;
+
+#[repr(u32)]
+#[derive(Deserialize_repr, Serialize_repr, Type, Debug, PartialEq)]
+pub enum MouseButton {
+    Left,
+    Middle,
+    Right,
+    WheelUp,
+    WheelDown,
+    Side,
+    Extra,
+}
+
+#[dbus_proxy(default_service = "org.qemu", interface = "org.qemu.Display1.Mouse")]
+pub trait Mouse {
+    /// Press method
+    fn press(&self, button: MouseButton) -> zbus::Result<()>;
+
+    /// Release method
+    fn release(&self, button: MouseButton) -> zbus::Result<()>;
+
+    /// SetAbsPosition method
+    fn set_abs_position(&self, x: u32, y: u32) -> zbus::Result<()>;
+}

qemu-gtk4/Cargo.toml

@@ -12,8 +12,19 @@ pretty_env_logger = "0.4"
 gettext-rs = { version = "0.5", features = ["gettext-system"] }
 gtk-macros = "0.2"
 once_cell = "1.5"
+khronos-egl = { version = "3.0.0", features = ["dynamic"] }
+libloading = "0.6"
+gl = "0.14.0"
 
 [dependencies.gtk]
 package = "gtk4"
 git = "https://github.com/gtk-rs/gtk4-rs"
-rev = "abea0c9980bc083494eceb30dfab5eeb99a73118"
+rev = "c43025157b12dba1112fad55962966769908a269"
+
+[dependencies.gdk-wl]
+package = "gdk4-wayland"
+git = "https://github.com/gtk-rs/gtk4-rs"
+rev = "c43025157b12dba1112fad55962966769908a269"
+
+[build-dependencies]
+gl_generator = "0.5.0"

qemu-gtk4/src/console.rs

@@ -1,11 +1,11 @@
-use glib::subclass::prelude::*;
 use glib::clone;
+use glib::subclass::prelude::*;
 use gtk::prelude::*;
 use gtk::subclass::widget::WidgetImplExt;
 use gtk::{glib, CompositeTemplate};
 use once_cell::sync::OnceCell;
 
-use qemu_display_listener::{Console, Event};
+use qemu_display_listener::{Console, Event, MouseButton};
 
 mod imp {
     use super::*;
@@ -48,6 +48,45 @@ mod imp {
     impl ObjectImpl for QemuConsole {
         fn constructed(&self, obj: &Self::Type) {
             self.parent_constructed(obj);
+
+            let ec = gtk::EventControllerKey::new();
+            ec.set_propagation_phase(gtk::PropagationPhase::Capture);
+            self.area.add_controller(&ec);
+            ec.connect_key_pressed(clone!(@weak obj => move |_, _keyval, keycode, _state| {
+                let c = obj.qemu_console();
+                let _ = c.keyboard.press(keycode);
+                glib::signal::Inhibit(true)
+            }));
+            ec.connect_key_released(clone!(@weak obj => move |_, _keyval, keycode, _state| {
+                let c = obj.qemu_console();
+                let _ = c.keyboard.release(keycode);
+            }));
+
+            let ec = gtk::EventControllerMotion::new();
+            self.area.add_controller(&ec);
+            ec.connect_motion(clone!(@weak obj => move |_, x, y| {
+                obj.motion(x, y);
+            }));
+
+            let ec = gtk::GestureClick::new();
+            ec.set_button(0);
+            self.area.add_controller(&ec);
+            ec.connect_pressed(clone!(@weak obj => move |gesture, _n_press, x, y| {
+                let c = obj.qemu_console();
+                let button = from_gdk_button(gesture.get_current_button());
+                obj.motion(x, y);
+                let _ = c.mouse.press(button);
+            }));
+            ec.connect_released(clone!(@weak obj => move |gesture, _n_press, x, y| {
+                let c = obj.qemu_console();
+                let button = from_gdk_button(gesture.get_current_button());
+                obj.motion(x, y);
+                let _ = c.mouse.release(button);
+            }));
+
+            self.area.set_sensitive(true);
+            self.area.set_focusable(true);
+            self.area.set_focus_on_click(true);
         }
 
         // Needed for direct subclasses of GtkWidget;
@@ -86,6 +125,9 @@ impl QemuConsole {
                         con.label.set_label(&format!("{:?}", s));
                         con.area.set_scanout(s);
                     }
+                    Event::Disconnected => {
+                        con.label.set_label("Console disconnected!");
+                    }
                     _ => ()
                 }
                 Continue(true)
@@ -93,4 +135,36 @@ impl QemuConsole {
         );
         priv_.console.set(console).unwrap();
     }
+
+    fn qemu_console(&self) -> &Console {
+        let priv_ = imp::QemuConsole::from_instance(self);
+        priv_.console.get().expect("Console is not yet set!")
+    }
+
+    fn motion(&self, x: f64, y: f64) {
+        let priv_ = imp::QemuConsole::from_instance(self);
+
+        // FIXME: scaling, centering etc..
+        let widget_w = self.get_width();
+        let widget_h = self.get_height();
+        let _widget_scale = self.get_scale_factor();
+
+        let c = self.qemu_console();
+        // FIXME: ideally, we would use ConsoleProxy cached properties instead
+        let x = (x / widget_w as f64) * priv_.area.scanout_size().0 as f64;
+        let y = (y / widget_h as f64) * priv_.area.scanout_size().1 as f64;
+        let _ = c.mouse.set_abs_position(x as u32, y as u32);
+
+        // FIXME: focus on click doesn't work
+        priv_.area.grab_focus();
+    }
+}
+
+fn from_gdk_button(button: u32) -> MouseButton {
+    match button {
+        1 => MouseButton::Left,
+        2 => MouseButton::Middle,
+        3 => MouseButton::Right,
+        _ => MouseButton::Extra,
+    }
 }

qemu-gtk4/src/console_area.rs

@@ -1,9 +1,11 @@
-use std::cell::Cell;
+use gdk_wl::WaylandDisplayManualExt;
 use glib::subclass::prelude::*;
-use glib::clone;
 use gtk::prelude::*;
-use gtk::{glib, graphene, gdk};
+use gtk::{gdk, glib, graphene};
+use std::cell::{Cell, RefCell};
 
+use crate::egl;
+use gl::{self, types::*};
 use qemu_display_listener::Scanout;
 
 mod imp {
@@ -13,6 +15,9 @@ mod imp {
 
     pub struct QemuConsoleArea {
         pub scanout: Cell<Option<Scanout>>,
+        pub scanout_size: Cell<(u32, u32)>,
+        pub tex_id: Cell<GLuint>,
+        pub texture: RefCell<Option<gdk::Texture>>,
     }
 
     impl ObjectSubclass for QemuConsoleArea {
@@ -28,24 +33,27 @@ mod imp {
         fn new() -> Self {
             Self {
                 scanout: Cell::new(None),
+                scanout_size: Cell::new((0, 0)),
+                tex_id: Cell::new(0),
+                texture: RefCell::new(None),
             }
         }
+
+        fn class_init(_klass: &mut Self::Class) {
+            // GL loading could be done earlier?
+            let egl = egl::egl();
+
+            gl::load_with(|s| {
+                egl.get_proc_address(s)
+                    .map(|f| f as _)
+                    .unwrap_or(std::ptr::null())
+            });
+        }
     }
 
     impl ObjectImpl for QemuConsoleArea {
         fn constructed(&self, obj: &Self::Type) {
             self.parent_constructed(obj);
-
-            let ec = gtk::EventControllerLegacy::new();
-            // XXX: where are the key events?
-            // ec.set_propagation_phase(gtk::PropagationPhase::Bubble);
-            obj.add_controller(&ec);
-            ec.connect_event(clone!(@weak obj => move |_, e| {
-                dbg!(e);
-                true
-            }));
-            obj.set_focusable(true);
-            obj.set_focus_on_click(true);
         }
     }
 
@@ -54,8 +62,10 @@ mod imp {
             let (width, height) = (widget.get_width() as f32, widget.get_height() as f32);
             let whole = &graphene::Rect::new(0_f32, 0_f32, width, height);
             // TODO: make this a CSS style?
-            snapshot.append_color(&gdk::RGBA::black(), whole);
-            //snapshot.append_texture(priv_.texture, whole);
+            //snapshot.append_color(&gdk::RGBA::black(), whole);
+            if let Some(texture) = &*self.texture.borrow() {
+                snapshot.append_texture(texture, whole);
+            }
         }
     }
 }
@@ -65,8 +75,93 @@ glib::wrapper! {
 }
 
 impl QemuConsoleArea {
+    pub fn tex_id(&self) -> GLuint {
+        let priv_ = imp::QemuConsoleArea::from_instance(self);
+        let mut tex_id = priv_.tex_id.get();
+        if tex_id == 0 {
+            unsafe { gl::GenTextures(1, &mut tex_id) }
+            priv_.tex_id.set(tex_id);
+        }
+        tex_id
+    }
+
+    fn update_texture(&self, s: &Scanout) {
+        let priv_ = imp::QemuConsoleArea::from_instance(self);
+        let ctxt = gdk::GLContext::get_current().unwrap();
+        let tex =
+            unsafe { gdk::GLTexture::new(&ctxt, self.tex_id(), s.width as i32, s.height as i32) };
+
+        //tex.save_to_png("/tmp/tex.png");
+        //tex.clone().downcast::<gdk::GLTexture>().unwrap().release();
+        tex.release();
+        *priv_.texture.borrow_mut() = Some(tex.upcast());
+    }
+
+    pub fn scanout_size(&self) -> (u32, u32) {
+        let priv_ = imp::QemuConsoleArea::from_instance(self);
+
+        priv_.scanout_size.get()
+    }
+
     pub fn set_scanout(&self, s: Scanout) {
         let priv_ = imp::QemuConsoleArea::from_instance(self);
-        priv_.scanout.replace(Some(s));
+        let egl = egl::egl();
+
+        let egl_dpy = if let Ok(dpy) = self.get_display().downcast::<gdk_wl::WaylandDisplay>() {
+            let wl_dpy = dpy.get_wl_display();
+            egl.get_display(wl_dpy.as_ref().c_ptr() as _)
+                .expect("Failed to get EGL display")
+        } else {
+            eprintln!("Unsupported display kind");
+            return;
+        };
+
+        let attribs = vec![
+            egl::WIDTH as usize,
+            s.width as usize,
+            egl::HEIGHT as usize,
+            s.height as usize,
+            egl::LINUX_DRM_FOURCC_EXT as usize,
+            s.fourcc as usize,
+            egl::DMA_BUF_PLANE0_FD_EXT as usize,
+            s.fd as usize,
+            egl::DMA_BUF_PLANE0_PITCH_EXT as usize,
+            s.stride as usize,
+            egl::DMA_BUF_PLANE0_OFFSET_EXT as usize,
+            0,
+            egl::DMA_BUF_PLANE0_MODIFIER_LO_EXT as usize,
+            (s.modifier & 0xffffffff) as usize,
+            egl::DMA_BUF_PLANE0_MODIFIER_HI_EXT as usize,
+            (s.modifier >> 32 & 0xffffffff) as usize,
+            egl::NONE as usize,
+        ];
+
+        let img = egl
+            .create_image(
+                egl_dpy,
+                unsafe { egl::Context::from_ptr(egl::NO_CONTEXT) },
+                egl::LINUX_DMA_BUF_EXT,
+                unsafe { egl::ClientBuffer::from_ptr(std::ptr::null_mut()) },
+                &attribs,
+            )
+            .expect("Failed to eglCreateImage");
+
+        let tex_id = self.tex_id();
+        unsafe { gl::BindTexture(gl::TEXTURE_2D, tex_id) }
+        if let Some(image_target) = egl::image_target_texture_2d_oes() {
+            image_target(gl::TEXTURE_2D, img.as_ptr() as gl::types::GLeglImageOES);
+        } else {
+            eprintln!("Failed to set texture image");
+        }
+
+        self.update_texture(&s);
+        self.queue_draw();
+
+        if let Err(e) = egl.destroy_image(egl_dpy, img) {
+            eprintln!("Destroy image failed: {}", e);
+        }
+
+        priv_.scanout_size.set((s.width, s.height));
+        priv_.scanout.set(Some(s));
     }
 }

qemu-gtk4/src/egl.rs

@@ -0,0 +1,35 @@
+pub use khronos_egl::*;
+use once_cell::sync::OnceCell;
+
+type EglInstance = Instance<khronos_egl::Dynamic<libloading::Library, khronos_egl::EGL1_5>>;
+
+pub(crate) fn egl() -> &'static EglInstance {
+    static INSTANCE: OnceCell<EglInstance> = OnceCell::new();
+    INSTANCE.get_or_init(|| {
+        let lib = libloading::Library::new("libEGL.so").expect("unable to find libEGL.so");
+        unsafe {
+            khronos_egl::DynamicInstance::<khronos_egl::EGL1_5>::load_required_from(lib)
+                .expect("unable to load libEGL.so")
+        }
+    })
+}
+
+pub const LINUX_DMA_BUF_EXT: Enum = 0x3270;
+pub const LINUX_DRM_FOURCC_EXT: Int = 0x3271;
+pub const DMA_BUF_PLANE0_FD_EXT: Int = 0x3272;
+pub const DMA_BUF_PLANE0_OFFSET_EXT: Int = 0x3273;
+pub const DMA_BUF_PLANE0_PITCH_EXT: Int = 0x3274;
+pub const DMA_BUF_PLANE0_MODIFIER_LO_EXT: Int = 0x3443;
+pub const DMA_BUF_PLANE0_MODIFIER_HI_EXT: Int = 0x3444;
+
+// GLAPI void APIENTRY glEGLImageTargetTexture2DOES (GLenum target, GLeglImageOES image);
+
+pub type ImageTargetTexture2DOesFn = extern "C" fn(gl::types::GLenum, gl::types::GLeglImageOES);
+
+pub fn image_target_texture_2d_oes() -> Option<ImageTargetTexture2DOesFn> {
+    unsafe {
+        egl()
+            .get_proc_address("glEGLImageTargetTexture2DOES")
+            .map(|f| std::mem::transmute::<_, ImageTargetTexture2DOesFn>(f))
+    }
+}

qemu-gtk4/src/main.rs

@@ -1,11 +1,11 @@
 #[allow(clippy::new_without_default)]
-
 mod application;
 #[rustfmt::skip]
 mod config;
-mod window;
 mod console;
 mod console_area;
+mod egl;
+mod window;
 
 use application::QemuApplication;
 use config::{GETTEXT_PACKAGE, LOCALEDIR, RESOURCES_FILE};

qemu-gtk4/src/meson.build

@@ -23,6 +23,7 @@ sources = files(
   'config.rs',
   'console.rs',
   'console_area.rs',
+  'egl.rs',
   'main.rs',
   'window.rs',
 )

qemu-gtk4/src/window.rs

@@ -1,6 +1,6 @@
 use crate::application::QemuApplication;
-use crate::console::QemuConsole;
 use crate::config::{APP_ID, PROFILE};
+use crate::console::QemuConsole;
 use glib::signal::Inhibit;
 use gtk::subclass::prelude::*;
 use gtk::{self, prelude::*};