From b60c756d8b992a9e59b0362effea5be6c71503ff Mon Sep 17 00:00:00 2001 From: UnderSampled Date: Sun, 17 Sep 2023 23:17:54 -0400 Subject: [PATCH] Import Dmabufs into CameraItems during render call. Consider replacing with a separate import_buffers call, or create buffers locally and expose export_buffers (slightly less state required) if this is too slow. --- Cargo.lock | 4 ++ Cargo.toml | 1 + src/core/buffers.rs | 50 ++++++++++++++++ src/core/mod.rs | 1 + src/main.rs | 14 +++-- src/nodes/items/camera.rs | 122 ++++++++++++++++++++++++-------------- src/wayland/mod.rs | 60 ++++--------------- 7 files changed, 154 insertions(+), 98 deletions(-) create mode 100644 src/core/buffers.rs diff --git a/Cargo.lock b/Cargo.lock index 78155a2..84a7bbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -613,6 +613,9 @@ name = "drm-fourcc" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" +dependencies = [ + "serde", +] [[package]] name = "drm-sys" @@ -2115,6 +2118,7 @@ dependencies = [ "console-subscriber", "ctrlc", "directories", + "drm-fourcc", "glam 0.23.0", "global_counter", "input-event-codes", diff --git a/Cargo.toml b/Cargo.toml index db999ac..e55e8e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ xkbcommon = { version = "0.6.0", default-features = false, optional = true } ctrlc = "3.4.1" libc = "0.2.148" input-event-codes = "5.16.8" +drm-fourcc = { version = "2.2.0", features = ["serde"] } [dependencies.smithay] # git = "https://github.com/technobaboo/smithay.git" # Until we get stereokit to understand OES samplers and external textures diff --git a/src/core/buffers.rs b/src/core/buffers.rs new file mode 100644 index 0000000..477ce76 --- /dev/null +++ b/src/core/buffers.rs @@ -0,0 +1,50 @@ +use std::ffi::c_void; + +use color_eyre::eyre::{ensure, Result}; +use smithay::backend::{egl::EGLContext, renderer::gles::GlesRenderer}; +use stereokit as sk; + +struct EGLRawHandles { + display: *const c_void, + config: *const c_void, + context: *const c_void, +} +fn get_sk_egl() -> Result { + ensure!( + unsafe { sk::sys::backend_graphics_get() } + == sk::sys::backend_graphics__backend_graphics_opengles_egl, + "StereoKit is not running using EGL!" + ); + + Ok(unsafe { + EGLRawHandles { + display: sk::sys::backend_opengl_egl_get_display() as *const c_void, + config: sk::sys::backend_opengl_egl_get_config() as *const c_void, + context: sk::sys::backend_opengl_egl_get_context() as *const c_void, + } + }) +} + +pub struct BufferManager { + pub renderer: GlesRenderer, +} +impl BufferManager { + pub fn new() -> Result { + let egl_raw_handles = get_sk_egl()?; + let renderer = unsafe { + GlesRenderer::new(EGLContext::from_raw( + egl_raw_handles.display, + egl_raw_handles.config, + egl_raw_handles.context, + )?)? + }; + + Ok(BufferManager { renderer }) + } + + pub fn make_context_current(&self) { + unsafe { + let _ = self.renderer.egl_context().make_current(); + } + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index e2910d1..ad692cd 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -9,3 +9,4 @@ pub mod resource; pub mod scenegraph; pub mod task; pub mod typed_datamap; +pub mod buffers; diff --git a/src/main.rs b/src/main.rs index 8f3603a..14f588d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ mod wayland; use crate::core::client::CLIENTS; use crate::core::client_state::ClientState; -use crate::core::destroy_queue; +use crate::core::{destroy_queue, buffers}; use crate::nodes::items::camera; use crate::nodes::{audio, drawable, hmd, input}; use crate::objects::input::eye_pointer::EyePointer; @@ -199,8 +199,10 @@ fn main() { let event_loop_info = info_receiver.blocking_recv().unwrap(); let _tokio_handle = event_loop_info.tokio_handle.enter(); + let mut buffer_manager = buffers::BufferManager::new().expect("Could not initialize buffer manager"); + #[cfg(feature = "wayland")] - let mut wayland = wayland::Wayland::new().expect("Could not initialize wayland"); + let mut wayland = wayland::Wayland::new(&buffer_manager).expect("Could not initialize wayland"); info!("Stardust ready!"); let mut startup_child = (|| { @@ -293,13 +295,13 @@ fn main() { ); #[cfg(feature = "wayland")] - wayland.update(sk); + wayland.update(sk, &mut buffer_manager); drawable::draw(sk); audio::update(sk); - #[cfg(feature = "wayland")] - wayland.make_context_current(); - camera::update(sk); + buffer_manager.make_context_current(); + + camera::update(sk, &mut buffer_manager); }, |_sk| { info!("Cleanly shut down StereoKit"); diff --git a/src/nodes/items/camera.rs b/src/nodes/items/camera.rs index dde195b..e0d47b5 100644 --- a/src/nodes/items/camera.rs +++ b/src/nodes/items/camera.rs @@ -3,7 +3,7 @@ use crate::{ core::{ client::{Client, INTERNAL_CLIENT}, registry::Registry, - scenegraph::MethodResponseSender, + scenegraph::MethodResponseSender, buffers::BufferManager, }, nodes::{ drawable::{model::ModelPart, shaders::UNLIT_SHADER_BYTES, Drawable}, @@ -12,23 +12,23 @@ use crate::{ Message, Node, }, }; -use color_eyre::eyre::{bail, eyre, Result}; +use color_eyre::eyre::{bail, Result}; use glam::Mat4; use lazy_static::lazy_static; use mint::{RowMatrix4, Vector2}; use nanoid::nanoid; use once_cell::sync::OnceCell; use parking_lot::Mutex; -use rustc_hash::FxHashMap; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use smithay::{backend::{renderer::ImportDma, allocator::{dmabuf::{Dmabuf, DmabufFlags}, Modifier, Fourcc, Buffer}}, utils::Size}; use stardust_xr::{ scenegraph::ScenegraphError, schemas::flex::{deserialize, serialize}, values::Transform, }; -use std::sync::Arc; +use std::{sync::Arc, ffi::c_void}; use stereokit::{ - Color128, Material, Rect, RenderLayer, StereoKitDraw, Tex, TextureType, Transparency, + Color128, Material, Rect, RenderLayer, StereoKitDraw, Tex, TextureType, Transparency, TextureFormat, }; use tokio::sync::{mpsc, oneshot}; @@ -49,13 +49,29 @@ struct FrameInfo { preview_size: Vector2, } +#[derive(Serialize, Deserialize)] +struct BufferPlaneInfo{ + idx: u32, + offset: u32, + stride: u32, + modifier: Modifier, +} + +#[derive(Serialize, Deserialize)] +struct BufferInfo { + size: (u32, u32), + fourcc: Fourcc, + flags: u32, + planes: Vec, +} + pub struct CameraItem { space: Arc, frame_info: Mutex, - sk_textures: Mutex>, + sk_tex: OnceCell, sk_mat: OnceCell>, - render_requests_tx: mpsc::Sender<((usize, usize), oneshot::Sender<()>)>, - render_requests_rx: Mutex)>>, + render_requests_tx: mpsc::Sender<(Dmabuf, oneshot::Sender<()>)>, + render_requests_rx: Mutex)>>, rendered_notifiers: Mutex>>, applied_to: Registry, apply_to: Registry, @@ -73,7 +89,7 @@ impl CameraItem { proj_matrix, preview_size, }), - sk_textures: Mutex::new(FxHashMap::default()), + sk_tex: OnceCell::new(), sk_mat: OnceCell::new(), render_requests_tx, render_requests_rx: Mutex::new(render_requests_rx), @@ -89,21 +105,9 @@ impl CameraItem { ); } - fn import_buffers_flex( - node: &Node, - calling_client: Arc, - message: Message, - ) -> Result<()> { - let ItemType::Camera(camera) = &node.item.get().unwrap().specialization else { - bail!("Wrong item type?") - }; - - Ok(()) - } - fn render_flex( node: &Node, - calling_client: Arc, + _calling_client: Arc, message: Message, response: MethodResponseSender, ) { @@ -116,17 +120,32 @@ impl CameraItem { let (rendered_tx, rendered_rx) = oneshot::channel(); - let buffer_to_render: (usize, usize) = (Arc::as_ptr(&calling_client) as usize, deserialize(&message.data).unwrap()); + let buffer_info: BufferInfo = deserialize(&message.data).unwrap(); + let mut fds = message.fds.iter(); + let mut builder = Dmabuf::builder( + Into::>::into(( + buffer_info.size.0 as i32, + buffer_info.size.1 as i32, + )), + buffer_info.fourcc, + DmabufFlags::from_bits_truncate(buffer_info.flags), + ); + for plane in buffer_info.planes { + builder.add_plane( + fds.next().unwrap().try_clone().unwrap(), + plane.idx, + plane.offset, + plane.stride, + plane.modifier, + ); + } + let buffer_to_render = builder.build().unwrap(); - tokio::task::spawn(async move { - camera - .render_requests_tx - .send((buffer_to_render, rendered_tx)) - .await; - - rendered_rx.await; - response.send(Ok(Vec::new().into())); - }); + let _ = camera.render_requests_tx.try_send((buffer_to_render, rendered_tx)); + tokio::task::spawn(async move { + let _ = rendered_rx.await; + response.send(Ok(Vec::new().into())); + }); } fn apply_preview_material_flex( @@ -156,18 +175,17 @@ impl CameraItem { Ok(serialize(id)?.into()) } - pub fn update(&self, sk: &impl StereoKitDraw) { + pub fn update(&self, sk: &impl StereoKitDraw, buffer_manager: &mut BufferManager) { let frame_info = self.frame_info.lock(); - let mut sk_textures = self.sk_textures.lock(); if !self.apply_to.is_empty() { - let sk_tex = sk_textures.entry((0, 0)).or_insert_with(|| { + let sk_tex = self.sk_tex.get_or_init(|| { sk.tex_gen_color( Color128::default(), frame_info.preview_size.x as i32, frame_info.preview_size.y as i32, TextureType::RENDER_TARGET, - stereokit::TextureFormat::RGBA32Linear, + TextureFormat::RGBA32Linear, ) }); let sk_mat = self.sk_mat.get_or_init(|| { @@ -184,7 +202,7 @@ impl CameraItem { if !self.applied_to.is_empty() { sk.render_to( - sk_textures.get(&(0, 0)).unwrap(), + self.sk_tex.get().unwrap(), frame_info.proj_matrix, self.space.global_transform(), RenderLayer::all(), @@ -201,13 +219,29 @@ impl CameraItem { let mut render_notifiers = self.rendered_notifiers.lock(); let mut render_requests_rx = self.render_requests_rx.lock(); while let Ok((buffer_to_render, rendered_tx)) = render_requests_rx.try_recv() { - let Some(sk_tex) = sk_textures.get(&buffer_to_render).take() else { - // TODO: Consider allocating a new buffer in a or_insert_with (requires desired formats and a size) - // For now, rendering to no buffer means not rendering at all. + let Ok(smithay_tex) = buffer_manager.renderer.import_dmabuf(&buffer_to_render, None) else { + // TODO: Failed to import the buffer somehow. This fails gracefully, but silently. render_notifiers.push(rendered_tx); continue; }; + let sk_tex = sk.tex_create( + TextureType::IMAGE_NO_MIPS, + TextureFormat::RGBA32, + ); + unsafe { + sk.tex_set_surface( + &sk_tex, + smithay_tex.tex_id() as usize as *mut c_void, + TextureType::IMAGE_NO_MIPS, + smithay::backend::renderer::gles::ffi::RGBA8.into(), + buffer_to_render.size().w, + buffer_to_render.size().h, + 1, + false, + ); + } + sk.render_to( sk_tex, frame_info.proj_matrix, @@ -227,17 +261,17 @@ impl CameraItem { pub fn send_rendered(&self) { for notifier in self.rendered_notifiers.lock().drain(..) { - notifier.send(()); + let _ = notifier.send(()); } } } -pub fn update(sk: &impl StereoKitDraw) { +pub fn update(sk: &impl StereoKitDraw, buffer_manager: &mut BufferManager) { for camera in ITEM_TYPE_INFO_CAMERA.items.get_valid_contents() { let ItemType::Camera(camera) = &camera.specialization else { continue; }; - camera.update(sk); + camera.update(sk, buffer_manager); } } diff --git a/src/wayland/mod.rs b/src/wayland/mod.rs index 81e1177..bc53100 100644 --- a/src/wayland/mod.rs +++ b/src/wayland/mod.rs @@ -10,16 +10,15 @@ mod xdg_shell; pub mod xwayland; use self::{state::WaylandState, surface::CORE_SURFACES}; +use crate::core::buffers::BufferManager; use crate::wayland::seat::SeatData; use crate::{core::task, wayland::state::ClientState}; -use color_eyre::eyre::{ensure, Result}; +use color_eyre::eyre::Result; use global_counter::primitive::exact::CounterU32; use once_cell::sync::OnceCell; use parking_lot::Mutex; use sk::StereoKitDraw; use smithay::backend::allocator::dmabuf::Dmabuf; -use smithay::backend::egl::EGLContext; -use smithay::backend::renderer::gles::GlesRenderer; use smithay::backend::renderer::ImportDma; use smithay::reexports::wayland_server::backend::ClientId; use smithay::reexports::wayland_server::DisplayHandle; @@ -28,7 +27,6 @@ use std::ffi::OsStr; use std::os::fd::OwnedFd; use std::os::unix::prelude::AsRawFd; use std::{ - ffi::c_void, os::unix::{net::UnixListener, prelude::FromRawFd}, sync::Arc, }; @@ -42,27 +40,6 @@ use tracing::{debug_span, info, instrument}; pub static WAYLAND_DISPLAY: OnceCell = OnceCell::new(); pub static SERIAL_COUNTER: CounterU32 = CounterU32::new(0); -struct EGLRawHandles { - display: *const c_void, - config: *const c_void, - context: *const c_void, -} -fn get_sk_egl() -> Result { - ensure!( - unsafe { sk::sys::backend_graphics_get() } - == sk::sys::backend_graphics__backend_graphics_opengles_egl, - "StereoKit is not running using EGL!" - ); - - Ok(unsafe { - EGLRawHandles { - display: sk::sys::backend_opengl_egl_get_display() as *const c_void, - config: sk::sys::backend_opengl_egl_get_config() as *const c_void, - context: sk::sys::backend_opengl_egl_get_context() as *const c_void, - } - }) -} - pub struct DisplayWrapper(Mutex>, DisplayHandle); impl DisplayWrapper { pub fn handle(&self) -> DisplayHandle { @@ -85,23 +62,13 @@ pub struct Wayland { display: Arc, pub socket_name: Option, join_handle: JoinHandle>, - renderer: GlesRenderer, dmabuf_rx: UnboundedReceiver, wayland_state: Arc>, #[cfg(feature = "xwayland")] pub xwayland_state: xwayland::XWaylandState, } impl Wayland { - pub fn new() -> Result { - let egl_raw_handles = get_sk_egl()?; - let renderer = unsafe { - GlesRenderer::new(EGLContext::from_raw( - egl_raw_handles.display, - egl_raw_handles.config, - egl_raw_handles.context, - )?)? - }; - + pub fn new(buffer_manager: &BufferManager) -> Result { let display: Display = Display::new()?; let display_handle = display.handle(); @@ -109,7 +76,7 @@ impl Wayland { let display = Arc::new(DisplayWrapper(Mutex::new(display), display_handle.clone())); #[cfg(feature = "xwayland")] let xwayland_state = xwayland::XWaylandState::create(&display_handle)?; - let wayland_state = WaylandState::new(display_handle, &renderer, dmabuf_tx); + let wayland_state = WaylandState::new(display_handle, &buffer_manager.renderer, dmabuf_tx); let socket = ListeningSocket::bind_auto("wayland", 0..33)?; let socket_name = socket @@ -127,7 +94,6 @@ impl Wayland { display, socket_name, join_handle, - renderer, dmabuf_rx, wayland_state, #[cfg(feature = "xwayland")] @@ -178,13 +144,17 @@ impl Wayland { })?) } - #[instrument(level = "debug", name = "Wayland frame", skip(self, sk))] - pub fn update(&mut self, sk: &impl StereoKitDraw) { + #[instrument( + level = "debug", + name = "Wayland frame", + skip(self, sk, buffer_manager) + )] + pub fn update(&mut self, sk: &impl StereoKitDraw, buffer_manager: &mut BufferManager) { while let Ok(dmabuf) = self.dmabuf_rx.try_recv() { - let _ = self.renderer.import_dmabuf(&dmabuf, None); + let _ = buffer_manager.renderer.import_dmabuf(&dmabuf, None); } for core_surface in CORE_SURFACES.get_valid_contents() { - core_surface.process(sk, &mut self.renderer); + core_surface.process(sk, &mut buffer_manager.renderer); } self.display.flush_clients(None); @@ -197,12 +167,6 @@ impl Wayland { core_surface.frame(sk, output.clone()); } } - - pub fn make_context_current(&self) { - unsafe { - let _ = self.renderer.egl_context().make_current(); - } - } } impl Drop for Wayland { fn drop(&mut self) {