Import Dmabufs into CameraItems during render call. Consider replacing with a separate import_buffers call, or create buffers locally and expose export_buffers (slightly less state required) if this is too slow.

Cargo.lock

@@ -613,6 +613,9 @@ name = "drm-fourcc"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "drm-sys"
@@ -2115,6 +2118,7 @@ dependencies = [
  "console-subscriber",
  "ctrlc",
  "directories",
+ "drm-fourcc",
  "glam 0.23.0",
  "global_counter",
  "input-event-codes",

Cargo.toml

@@ -62,6 +62,7 @@ xkbcommon = { version = "0.6.0", default-features = false, optional = true }
 ctrlc = "3.4.1"
 libc = "0.2.148"
 input-event-codes = "5.16.8"
+drm-fourcc = { version = "2.2.0", features = ["serde"] }
 
 [dependencies.smithay]
 # git = "https://github.com/technobaboo/smithay.git" # Until we get stereokit to understand OES samplers and external textures

src/core/buffers.rs

@@ -0,0 +1,50 @@
+use std::ffi::c_void;
+
+use color_eyre::eyre::{ensure, Result};
+use smithay::backend::{egl::EGLContext, renderer::gles::GlesRenderer};
+use stereokit as sk;
+
+struct EGLRawHandles {
+	display: *const c_void,
+	config: *const c_void,
+	context: *const c_void,
+}
+fn get_sk_egl() -> Result<EGLRawHandles> {
+	ensure!(
+		unsafe { sk::sys::backend_graphics_get() }
+			== sk::sys::backend_graphics__backend_graphics_opengles_egl,
+		"StereoKit is not running using EGL!"
+	);
+
+	Ok(unsafe {
+		EGLRawHandles {
+			display: sk::sys::backend_opengl_egl_get_display() as *const c_void,
+			config: sk::sys::backend_opengl_egl_get_config() as *const c_void,
+			context: sk::sys::backend_opengl_egl_get_context() as *const c_void,
+		}
+	})
+}
+
+pub struct BufferManager {
+	pub renderer: GlesRenderer,
+}
+impl BufferManager {
+	pub fn new() -> Result<Self> {
+		let egl_raw_handles = get_sk_egl()?;
+		let renderer = unsafe {
+			GlesRenderer::new(EGLContext::from_raw(
+				egl_raw_handles.display,
+				egl_raw_handles.config,
+				egl_raw_handles.context,
+			)?)?
+		};
+
+		Ok(BufferManager { renderer })
+	}
+
+	pub fn make_context_current(&self) {
+		unsafe {
+			let _ = self.renderer.egl_context().make_current();
+		}
+	}
+}

src/core/mod.rs

@@ -9,3 +9,4 @@ pub mod resource;
 pub mod scenegraph;
 pub mod task;
 pub mod typed_datamap;
+pub mod buffers;

src/main.rs

@@ -6,7 +6,7 @@ mod wayland;
 
 use crate::core::client::CLIENTS;
 use crate::core::client_state::ClientState;
-use crate::core::destroy_queue;
+use crate::core::{destroy_queue, buffers};
 use crate::nodes::items::camera;
 use crate::nodes::{audio, drawable, hmd, input};
 use crate::objects::input::eye_pointer::EyePointer;
@@ -199,8 +199,10 @@ fn main() {
 	let event_loop_info = info_receiver.blocking_recv().unwrap();
 	let _tokio_handle = event_loop_info.tokio_handle.enter();
 
+	let mut buffer_manager = buffers::BufferManager::new().expect("Could not initialize buffer manager");
+
 	#[cfg(feature = "wayland")]
-	let mut wayland = wayland::Wayland::new().expect("Could not initialize wayland");
+	let mut wayland = wayland::Wayland::new(&buffer_manager).expect("Could not initialize wayland");
 	info!("Stardust ready!");
 
 	let mut startup_child = (|| {
@@ -293,13 +295,13 @@ fn main() {
 				);
 
 				#[cfg(feature = "wayland")]
-				wayland.update(sk);
+				wayland.update(sk, &mut buffer_manager);
 				drawable::draw(sk);
 				audio::update(sk);
-				#[cfg(feature = "wayland")]
-				wayland.make_context_current();
 
-				camera::update(sk);
+				buffer_manager.make_context_current();
+
+				camera::update(sk, &mut buffer_manager);
 			},
 			|_sk| {
 				info!("Cleanly shut down StereoKit");

src/nodes/items/camera.rs

@@ -3,7 +3,7 @@ use crate::{
 	core::{
 		client::{Client, INTERNAL_CLIENT},
 		registry::Registry,
-		scenegraph::MethodResponseSender,
+		scenegraph::MethodResponseSender, buffers::BufferManager,
 	},
 	nodes::{
 		drawable::{model::ModelPart, shaders::UNLIT_SHADER_BYTES, Drawable},
@@ -12,23 +12,23 @@ use crate::{
 		Message, Node,
 	},
 };
-use color_eyre::eyre::{bail, eyre, Result};
+use color_eyre::eyre::{bail, Result};
 use glam::Mat4;
 use lazy_static::lazy_static;
 use mint::{RowMatrix4, Vector2};
 use nanoid::nanoid;
 use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
-use rustc_hash::FxHashMap;
+use serde::{Deserialize, Serialize};
-use serde::Deserialize;
+use smithay::{backend::{renderer::ImportDma, allocator::{dmabuf::{Dmabuf, DmabufFlags}, Modifier, Fourcc, Buffer}}, utils::Size};
 use stardust_xr::{
 	scenegraph::ScenegraphError,
 	schemas::flex::{deserialize, serialize},
 	values::Transform,
 };
-use std::sync::Arc;
+use std::{sync::Arc, ffi::c_void};
 use stereokit::{
-	Color128, Material, Rect, RenderLayer, StereoKitDraw, Tex, TextureType, Transparency,
+	Color128, Material, Rect, RenderLayer, StereoKitDraw, Tex, TextureType, Transparency, TextureFormat,
 };
 use tokio::sync::{mpsc, oneshot};
 
@@ -49,13 +49,29 @@ struct FrameInfo {
 	preview_size: Vector2<u32>,
 }
 
+#[derive(Serialize, Deserialize)]
+struct BufferPlaneInfo{
+	idx: u32,
+	offset: u32,
+	stride: u32,
+	modifier: Modifier,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BufferInfo {
+	size: (u32, u32),
+	fourcc: Fourcc,
+	flags: u32,
+	planes: Vec<BufferPlaneInfo>,
+}
+
 pub struct CameraItem {
 	space: Arc<Spatial>,
 	frame_info: Mutex<FrameInfo>,
-	sk_textures: Mutex<FxHashMap<(usize, usize), Tex>>,
+	sk_tex: OnceCell<Tex>,
 	sk_mat: OnceCell<Arc<Material>>,
-	render_requests_tx: mpsc::Sender<((usize, usize), oneshot::Sender<()>)>,
+	render_requests_tx: mpsc::Sender<(Dmabuf, oneshot::Sender<()>)>,
-	render_requests_rx: Mutex<mpsc::Receiver<((usize, usize), oneshot::Sender<()>)>>,
+	render_requests_rx: Mutex<mpsc::Receiver<(Dmabuf, oneshot::Sender<()>)>>,
 	rendered_notifiers: Mutex<Vec<oneshot::Sender<()>>>,
 	applied_to: Registry<ModelPart>,
 	apply_to: Registry<ModelPart>,
@@ -73,7 +89,7 @@ impl CameraItem {
 					proj_matrix,
 					preview_size,
 				}),
-				sk_textures: Mutex::new(FxHashMap::default()),
+				sk_tex: OnceCell::new(),
 				sk_mat: OnceCell::new(),
 				render_requests_tx,
 				render_requests_rx: Mutex::new(render_requests_rx),
@@ -89,21 +105,9 @@ impl CameraItem {
 		);
 	}
 
-	fn import_buffers_flex(
-		node: &Node,
-		calling_client: Arc<Client>,
-		message: Message,
-	) -> Result<()> {
-		let ItemType::Camera(camera) = &node.item.get().unwrap().specialization else {
-			bail!("Wrong item type?")
-		};
-
-		Ok(())
-	}
-
 	fn render_flex(
 		node: &Node,
-		calling_client: Arc<Client>,
+		_calling_client: Arc<Client>,
 		message: Message,
 		response: MethodResponseSender,
 	) {
@@ -116,17 +120,32 @@ impl CameraItem {
 
 		let (rendered_tx, rendered_rx) = oneshot::channel();
 
-		let buffer_to_render: (usize, usize) = (Arc::as_ptr(&calling_client) as usize, deserialize(&message.data).unwrap());
+		let buffer_info: BufferInfo = deserialize(&message.data).unwrap();
+		let mut fds = message.fds.iter();
+		let mut builder = Dmabuf::builder(
+			Into::<Size<i32, smithay::utils::Buffer>>::into((
+				buffer_info.size.0 as i32,
+				buffer_info.size.1 as i32,
+			)),
+			buffer_info.fourcc,
+			DmabufFlags::from_bits_truncate(buffer_info.flags),
+		);
+		for plane in buffer_info.planes {
+			builder.add_plane(
+				fds.next().unwrap().try_clone().unwrap(),
+				plane.idx,
+				plane.offset,
+				plane.stride,
+				plane.modifier,
+			);
+		}
+		let buffer_to_render = builder.build().unwrap();
 
-		tokio::task::spawn(async move {
+        let _ = camera.render_requests_tx.try_send((buffer_to_render, rendered_tx));
-			camera
+        tokio::task::spawn(async move {
-				.render_requests_tx
+            let _ = rendered_rx.await;
-				.send((buffer_to_render, rendered_tx))
+            response.send(Ok(Vec::new().into()));
-				.await;
+        });
-
-			rendered_rx.await;
-			response.send(Ok(Vec::new().into()));
-		});
 	}
 
 	fn apply_preview_material_flex(
@@ -156,18 +175,17 @@ impl CameraItem {
 		Ok(serialize(id)?.into())
 	}
 
-	pub fn update(&self, sk: &impl StereoKitDraw) {
+	pub fn update(&self, sk: &impl StereoKitDraw, buffer_manager: &mut BufferManager) {
 		let frame_info = self.frame_info.lock();
-		let mut sk_textures = self.sk_textures.lock();
 
 		if !self.apply_to.is_empty() {
-			let sk_tex = sk_textures.entry((0, 0)).or_insert_with(|| {
+			let sk_tex = self.sk_tex.get_or_init(|| {
 				sk.tex_gen_color(
 					Color128::default(),
 					frame_info.preview_size.x as i32,
 					frame_info.preview_size.y as i32,
 					TextureType::RENDER_TARGET,
-					stereokit::TextureFormat::RGBA32Linear,
+					TextureFormat::RGBA32Linear,
 				)
 			});
 			let sk_mat = self.sk_mat.get_or_init(|| {
@@ -184,7 +202,7 @@ impl CameraItem {
 
 		if !self.applied_to.is_empty() {
 			sk.render_to(
-				sk_textures.get(&(0, 0)).unwrap(),
+				self.sk_tex.get().unwrap(),
 				frame_info.proj_matrix,
 				self.space.global_transform(),
 				RenderLayer::all(),
@@ -201,13 +219,29 @@ impl CameraItem {
 		let mut render_notifiers = self.rendered_notifiers.lock();
 		let mut render_requests_rx = self.render_requests_rx.lock();
 		while let Ok((buffer_to_render, rendered_tx)) = render_requests_rx.try_recv() {
-			let Some(sk_tex) = sk_textures.get(&buffer_to_render).take() else {
+			let Ok(smithay_tex) = buffer_manager.renderer.import_dmabuf(&buffer_to_render, None) else {
-				// TODO: Consider allocating a new buffer in a or_insert_with (requires desired formats and a size)
+				// TODO: Failed to import the buffer somehow. This fails gracefully, but silently.
-				// For now, rendering to no buffer means not rendering at all.
 				render_notifiers.push(rendered_tx);
 				continue;
 			};
 
+			let sk_tex = sk.tex_create(
+				TextureType::IMAGE_NO_MIPS,
+				TextureFormat::RGBA32,
+			);
+			unsafe {
+				sk.tex_set_surface(
+					&sk_tex,
+					smithay_tex.tex_id() as usize as *mut c_void,
+					TextureType::IMAGE_NO_MIPS,
+					smithay::backend::renderer::gles::ffi::RGBA8.into(),
+					buffer_to_render.size().w,
+					buffer_to_render.size().h,
+					1,
+					false,
+				);
+			}
+
 			sk.render_to(
 				sk_tex,
 				frame_info.proj_matrix,
@@ -227,17 +261,17 @@ impl CameraItem {
 
 	pub fn send_rendered(&self) {
 		for notifier in self.rendered_notifiers.lock().drain(..) {
-			notifier.send(());
+			let _ = notifier.send(());
 		}
 	}
 }
 
-pub fn update(sk: &impl StereoKitDraw) {
+pub fn update(sk: &impl StereoKitDraw, buffer_manager: &mut BufferManager) {
 	for camera in ITEM_TYPE_INFO_CAMERA.items.get_valid_contents() {
 		let ItemType::Camera(camera) = &camera.specialization else {
 			continue;
 		};
-		camera.update(sk);
+		camera.update(sk, buffer_manager);
 	}
 }
 

src/wayland/mod.rs

@@ -10,16 +10,15 @@ mod xdg_shell;
 pub mod xwayland;
 
 use self::{state::WaylandState, surface::CORE_SURFACES};
+use crate::core::buffers::BufferManager;
 use crate::wayland::seat::SeatData;
 use crate::{core::task, wayland::state::ClientState};
-use color_eyre::eyre::{ensure, Result};
+use color_eyre::eyre::Result;
 use global_counter::primitive::exact::CounterU32;
 use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
 use sk::StereoKitDraw;
 use smithay::backend::allocator::dmabuf::Dmabuf;
-use smithay::backend::egl::EGLContext;
-use smithay::backend::renderer::gles::GlesRenderer;
 use smithay::backend::renderer::ImportDma;
 use smithay::reexports::wayland_server::backend::ClientId;
 use smithay::reexports::wayland_server::DisplayHandle;
@@ -28,7 +27,6 @@ use std::ffi::OsStr;
 use std::os::fd::OwnedFd;
 use std::os::unix::prelude::AsRawFd;
 use std::{
-	ffi::c_void,
 	os::unix::{net::UnixListener, prelude::FromRawFd},
 	sync::Arc,
 };
@@ -42,27 +40,6 @@ use tracing::{debug_span, info, instrument};
 pub static WAYLAND_DISPLAY: OnceCell<String> = OnceCell::new();
 pub static SERIAL_COUNTER: CounterU32 = CounterU32::new(0);
 
-struct EGLRawHandles {
-	display: *const c_void,
-	config: *const c_void,
-	context: *const c_void,
-}
-fn get_sk_egl() -> Result<EGLRawHandles> {
-	ensure!(
-		unsafe { sk::sys::backend_graphics_get() }
-			== sk::sys::backend_graphics__backend_graphics_opengles_egl,
-		"StereoKit is not running using EGL!"
-	);
-
-	Ok(unsafe {
-		EGLRawHandles {
-			display: sk::sys::backend_opengl_egl_get_display() as *const c_void,
-			config: sk::sys::backend_opengl_egl_get_config() as *const c_void,
-			context: sk::sys::backend_opengl_egl_get_context() as *const c_void,
-		}
-	})
-}
-
 pub struct DisplayWrapper(Mutex<Display<WaylandState>>, DisplayHandle);
 impl DisplayWrapper {
 	pub fn handle(&self) -> DisplayHandle {
@@ -85,23 +62,13 @@ pub struct Wayland {
 	display: Arc<DisplayWrapper>,
 	pub socket_name: Option<String>,
 	join_handle: JoinHandle<Result<()>>,
-	renderer: GlesRenderer,
 	dmabuf_rx: UnboundedReceiver<Dmabuf>,
 	wayland_state: Arc<Mutex<WaylandState>>,
 	#[cfg(feature = "xwayland")]
 	pub xwayland_state: xwayland::XWaylandState,
 }
 impl Wayland {
-	pub fn new() -> Result<Self> {
+	pub fn new(buffer_manager: &BufferManager) -> Result<Self> {
-		let egl_raw_handles = get_sk_egl()?;
-		let renderer = unsafe {
-			GlesRenderer::new(EGLContext::from_raw(
-				egl_raw_handles.display,
-				egl_raw_handles.config,
-				egl_raw_handles.context,
-			)?)?
-		};
-
 		let display: Display<WaylandState> = Display::new()?;
 		let display_handle = display.handle();
 
@@ -109,7 +76,7 @@ impl Wayland {
 		let display = Arc::new(DisplayWrapper(Mutex::new(display), display_handle.clone()));
 		#[cfg(feature = "xwayland")]
 		let xwayland_state = xwayland::XWaylandState::create(&display_handle)?;
-		let wayland_state = WaylandState::new(display_handle, &renderer, dmabuf_tx);
+		let wayland_state = WaylandState::new(display_handle, &buffer_manager.renderer, dmabuf_tx);
 
 		let socket = ListeningSocket::bind_auto("wayland", 0..33)?;
 		let socket_name = socket
@@ -127,7 +94,6 @@ impl Wayland {
 			display,
 			socket_name,
 			join_handle,
-			renderer,
 			dmabuf_rx,
 			wayland_state,
 			#[cfg(feature = "xwayland")]
@@ -178,13 +144,17 @@ impl Wayland {
 		})?)
 	}
 
-	#[instrument(level = "debug", name = "Wayland frame", skip(self, sk))]
+	#[instrument(
-	pub fn update(&mut self, sk: &impl StereoKitDraw) {
+		level = "debug",
+		name = "Wayland frame",
+		skip(self, sk, buffer_manager)
+	)]
+	pub fn update(&mut self, sk: &impl StereoKitDraw, buffer_manager: &mut BufferManager) {
 		while let Ok(dmabuf) = self.dmabuf_rx.try_recv() {
-			let _ = self.renderer.import_dmabuf(&dmabuf, None);
+			let _ = buffer_manager.renderer.import_dmabuf(&dmabuf, None);
 		}
 		for core_surface in CORE_SURFACES.get_valid_contents() {
-			core_surface.process(sk, &mut self.renderer);
+			core_surface.process(sk, &mut buffer_manager.renderer);
 		}
 
 		self.display.flush_clients(None);
@@ -197,12 +167,6 @@ impl Wayland {
 			core_surface.frame(sk, output.clone());
 		}
 	}
-
-	pub fn make_context_current(&self) {
-		unsafe {
-			let _ = self.renderer.egl_context().make_current();
-		}
-	}
 }
 impl Drop for Wayland {
 	fn drop(&mut self) {