feat(wayland): core surface

src/main.rs

@@ -3,20 +3,15 @@ mod nodes;
 mod wayland;
 
 use crate::nodes::model::{MODELS_TO_DROP, MODEL_REGISTRY};
-use crate::wayland::{ClientState, WaylandState};
+use crate::wayland::WaylandState;
 
 use self::core::eventloop::EventLoop;
-use anyhow::{ensure, Result};
+use anyhow::Result;
 use clap::Parser;
 use once_cell::sync::Lazy;
 use parking_lot::Mutex;
 use slog::Drain;
-use smithay::backend::egl::EGLContext;
-use smithay::backend::renderer::gles2::Gles2Renderer;
-use smithay::reexports::wayland_server::{Display, ListeningSocket};
-use std::ffi::c_void;
 use std::sync::Arc;
-use stereokit as sk;
 use stereokit::{lifecycle::DisplayMode, Settings};
 use tokio::{runtime::Handle, sync::oneshot};
 
@@ -34,27 +29,6 @@ struct CliArgs {
 	overlay: bool,
 }
 
-struct EGLRawHandles {
-	display: *const c_void,
-	config: *const c_void,
-	context: *const c_void,
-}
-fn get_sk_egl() -> Result<EGLRawHandles> {
-	ensure!(
-		unsafe { sk::sys::backend_graphics_get() }
-			== sk::sys::backend_graphics__backend_graphics_opengles_egl,
-		"StereoKit is not running using EGL!"
-	);
-
-	Ok(unsafe {
-		EGLRawHandles {
-			display: sk::sys::backend_opengl_egl_get_display() as *const c_void,
-			config: sk::sys::backend_opengl_egl_get_config() as *const c_void,
-			context: sk::sys::backend_opengl_egl_get_context() as *const c_void,
-		}
-	})
-}
-
 fn main() -> Result<()> {
 	let cli_args = Arc::new(CliArgs::parse());
 	let log = ::slog::Logger::root(::slog_stdlog::StdLog.fuse(), slog::o!());
@@ -72,41 +46,18 @@ fn main() -> Result<()> {
 		})
 		.init()
 		.expect("StereoKit failed to initialize");
+	println!("Init StereoKit");
 
 	let (event_stop_tx, event_stop_rx) = oneshot::channel::<()>();
 	let event_thread = std::thread::Builder::new()
 		.name("event_loop".to_owned())
 		.spawn(move || event_loop(event_stop_rx))?;
 
-	let egl_raw_handles = get_sk_egl()?;
+	let mut wayland = WaylandState::new(log)?;
-	let renderer = unsafe {
+	println!("Stardust ready!");
-		Gles2Renderer::new(
-			EGLContext::from_raw(
-				egl_raw_handles.display,
-				egl_raw_handles.config,
-				egl_raw_handles.context,
-				log.clone(),
-			)?,
-			log.clone(),
-		)?
-	};
-
-	let mut display: Display<WaylandState> = Display::new()?;
-	let socket = ListeningSocket::bind_auto("wayland", 0..33)?;
-	if let Some(socket_name) = socket.socket_name() {
-		println!("Wayland compositor {:?} active", socket_name);
-	}
-	let mut wayland_state = WaylandState::new(&display, renderer, log)?;
-
 	stereokit.run(
 		|draw_ctx| {
-			if let Ok(Some(client)) = socket.accept() {
+			wayland.frame(&stereokit);
-				let _ = display
-					.handle()
-					.insert_client(client, Arc::new(ClientState));
-			}
-			display.dispatch_clients(&mut wayland_state).unwrap();
-			display.flush_clients().unwrap();
 
 			nodes::root::Root::logic_step(stereokit.time_elapsed());
 			for model in MODEL_REGISTRY.get_valid_contents() {
@@ -114,17 +65,14 @@ fn main() -> Result<()> {
 			}
 			MODELS_TO_DROP.lock().clear();
 
-			unsafe { wayland_state.renderer.egl_context().make_current().unwrap() };
+			unsafe { wayland.renderer.egl_context().make_current().unwrap() };
 		},
 		|| {
 			println!("Cleanly shut down StereoKit");
 		},
 	);
 
-	drop(wayland_state);
+	drop(wayland);
-	drop(socket);
-	drop(display);
-	println!("Cleanly shut down the Wayland compositor");
 
 	let _ = event_stop_tx.send(());
 	event_thread

src/wayland/compositor.rs

@@ -1,8 +1,11 @@
-use super::WaylandState;
+use super::{surface::CoreSurface, WaylandState};
+use send_wrapper::SendWrapper;
 use smithay::{
-	backend::renderer::utils::{import_surface_tree, on_commit_buffer_handler},
+	backend::renderer::utils::{
+		import_surface_tree, on_commit_buffer_handler, RendererSurfaceStateUserData,
+	},
 	delegate_compositor,
-	wayland::compositor::CompositorHandler,
+	wayland::compositor::{self, CompositorHandler},
 };
 
 impl CompositorHandler for WaylandState {
@@ -17,6 +20,20 @@ impl CompositorHandler for WaylandState {
 	) {
 		on_commit_buffer_handler(surface);
 		import_surface_tree(&mut self.renderer, surface, &self.log).unwrap();
+
+		compositor::with_states(surface, |data| {
+			if let Some(surface_states) = data.data_map.get::<RendererSurfaceStateUserData>() {
+				if let Some(core_surface) = data.data_map.get::<CoreSurface>() {
+					core_surface.wl_tex.replace(
+						surface_states
+							.borrow()
+							.texture(&self.renderer)
+							.cloned()
+							.map(|renderer| SendWrapper::new(renderer)),
+					);
+				}
+			}
+		});
 	}
 }
 

src/wayland/mod.rs

@@ -1,26 +1,55 @@
 pub mod compositor;
+pub mod surface;
 pub mod xdg_decoration;
 mod xdg_shell;
-use anyhow::Result;
+use std::{ffi::c_void, sync::Arc};
+
+use anyhow::{ensure, Result};
+use parking_lot::Mutex;
 use slog::Logger;
 use smithay::{
-	backend::renderer::gles2::Gles2Renderer,
+	backend::{egl::EGLContext, renderer::gles2::Gles2Renderer},
 	delegate_output, delegate_shm,
+	desktop::utils::send_frames_surface_tree,
 	reexports::wayland_server::{
 		backend::{ClientData, ClientId, DisconnectReason},
 		protocol::wl_output::Subpixel,
-		Display, DisplayHandle,
+		Display, DisplayHandle, ListeningSocket,
 	},
 	utils::Size,
 	wayland::{
 		buffer::BufferHandler,
-		compositor::CompositorState,
+		compositor::{with_states, CompositorState},
 		output::{Output, OutputManagerState, Scale::Integer},
 		seat::SeatState,
 		shell::xdg::{decoration::XdgDecorationState, XdgShellState},
 		shm::{ShmHandler, ShmState},
 	},
 };
+use stereokit as sk;
+use stereokit::StereoKit;
+use surface::CoreSurface;
+
+struct EGLRawHandles {
+	display: *const c_void,
+	config: *const c_void,
+	context: *const c_void,
+}
+fn get_sk_egl() -> Result<EGLRawHandles> {
+	ensure!(
+		unsafe { sk::sys::backend_graphics_get() }
+			== sk::sys::backend_graphics__backend_graphics_opengles_egl,
+		"StereoKit is not running using EGL!"
+	);
+
+	Ok(unsafe {
+		EGLRawHandles {
+			display: sk::sys::backend_opengl_egl_get_display() as *const c_void,
+			config: sk::sys::backend_opengl_egl_get_config() as *const c_void,
+			context: sk::sys::backend_opengl_egl_get_context() as *const c_void,
+		}
+	})
+}
 
 pub struct ClientState;
 impl ClientData for ClientState {
@@ -39,7 +68,9 @@ impl ClientData for ClientState {
 pub struct WaylandState {
 	pub log: slog::Logger,
 
+	pub display: Arc<Mutex<Display<WaylandState>>>,
 	pub display_handle: DisplayHandle,
+	pub socket: ListeningSocket,
 	pub renderer: Gles2Renderer,
 	pub compositor_state: CompositorState,
 	pub xdg_shell_state: XdgShellState,
@@ -52,11 +83,25 @@ pub struct WaylandState {
 }
 
 impl WaylandState {
-	pub fn new(
+	pub fn new(log: Logger) -> Result<Self> {
-		display: &Display<WaylandState>,
+		let egl_raw_handles = get_sk_egl()?;
-		renderer: Gles2Renderer,
+		let renderer = unsafe {
-		log: Logger,
+			Gles2Renderer::new(
-	) -> Result<Self> {
+				EGLContext::from_raw(
+					egl_raw_handles.display,
+					egl_raw_handles.config,
+					egl_raw_handles.context,
+					log.clone(),
+				)?,
+				log.clone(),
+			)?
+		};
+
+		let display: Display<WaylandState> = Display::new()?;
+		let socket = ListeningSocket::bind_auto("wayland", 0..33)?;
+		if let Some(socket_name) = socket.socket_name() {
+			println!("Wayland compositor {:?} active", socket_name);
+		}
 		let display_handle = display.handle();
 
 		let compositor_state = CompositorState::new::<Self, _>(&display_handle, log.clone());
@@ -79,9 +124,12 @@ impl WaylandState {
 		let seat_state = SeatState::new();
 		// let data_device_state = DataDeviceState::new(&dh, log.clone());
 
+		println!("Init Wayland compositor");
 		Ok(WaylandState {
 			log,
+			display: Arc::new(Mutex::new(display)),
 			display_handle,
+			socket,
 			renderer,
 			compositor_state,
 			xdg_shell_state,
@@ -93,6 +141,46 @@ impl WaylandState {
 			// data_device_state,
 		})
 	}
+
+	pub fn frame(&mut self, sk: &StereoKit) {
+		let display_clone = self.display.clone();
+		let mut display = display_clone.lock();
+		if let Ok(Some(client)) = self.socket.accept() {
+			let _ = display
+				.handle()
+				.insert_client(client, Arc::new(ClientState));
+		}
+		display.dispatch_clients(self).unwrap();
+		display.flush_clients().unwrap();
+
+		drop(display);
+		drop(display_clone);
+
+		let time_ms = (sk.time_getf() * 1000.) as u32;
+		self.xdg_shell_state.toplevel_surfaces(|surfs| {
+			for surf in surfs.iter() {
+				with_states(surf.wl_surface(), |data| {
+					let core_surface = data.data_map.get::<CoreSurface>().unwrap();
+					core_surface.update_tex(sk);
+				});
+				send_frames_surface_tree(surf.wl_surface(), time_ms);
+			}
+		});
+		self.xdg_shell_state.popup_surfaces(|surfs| {
+			for surf in surfs.iter() {
+				with_states(surf.wl_surface(), |data| {
+					let core_surface = data.data_map.get::<CoreSurface>().unwrap();
+					core_surface.update_tex(sk);
+				});
+				send_frames_surface_tree(surf.wl_surface(), time_ms);
+			}
+		});
+	}
+}
+impl Drop for WaylandState {
+	fn drop(&mut self) {
+		println!("Cleanly shut down the Wayland compositor");
+	}
 }
 impl BufferHandler for WaylandState {
 	fn buffer_destroyed(

src/wayland/shader_unlit_gamma.hlsl

@@ -0,0 +1,39 @@
+#include "stereokit.hlsli"
+
+//--name = sk/unlit
+//--diffuse     = white
+//--uv_offset   = 0.0, 0.0
+//--uv_scale    = 1.0, 1.0
+Texture2D    diffuse   : register(t0);
+SamplerState diffuse_s : register(s0);
+float2       uv_scale;
+float2       uv_offset;
+
+struct vsIn {
+	float4 pos  : SV_Position;
+	float3 norm : NORMAL0;
+	float2 uv   : TEXCOORD0;
+};
+struct psIn {
+	float4 pos   : SV_POSITION;
+	float2 uv    : TEXCOORD0;
+	uint view_id : SV_RenderTargetArrayIndex;
+};
+
+psIn vs(vsIn input, uint id : SV_InstanceID) {
+	psIn o;
+	o.view_id = id % sk_view_count;
+	id        = id / sk_view_count;
+
+	float3 world = mul(float4(input.pos.xyz, 1), sk_inst[id].world).xyz;
+	o.pos        = mul(float4(world,         1), sk_viewproj[o.view_id]);
+
+	o.uv    = (input.uv) + uv_offset * uv_scale;
+	return o;
+}
+float4 ps(psIn input) : SV_TARGET {
+	float4 col = diffuse.Sample(diffuse_s, input.uv);
+	col.rgb = pow(col.rgb, float3(2.2));
+
+	return col; 
+}

src/wayland/shader_unlit_simula.hlsl

@@ -0,0 +1,111 @@
+#include "stereokit.hlsli"
+
+// Port of https://github.com/SimulaVR/Simula/blob/master/addons/godot-haskell-plugin/TextShader.tres to StereoKit and HLSL.
+
+//--name = stardust/text_shader
+//--diffuse     = white
+//--fcFactor    = 1.0
+//--ripple      = 4.0
+//--size        = 256, 256
+//--uv_offset   = 0.0, 0.0
+//--uv_scale    = 1.0, 1.0
+Texture2D    diffuse   : register(t0);
+SamplerState diffuse_s : register(s0);
+int2         size;
+float        fcFactor;
+float        ripple;
+float2       uv_scale;
+float2       uv_offset;
+
+struct vsIn {
+	float4 pos  : SV_Position;
+	float3 norm : NORMAL0;
+	float2 uv   : TEXCOORD0;
+};
+struct psIn {
+	float4 pos   : SV_POSITION;
+	float2 uv    : TEXCOORD0;
+	uint view_id : SV_RenderTargetArrayIndex;
+};
+
+psIn vs(vsIn input, uint id : SV_InstanceID) {
+	psIn o;
+	o.view_id = id % sk_view_count;
+	id        = id / sk_view_count;
+
+	float3 world = mul(float4(input.pos.xyz, 1), sk_inst[id].world).xyz;
+	o.pos        = mul(float4(world,         1), sk_viewproj[o.view_id]);
+
+	o.uv    = (input.uv) + uv_offset * uv_scale;
+	return o;
+}
+
+// float gaussian(float x, float t) {
+// 	float PI = 3.14159265358;
+// 	return exp(-x*x/(2.0 * t*t))/(sqrt(2.0*PI)*t);
+// }
+
+float besselI0(float x) {
+	return 1.0 + pow(x, 2.0) * (0.25 +  pow(x, 2.0) * (0.015625 +  pow(x, 2.0) * (0.000434028 +  pow(x, 2.0) * (6.78168e-6 +  pow(x, 2.0) * (6.78168e-8 +  pow(x, 2.0) * (4.7095e-10 +  pow(x, 2.0) * (2.40281e-12 + pow(x, 2.0) * (9.38597e-15 + pow(x, 2.0) * (2.8969e-17 + 7.24226e-20 * pow(x, 2.0))))))))));
+}
+
+float kaiser(float x, float alpha) {
+	if (x > 1.0) { 
+		return 0.0;
+	}
+	return besselI0(alpha * sqrt(1.0-x*x));
+}
+
+float4 lowpassFilter(Texture2D tex, sampler2D texSampler, float2 uv, float alpha) {
+	float PI = 3.14159265358;
+	
+	float4 q = float4(0.0);
+	
+	float2 dx_uv = ddx(uv);
+	float2 dy_uv = ddy(uv);
+	//float width = sqrt(max(dot(dx_uv, dx_uv), dot(dy_uv, dy_uv)));
+	float2 width = abs(float2(dx_uv.x, dy_uv.y));
+	
+	float2 pixelWidth = floor(width * float2(size));
+	float2 aspectRatio = normalize(pixelWidth);
+	
+	float2 xyf = uv * float2(size);
+	int2 xy = int2(xyf);
+	
+	pixelWidth = clamp(pixelWidth, float2(1.0), float2(2.0));
+
+	int2 start = xy - int2(pixelWidth);
+	int2 end = xy + int2(pixelWidth);
+	
+	float4 outColor = float4(0.0);
+	
+	float qSum = 0.0;
+	
+	for (int v = start.y; v <= end.y; v++) {
+		for (int u = start.x; u <= end.x; u++) {
+			float kx = fcFactor * (xyf.x - float(u))/pixelWidth.x;
+			float ky = fcFactor * (xyf.y - float(v))/pixelWidth.y;
+			 
+			//float lanczosValue = gaussian(kx, fcx);
+			float lanczosValue = kaiser(sqrt(kx*kx + ky*ky), alpha);
+			
+			q += tex.Sample(texSampler, (float2(u, v)+float2(0.5))/float2(size)) * lanczosValue;
+			// q += tex.Load(int3(u, v, 0)) * lanczosValue;
+			qSum += lanczosValue;
+		}
+	}
+	
+	return q/qSum;
+}
+
+float4 ps(psIn input) : SV_TARGET {
+	float gamma = 2.2;
+	// float4 col = diffuse.Sample(diffuse_s, input.uv);
+
+	// float4 col = lowpassFilter(diffuse, diffuse_s, diffuse_i.xy, float2(1.0 - input.uv.x, input.uv.y), ripple);
+	float4 col = lowpassFilter(diffuse, diffuse_s, input.uv, ripple);
+	// float4 col = diffuse.Sample(diffuse_s, input.uv);
+	col.rgb = pow(col.rgb, float3(gamma));
+
+	return col; 
+}

src/wayland/surface.rs

@@ -0,0 +1,67 @@
+use std::{cell::RefCell, fmt::Error};
+
+use glam::vec2;
+use once_cell::sync::OnceCell;
+use send_wrapper::SendWrapper;
+use smithay::backend::renderer::{gles2::Gles2Texture, Texture};
+use stereokit::{
+	material::Material,
+	shader::Shader,
+	texture::{Texture as SKTexture, TextureAddress, TextureFormat, TextureSample, TextureType},
+	StereoKit,
+};
+
+// const GAMMA_SHADER_BYTES: &[u8] = include_bytes!("shader_unlit_gamma.sks");
+const SIMULA_SHADER_BYTES: &[u8] = include_bytes!("shader_unlit_simula.sks");
+
+pub struct CoreSurface {
+	pub(crate) wl_tex: RefCell<Option<SendWrapper<Gles2Texture>>>,
+	sk_tex: OnceCell<SKTexture>,
+	sk_mat: OnceCell<Material>,
+}
+
+impl CoreSurface {
+	pub fn new() -> Self {
+		CoreSurface {
+			wl_tex: RefCell::new(None),
+			sk_tex: OnceCell::new(),
+			sk_mat: OnceCell::new(),
+		}
+	}
+
+	pub fn update_tex(&self, sk: &StereoKit) {
+		let sk_tex = self
+			.sk_tex
+			.get_or_try_init(|| {
+				SKTexture::create(sk, TextureType::ImageNoMips, TextureFormat::RGBA32).ok_or(Error)
+			})
+			.unwrap();
+		let sk_mat = self
+			.sk_mat
+			.get_or_try_init(|| {
+				let shader = Shader::from_mem(sk, SIMULA_SHADER_BYTES).unwrap();
+				Material::create(sk, &shader).ok_or(Error).map(|mat| {
+					mat.set_parameter("diffuse", self.sk_tex.get().unwrap());
+					mat
+				})
+			})
+			.unwrap();
+		if let Some(smithay_tex) = self.wl_tex.borrow().as_ref() {
+			unsafe {
+				sk_tex.set_native(
+					smithay_tex.tex_id() as usize,
+					smithay::backend::renderer::gles2::ffi::RGBA8.into(),
+					TextureType::Image,
+					smithay_tex.width(),
+					smithay_tex.height(),
+					false,
+				);
+				let size: mint::Vector2<f32> =
+					vec2(smithay_tex.width() as f32, smithay_tex.height() as f32).into();
+				sk_mat.set_parameter("size", &size);
+				sk_tex.set_sample(TextureSample::Point);
+				sk_tex.set_address_mode(TextureAddress::Clamp);
+			}
+		}
+	}
+}

src/wayland/xdg_shell.rs

@@ -4,10 +4,10 @@ use smithay::{
 		decoration::zv1::server::zxdg_toplevel_decoration_v1::Mode,
 		shell::server::xdg_toplevel::State,
 	},
-	wayland::shell::xdg::XdgShellHandler,
+	wayland::{compositor, shell::xdg::XdgShellHandler},
 };
 
-use super::WaylandState;
+use super::{surface::CoreSurface, WaylandState};
 
 impl XdgShellHandler for WaylandState {
 	fn xdg_shell_state(&mut self) -> &mut smithay::wayland::shell::xdg::XdgShellState {
@@ -26,6 +26,10 @@ impl XdgShellHandler for WaylandState {
 			state.decoration_mode = Some(Mode::ServerSide);
 		});
 		surface.send_configure();
+
+		compositor::with_states(surface.wl_surface(), |data| {
+			data.data_map.insert_if_missing(|| CoreSurface::new());
+		});
 	}
 
 	fn new_popup(
@@ -37,6 +41,10 @@ impl XdgShellHandler for WaylandState {
 		self.output
 			.enter(&self.display_handle, surface.wl_surface());
 		let _ = surface.send_configure();
+
+		compositor::with_states(surface.wl_surface(), |data| {
+			data.data_map.insert_if_missing(|| CoreSurface::new());
+		});
 	}
 
 	fn grab(